update
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"archives": [
|
||||
{
|
||||
"id": "metadata-vfs",
|
||||
"lines": 277,
|
||||
"path": "tests/results/manual/2026-08-08T2337Z-metadata-vfs/prepare-fixtures.sh",
|
||||
"sha256": "93618e0aa5528177a398b1923befc01397db70db24ceda0c32defe84728004fd"
|
||||
},
|
||||
{
|
||||
"id": "final-review",
|
||||
"lines": 271,
|
||||
"path": "tests/results/manual/2026-08-09T0124Z-final-review/prepare-fixtures.sh",
|
||||
"sha256": "e1ee03c5d845f6d3fb0107dce5952458b4f64d1112c7dafd2129aa01f12d978d"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"controls": {
|
||||
"cleanup-failure": ["RUNNER_FAIL", "runner", "FAIL", "0"],
|
||||
"dut-mismatch": ["DUT_FAIL", "dut", "PASS", "0"],
|
||||
"guest-command-failure": ["RUNNER_FAIL", "runner", "PASS", "0"],
|
||||
"known-good": ["PASS", "none", "PASS", "0"],
|
||||
"owned-cleanup": ["PASS", "none", "PASS", "0"],
|
||||
"qemu-early-exit": ["INFRA_BLOCKED", "infrastructure", "PASS", "0"],
|
||||
"ssh-failure": ["INFRA_BLOCKED", "infrastructure", "PASS", "0"],
|
||||
"timeout": ["RUNNER_FAIL", "runner", "PASS", "1"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"base_index": 1,
|
||||
"infix": "repo.pre15.legacy.",
|
||||
"prefix_start": 2048,
|
||||
"positive": [
|
||||
{
|
||||
"name": "legacy-primary",
|
||||
"plain": false,
|
||||
"carrier": "primary-legacy"
|
||||
},
|
||||
{
|
||||
"name": "explicit-plain",
|
||||
"plain": true,
|
||||
"carrier": "primary"
|
||||
},
|
||||
{
|
||||
"name": "packed-carrier",
|
||||
"plain": false,
|
||||
"carrier": "packed"
|
||||
},
|
||||
{
|
||||
"name": "metabox-carrier",
|
||||
"plain": false,
|
||||
"carrier": "metabox"
|
||||
}
|
||||
],
|
||||
"negative": [
|
||||
{
|
||||
"name": "legacy-length-invalid",
|
||||
"field": "record_length",
|
||||
"value": 257,
|
||||
"expected_errno": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"name": "legacy-offset-outside",
|
||||
"field": "prefix_start",
|
||||
"value": 8192,
|
||||
"expected_errno": "EINTEGRITY"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"map_blocks_sha256": "ae883c958dfad3a3df9c6950af0e24723c64523908b901d400c2a3c5eeaf515e",
|
||||
"freebsd_backend_function_sha256": {
|
||||
"data.c:erofs_map_dev": "de77c2b1f5f12a6a7a9bc0414c42bf1ce1745a60331b0abed8c8df87f5b7dcd7",
|
||||
"data.c:erofs_bread_device": "c274cd0274ecf79ceea214e123c9d70f3e1f41c6a56449470be554a291ffa808",
|
||||
"data.c:erofs_bread": "74ae2446f1ce8629642402ff63b83d3258d768fdc20dc2196643e3e1bd0382a3",
|
||||
"data.c:erofs_read_physical": "aa33e6fcf99ebf46c1eda54da4308ca160a4948f5afbb14038630ce7ce964e09",
|
||||
"data.c:erofs_brelse": "1d50316f726878cf3390e92eaf6c27072ca8fccc3efceb14fde11f796886e1c5",
|
||||
"super.c:erofs_release_device_info": "6504bb87bb1c0f5e8bbaba70ab093053cfef2850c6f36eae588603277068b185",
|
||||
"super.c:erofs_init_device": "b1cb3b21a2b2b581fc4924e361f4b763bd97a6a909c2c081cfa0861dc837b0c1"
|
||||
},
|
||||
"consumers": ["compressed", "inode", "map", "plain", "super", "xattr"],
|
||||
"direct_metadata_functions": [
|
||||
"data.c:erofs_map_blocks_chunk",
|
||||
"data.c:erofs_read_data",
|
||||
"data.c:erofs_read_uio",
|
||||
"decompressor.c:z_erofs_read_cfg",
|
||||
"inode.c:erofs_read_inode",
|
||||
"super.c:erofs_load_generation_seed",
|
||||
"super.c:erofs_read_superblock",
|
||||
"super.c:erofs_scan_devices",
|
||||
"super.c:erofs_superblock_csum_verify",
|
||||
"xattr.c:erofs_xattr_read_backing",
|
||||
"zdata.c:z_erofs_read_extent",
|
||||
"zmap.c:z_erofs_fill_inode",
|
||||
"zmap.c:z_erofs_map_blocks_ext",
|
||||
"zmap.c:z_erofs_read_extent",
|
||||
"zmap.c:z_erofs_read_index"
|
||||
],
|
||||
"helper_paths": [
|
||||
"helper\tprimary-success\t1\t1",
|
||||
"helper\tmetabox-success\t1\t1",
|
||||
"helper\tprovider-error\t0\t0",
|
||||
"helper\toffset-overflow\t0\t0",
|
||||
"helper\tnull-release\t0\t0"
|
||||
],
|
||||
"function_metrics": {
|
||||
"data.c:erofs_map_blocks_chunk": [1, 1, 0, 0],
|
||||
"data.c:erofs_read_data": [1, 1, 1, 0],
|
||||
"data.c:erofs_read_metadata": [0, 0, 0, 1],
|
||||
"data.c:erofs_read_uio": [1, 1, 1, 0],
|
||||
"decompressor.c:z_erofs_parse_cfgs": [1, 1, 0, 0],
|
||||
"decompressor.c:z_erofs_read_cfg": [1, 1, 0, 0],
|
||||
"inode.c:erofs_read_inode": [1, 18, 0, 0],
|
||||
"super.c:erofs_load_generation_seed": [1, 1, 0, 0],
|
||||
"super.c:erofs_read_superblock": [1, 1, 0, 0],
|
||||
"super.c:erofs_scan_devices": [1, 2, 0, 0],
|
||||
"super.c:erofs_superblock_csum_verify": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_getxattr": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_inode_has_noacl": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_listxattr": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_xattr_iter_shared": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_xattr_load_body": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_xattr_load_shared_entry": [1, 1, 0, 0],
|
||||
"xattr.c:erofs_xattr_prefixes_init": [1, 2, 0, 0],
|
||||
"xattr.c:erofs_xattr_read_backing": [0, 0, 0, 1],
|
||||
"xattr.c:erofs_xattr_read_metadata": [1, 1, 0, 0],
|
||||
"zdata.c:z_erofs_do_read": [0, 0, 1, 0],
|
||||
"zdata.c:z_erofs_read_extent": [1, 1, 1, 0],
|
||||
"zmap.c:z_erofs_fill_inode": [1, 6, 0, 0],
|
||||
"zmap.c:z_erofs_load_compact_lcluster": [1, 7, 0, 0],
|
||||
"zmap.c:z_erofs_load_full_lcluster": [1, 2, 0, 0],
|
||||
"zmap.c:z_erofs_map_blocks_ext": [1, 1, 0, 0],
|
||||
"zmap.c:z_erofs_read_extent": [1, 1, 0, 0],
|
||||
"zmap.c:z_erofs_read_index": [0, 0, 0, 0]
|
||||
},
|
||||
"metric_order": [
|
||||
"object_local_count",
|
||||
"put_count",
|
||||
"legacy_release_count",
|
||||
"raw_bread_count"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"base": "edf098905a34de764185e72fc7e92d7b8f4e7285",
|
||||
"fields": [
|
||||
"m_la",
|
||||
"m_pa",
|
||||
"m_llen",
|
||||
"m_plen",
|
||||
"m_deviceid",
|
||||
"m_flags",
|
||||
"errno",
|
||||
"acquire_count",
|
||||
"release_count"
|
||||
],
|
||||
"records": [
|
||||
{"id": "plain-block-edge", "m_la": 4095, "m_pa": 135167, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "inline-tail", "m_la": 4500, "m_pa": 8692, "m_llen": 500, "m_plen": 500, "m_deviceid": 0, "m_flags": 3, "errno": 0, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "plain-hole", "m_la": 512, "m_pa": 0, "m_llen": 3584, "m_plen": 3584, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "chunk-index", "m_la": 9000, "m_pa": 316200, "m_llen": 7384, "m_plen": 7384, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "chunk-48bit-device2", "m_la": 12345, "m_pa": 17592186069049, "m_llen": 4039, "m_plen": 4039, "m_deviceid": 2, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "chunk-index-out-of-provider", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 97, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "chunk-shift-overflow-after-acquire", "m_la": 0, "m_pa": 0, "m_llen": 4096, "m_plen": 4096, "m_deviceid": 0, "m_flags": 0, "errno": 84, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "chunk-index-short-read", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "explicit-compressed", "m_la": 0, "m_pa": 74565, "m_llen": 4096, "m_plen": 2048, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "explicit-partial-reference", "m_la": 0, "m_pa": 262144, "m_llen": 4096, "m_plen": 1024, "m_deviceid": 0, "m_flags": 9, "errno": 0, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "explicit-fragment-tail", "m_la": 4096, "m_pa": 0, "m_llen": 2048, "m_plen": 8192, "m_deviceid": 0, "m_flags": 16, "errno": 0, "acquire_count": 1, "release_count": 1},
|
||||
{"id": "compressed-post-eof", "m_la": 4096, "m_pa": 0, "m_llen": 18, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "compressed-index-short-read", "m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0},
|
||||
{"id": "compressed-sanity-error-after-acquire", "m_la": 0, "m_pa": 74565, "m_llen": 0, "m_plen": 1024, "m_deviceid": 0, "m_flags": 1, "errno": 45, "acquire_count": 1, "release_count": 1}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"baseline": "6673f51152a5195a8a8903aa801f820abce7936e",
|
||||
"coverage": [
|
||||
"bounds",
|
||||
"chunk",
|
||||
"compressed",
|
||||
"fragment",
|
||||
"hole",
|
||||
"inline",
|
||||
"invalid",
|
||||
"multidevice",
|
||||
"overflow",
|
||||
"partial-reference",
|
||||
"plain",
|
||||
"post-EOF"
|
||||
],
|
||||
"map_case_count": 80,
|
||||
"model_sha256": "effcf0b6cc6a6e916eae89a14ad52d7b96273a453556647b6eabf2926fcb4abb",
|
||||
"records": [
|
||||
{"id": "plain-start", "tuple_hex": "0000000000000000000002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "plain-block-last", "tuple_hex": "ff0f000000000000ff0f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "plain-second-start", "tuple_hex": "0010000000000000001002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "plain-final-byte", "tuple_hex": "ff2f000000000000ff2f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "plain-eof", "tuple_hex": "0030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "plain-post-eof", "tuple_hex": "1130000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "plain-empty", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "plain-hole-start", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "plain-hole-edge", "tuple_hex": "ff0f000000000000000000000000000001000000000000000100000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "plain-shift-overflow", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "plain-add-overflow", "tuple_hex": "0010000000000000000000000000000000100000000000000010000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "plain-invalid-layout", "tuple_hex": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000002d0000000000000000000000"},
|
||||
{"id": "inline-head-start", "tuple_hex": "0000000000000000008002000000000000100000000000000010000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "inline-head-last", "tuple_hex": "ff0f000000000000ff8f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "inline-tail-start", "tuple_hex": "0010000000000000602000000000000088030000000000008803000000000000000000000300000000000000000000000000000000000000"},
|
||||
{"id": "inline-tail-middle", "tuple_hex": "9411000000000000f421000000000000f401000000000000f401000000000000000000000300000000000000000000000000000000000000"},
|
||||
{"id": "inline-final-byte", "tuple_hex": "8713000000000000e72300000000000001000000000000000100000000000000000000000300000000000000000000000000000000000000"},
|
||||
{"id": "inline-eof", "tuple_hex": "8813000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "inline-head-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "inline-one-block", "tuple_hex": "0000000000000000602000000000000000100000000000000010000000000000000000000300000000000000000000000000000000000000"},
|
||||
{"id": "inline-one-byte-tail-head", "tuple_hex": "ff0f000000000000ff8f02000000000001000000000000000100000000000000000000000100000000000000000000000000000000000000"},
|
||||
{"id": "inline-one-byte-tail", "tuple_hex": "0010000000000000602000000000000001000000000000000100000000000000000000000300000000000000000000000000000000000000"},
|
||||
{"id": "inline-inode-add-overflow", "tuple_hex": "0010000000000000200000000000000088030000000000008803000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "inline-xattr-add-overflow", "tuple_hex": "0010000000000000310000000000000088030000000000008803000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "inline-offset-add-overflow", "tuple_hex": "7017000000000000c70300000000000090080000000000009008000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "chunk-raw32-start", "tuple_hex": "000000000000000000d004000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-raw32-last", "tuple_hex": "ff0f000000000000ffdf04000000000001000000000000000100000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-raw32-next", "tuple_hex": "001000000000000000e004000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-raw32-final", "tuple_hex": "0f270000000000000ff704000000000001000000000000000100000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-raw32-hole", "tuple_hex": "00020000000000000000000000000000000e000000000000000e000000000000000000000000000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index32-mask", "tuple_hex": "000000000000000000d004000000000000100000000000000010000000000000030000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index32-middle", "tuple_hex": "282300000000000028d3040000000000d81c000000000000d81c000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index32-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index48-start", "tuple_hex": "0000000000000000005000000010000000100000000000000010000000000000020000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index48-middle", "tuple_hex": "39300000000000003960000000100000c70f000000000000c70f000000000000020000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-index48-hole", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"},
|
||||
{"id": "chunk-device-mask-zero", "tuple_hex": "0000000000000000001005000000000000100000000000000010000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "chunk-eof", "tuple_hex": "0010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "chunk-inode-add-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"},
|
||||
{"id": "chunk-xattr-add-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"},
|
||||
{"id": "chunk-align-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"},
|
||||
{"id": "chunk-index-multiply-overflow", "tuple_hex": "feffffffffffffff000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"},
|
||||
{"id": "chunk-image-size-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000540000000000000000000000"},
|
||||
{"id": "chunk-primary-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "chunk-primary-tail-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "chunk-metabox-missing", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "chunk-metabox-bounds", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "chunk-reader-short", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000"},
|
||||
{"id": "chunk-shift-overflow", "tuple_hex": "0000000000000000000000000000000000100000000000000010000000000000000000000000000000000000540000000100000001000000"},
|
||||
{"id": "chunk-offset-overflow", "tuple_hex": "0010000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000100000001000000"},
|
||||
{"id": "extent4-start", "tuple_hex": "0000000000000000000001000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent4-first-last", "tuple_hex": "0000000000000000000001000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent4-second-start", "tuple_hex": "0010000000000000000401000000000000100000000000000008000000000000000000000100000000000000000000000300000003000000"},
|
||||
{"id": "extent4-second-middle", "tuple_hex": "0010000000000000000401000000000000100000000000000008000000000000000000000100000000000000000000000300000003000000"},
|
||||
{"id": "extent8-start", "tuple_hex": "0000000000000000000002000000000000100000000000000004000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "extent8-second-start", "tuple_hex": "0010000000000000000003000000000000100000000000000008000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "extent8-hole", "tuple_hex": "0010000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000100000001000000"},
|
||||
{"id": "extent16-start", "tuple_hex": "0000000000000000000004000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent16-first-last", "tuple_hex": "0000000000000000000004000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent16-second-start", "tuple_hex": "0010000000000000000005000000000000100000000000000008000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent16-second-middle", "tuple_hex": "0010000000000000000005000000000000100000000000000008000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent16-third-start", "tuple_hex": "0020000000000000000006000000000000100000000000000004000000000000000000000100000000000000000000000200000002000000"},
|
||||
{"id": "extent16-physical-48bit", "tuple_hex": "0000000000000000090000000001000000100000000000000004000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "extent32-logical-64bit", "tuple_hex": "0000000001000000000008000000000000100000000000000008000000000000000000000100000000000000000000000100000001000000"},
|
||||
{"id": "extent-partial-reference", "tuple_hex": "0000000000000000000009000000000000100000000000000004000000000000000000000900000000000000000000000100000001000000"},
|
||||
{"id": "extent-fragment-tail", "tuple_hex": "0010000000000000000000000000000000080000000000000020000000000000000000001000000000000000000000000100000001000000"},
|
||||
{"id": "extent-interlaced-shifted", "tuple_hex": "000000000000000000000a000000000000100000000000000010000000000000000000000100000005000000000000000100000001000000"},
|
||||
{"id": "extent-algorithm-one", "tuple_hex": "000000000000000000000b000000000000100000000000000004000000000000000000000100000001000000000000000100000001000000"},
|
||||
{"id": "extent-algorithm-unavailable", "tuple_hex": "000000000000000000000b000000000000000000000000000004000000000000000000000100000001000000610000000100000001000000"},
|
||||
{"id": "extent-algorithm-runtime-invalid", "tuple_hex": "000000000000000000000c0000000000000000000000000000040000000000000000000001000000060000002d0000000100000001000000"},
|
||||
{"id": "extent-plen-too-large", "tuple_hex": "000000000000000000000d0000000000000000000000000001001000000000000000000001000000040000002d0000000100000001000000"},
|
||||
{"id": "extent-full-short-logical", "tuple_hex": "000000000000000000000e000000000000000000000000000020000000000000000000000100000000000000610000000100000001000000"},
|
||||
{"id": "extent-pa-add-overflow", "tuple_hex": "000000000000000000feffffffffffff00000000000000000004000000000000000000000100000000000000610000000100000001000000"},
|
||||
{"id": "extent-physical-48bit-limit", "tuple_hex": "0000000000000000000000000000001000000000000000000004000000000000000000000100000000000000610000000100000001000000"},
|
||||
{"id": "extent-reader-short", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000"},
|
||||
{"id": "extent-table-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "extent-record-multiply-overflow", "tuple_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000610000000000000000000000"},
|
||||
{"id": "extent-fragment-bounds", "tuple_hex": "0010000000000000000000000000000000000000000000000020000000000000000000001000000000000000610000000100000001000000"},
|
||||
{"id": "compressed-eof", "tuple_hex": "0010000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{"id": "compressed-post-eof", "tuple_hex": "0010000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000000000000000000000"}
|
||||
],
|
||||
"schema": 1,
|
||||
"subcategories": [
|
||||
"chunk-bounds",
|
||||
"chunk-holes",
|
||||
"chunk-index32",
|
||||
"chunk-index48",
|
||||
"chunk-overflow",
|
||||
"chunk-raw32",
|
||||
"cleanup-after-acquire",
|
||||
"cleanup-before-acquire",
|
||||
"compressed-bounds",
|
||||
"compressed-flags",
|
||||
"compressed-invalid",
|
||||
"compressed-overflow",
|
||||
"device-mask",
|
||||
"device-resolution",
|
||||
"extent-16",
|
||||
"extent-32",
|
||||
"extent-4",
|
||||
"extent-8",
|
||||
"inline-boundaries",
|
||||
"inline-overflow",
|
||||
"plain-boundaries",
|
||||
"plain-overflow"
|
||||
],
|
||||
"tuple_byte_layout": "<QQQQH2xIiiII",
|
||||
"tuple_bytes_sha256": "8d3fec7ccbdfb7f7c851a494b6ecad02b8a019247bd9486188f8d37c169faedd"
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"candidate": "P15-074",
|
||||
"harness_maxphys": 16384,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "read-data-mapped-over-maxphys",
|
||||
"api": "read_data",
|
||||
"mapping": "mapped",
|
||||
"inode_size": 50000,
|
||||
"offset": 0,
|
||||
"length": 32785,
|
||||
"expected_errno": 0,
|
||||
"expected_map_calls": 3,
|
||||
"expected_physical_offsets": [1048576, 1064960, 1081344],
|
||||
"expected_physical_lengths": [16384, 16384, 17]
|
||||
},
|
||||
{
|
||||
"id": "read-data-hole-over-maxphys",
|
||||
"api": "read_data",
|
||||
"mapping": "hole",
|
||||
"inode_size": 50000,
|
||||
"offset": 11,
|
||||
"length": 32771,
|
||||
"expected_errno": 0,
|
||||
"expected_map_calls": 3,
|
||||
"expected_physical_offsets": [],
|
||||
"expected_physical_lengths": []
|
||||
},
|
||||
{
|
||||
"id": "read-data-overflow",
|
||||
"api": "read_data",
|
||||
"mapping": "mapped",
|
||||
"inode_size": 18446744073709551615,
|
||||
"offset": 18446744073709551612,
|
||||
"length": 8,
|
||||
"expected_errno": 84,
|
||||
"expected_map_calls": 0,
|
||||
"expected_physical_offsets": [],
|
||||
"expected_physical_lengths": []
|
||||
},
|
||||
{
|
||||
"id": "read-uio-mapped-over-maxphys",
|
||||
"api": "read_uio",
|
||||
"mapping": "mapped",
|
||||
"inode_size": 50000,
|
||||
"offset": 7,
|
||||
"length": 16403,
|
||||
"expected_errno": 0,
|
||||
"expected_map_calls": 2,
|
||||
"expected_final_offset": 16410,
|
||||
"expected_resid": 0,
|
||||
"expected_physical_offsets": [1048583, 1064967],
|
||||
"expected_physical_lengths": [16384, 19]
|
||||
},
|
||||
{
|
||||
"id": "read-uio-short",
|
||||
"api": "read_uio",
|
||||
"mapping": "mapped",
|
||||
"inode_size": 50000,
|
||||
"offset": 123,
|
||||
"length": 9,
|
||||
"expected_errno": 0,
|
||||
"expected_map_calls": 1,
|
||||
"expected_final_offset": 132,
|
||||
"expected_resid": 0,
|
||||
"expected_physical_offsets": [1048699],
|
||||
"expected_physical_lengths": [9]
|
||||
},
|
||||
{
|
||||
"id": "read-uio-eof",
|
||||
"api": "read_uio",
|
||||
"mapping": "mapped",
|
||||
"inode_size": 50000,
|
||||
"offset": 50000,
|
||||
"length": 37,
|
||||
"expected_errno": 0,
|
||||
"expected_map_calls": 0,
|
||||
"expected_final_offset": 50000,
|
||||
"expected_resid": 37,
|
||||
"expected_physical_offsets": [],
|
||||
"expected_physical_lengths": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"candidate": "P15-074",
|
||||
"b07_oracle_sha256": "811578b398ea81cb4e944630d46a7be501b3e87fa771eb82dc3ca6e6d387841a",
|
||||
"b07_tuple_layout": "<QQQQH2xIiiII",
|
||||
"expected_transformed_tuple_sha256": "cad0ec3fd5c4b72e316fed5c70fe0d080d37c56d579913c50f887cc6ca5add8e",
|
||||
"b07_transforms": [
|
||||
{"id": "plain-start", "m_llen": 12288, "m_plen": 12288},
|
||||
{"id": "plain-block-last", "m_llen": 8193, "m_plen": 8193},
|
||||
{"id": "plain-second-start", "m_llen": 8192, "m_plen": 8192},
|
||||
{"id": "plain-hole-start", "m_llen": 12288, "m_plen": 12288},
|
||||
{"id": "plain-hole-edge", "m_llen": 8193, "m_plen": 8193},
|
||||
{"id": "plain-shift-overflow", "m_llen": 12288, "m_plen": 12288},
|
||||
{"id": "plain-add-overflow", "m_llen": 8192, "m_plen": 8192}
|
||||
],
|
||||
"map_cases": [
|
||||
{
|
||||
"id": "plain-full-run",
|
||||
"request": 0,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 32},
|
||||
"expected": {"m_la": 0, "m_pa": 131072, "m_llen": 20000, "m_plen": 20000, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-unaligned-run",
|
||||
"request": 4095,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 32},
|
||||
"expected": {"m_la": 4095, "m_pa": 135167, "m_llen": 15905, "m_plen": 15905, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-run-exceeds-test-maxphys",
|
||||
"request": 1,
|
||||
"inode": {"layout": 0, "size": 50000, "startblk": 32},
|
||||
"expected": {"m_la": 1, "m_pa": 131073, "m_llen": 49999, "m_plen": 49999, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-hole-full-run",
|
||||
"request": 257,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 4294967295},
|
||||
"expected": {"m_la": 257, "m_pa": 0, "m_llen": 19743, "m_plen": 19743, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-final-byte",
|
||||
"request": 19999,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 32},
|
||||
"expected": {"m_la": 19999, "m_pa": 151071, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-eof",
|
||||
"request": 20000,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 32},
|
||||
"expected": {"m_la": 20000, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-post-eof",
|
||||
"request": 20017,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 32},
|
||||
"expected": {"m_la": 20017, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-empty",
|
||||
"request": 0,
|
||||
"inode": {"layout": 0, "size": 0, "startblk": 32},
|
||||
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "plain-shift-overflow",
|
||||
"request": 0,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 4503599627370496},
|
||||
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 20000, "m_plen": 20000, "m_deviceid": 0, "m_flags": 0, "errno": 97}
|
||||
},
|
||||
{
|
||||
"id": "plain-add-overflow",
|
||||
"request": 4096,
|
||||
"inode": {"layout": 0, "size": 20000, "startblk": 4503599627370495},
|
||||
"expected": {"m_la": 4096, "m_pa": 0, "m_llen": 15904, "m_plen": 15904, "m_deviceid": 0, "m_flags": 0, "errno": 97}
|
||||
},
|
||||
{
|
||||
"id": "pre-inline-head-start",
|
||||
"request": 0,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 0, "m_pa": 163840, "m_llen": 12288, "m_plen": 12288, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "pre-inline-head-unaligned",
|
||||
"request": 4095,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 4095, "m_pa": 167935, "m_llen": 8193, "m_plen": 8193, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "pre-inline-head-second-block",
|
||||
"request": 4096,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 4096, "m_pa": 167936, "m_llen": 8192, "m_plen": 8192, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "pre-inline-head-final-byte",
|
||||
"request": 12287,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 12287, "m_pa": 176127, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "pre-inline-head-hole",
|
||||
"request": 1,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 4294967295, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 1, "m_pa": 0, "m_llen": 12287, "m_plen": 12287, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "inline-tail-start",
|
||||
"request": 12288,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 12288, "m_pa": 8288, "m_llen": 712, "m_plen": 712, "m_deviceid": 0, "m_flags": 3, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "inline-tail-middle",
|
||||
"request": 12500,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 12500, "m_pa": 8500, "m_llen": 500, "m_plen": 500, "m_deviceid": 0, "m_flags": 3, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "inline-tail-final-byte",
|
||||
"request": 12999,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 12999, "m_pa": 8999, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 3, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "inline-tail-after-hole-head",
|
||||
"request": 12288,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 4294967295, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 12288, "m_pa": 8288, "m_llen": 712, "m_plen": 712, "m_deviceid": 0, "m_flags": 3, "errno": 0}
|
||||
},
|
||||
{
|
||||
"id": "inline-eof",
|
||||
"request": 13000,
|
||||
"inode": {"layout": 2, "size": 13000, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32},
|
||||
"expected": {"m_la": 13000, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0}
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+464
@@ -0,0 +1,464 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
EINTEGRITY = "EINTEGRITY"
|
||||
PASS = "PASS"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", required=True, type=Path)
|
||||
parser.add_argument("--dut", required=True, type=Path)
|
||||
parser.add_argument("--baseline", required=True)
|
||||
parser.add_argument("--spec", required=True, type=Path)
|
||||
parser.add_argument("--report", required=True, type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def committed(root: Path, baseline: str, path: str) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(root), "show", f"{baseline}:{path}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"cannot read B10 baseline {path}: {completed.stderr}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def extract_function(source: str, name: str) -> str:
|
||||
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
|
||||
if not match:
|
||||
raise SystemExit(f"missing function: {name}")
|
||||
name_line = source.rfind("\n", 0, match.start()) + 1
|
||||
start = source.rfind("\n", 0, name_line - 1) + 1
|
||||
brace = source.find("{", match.end())
|
||||
if brace < 0:
|
||||
raise SystemExit(f"missing function body: {name}")
|
||||
depth = 0
|
||||
state = "code"
|
||||
index = brace
|
||||
while index < len(source):
|
||||
char = source[index]
|
||||
following = source[index + 1] if index + 1 < len(source) else ""
|
||||
if state == "code":
|
||||
if char == "/" and following == "*":
|
||||
state = "block"
|
||||
index += 2
|
||||
continue
|
||||
if char == "/" and following == "/":
|
||||
state = "line"
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
state = "string"
|
||||
elif char == "'":
|
||||
state = "character"
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start : index + 1]
|
||||
elif state == "block" and char == "*" and following == "/":
|
||||
state = "code"
|
||||
index += 2
|
||||
continue
|
||||
elif state == "line" and char == "\n":
|
||||
state = "code"
|
||||
elif state in {"string", "character"}:
|
||||
if char == "\\":
|
||||
index += 2
|
||||
continue
|
||||
if (state == "string" and char == '"') or (
|
||||
state == "character" and char == "'"
|
||||
):
|
||||
state = "code"
|
||||
index += 1
|
||||
raise SystemExit(f"unterminated function: {name}")
|
||||
|
||||
|
||||
def require_order(source: str, markers: list[str], label: str) -> None:
|
||||
position = -1
|
||||
for marker in markers:
|
||||
position = source.find(marker, position + 1)
|
||||
if position < 0:
|
||||
raise SystemExit(f"{label} is missing ordered marker: {marker}")
|
||||
|
||||
|
||||
def function_position(source: str, name: str) -> int:
|
||||
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
|
||||
if not match:
|
||||
raise SystemExit(f"missing function position: {name}")
|
||||
return match.start()
|
||||
|
||||
|
||||
def source_checks(root: Path, dut: Path, baseline: str) -> dict[str, object]:
|
||||
src = dut / "src"
|
||||
current = {
|
||||
name: (src / name).read_text(encoding="utf-8")
|
||||
for name in ("dir.c", "namei.c", "internal.h", "erofs_vnops.c")
|
||||
}
|
||||
base = {
|
||||
name: committed(root, baseline, f"repo-pre-15/src/{name}")
|
||||
for name in current
|
||||
}
|
||||
if current["dir.c"] == base["dir.c"] or current["namei.c"] == base["namei.c"]:
|
||||
raise SystemExit("B10 did not change both declared directory sources")
|
||||
if current["internal.h"] != base["internal.h"]:
|
||||
raise SystemExit("B10 added directory-wide state outside its write set")
|
||||
if current["erofs_vnops.c"] != base["erofs_vnops.c"]:
|
||||
raise SystemExit("B10 changed the FreeBSD VOP adapter")
|
||||
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(root),
|
||||
"diff",
|
||||
"--name-only",
|
||||
baseline,
|
||||
"--",
|
||||
"repo-pre-15/src",
|
||||
],
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
changed_sources = set(completed.stdout.splitlines())
|
||||
expected_sources = {
|
||||
"repo-pre-15/src/dir.c",
|
||||
"repo-pre-15/src/namei.c",
|
||||
}
|
||||
if changed_sources != expected_sources:
|
||||
raise SystemExit(f"unexpected B10 source write set: {sorted(changed_sources)}")
|
||||
|
||||
dir_source = current["dir.c"]
|
||||
namei_source = current["namei.c"]
|
||||
validator = extract_function(dir_source, "erofs_validate_dirblock")
|
||||
fill = extract_function(dir_source, "erofs_fill_dentries")
|
||||
readdir = extract_function(dir_source, "erofs_readdir_block")
|
||||
previous = extract_function(dir_source, "erofs_previous_dirname")
|
||||
read_block = extract_function(namei_source, "erofs_read_dirblock")
|
||||
neighbors = extract_function(namei_source, "erofs_validate_dirblock_neighbors")
|
||||
find_block = extract_function(namei_source, "erofs_find_target_block")
|
||||
namei = extract_function(namei_source, "erofs_namei")
|
||||
lookup = extract_function(namei_source, "erofs_lookup")
|
||||
|
||||
require_order(
|
||||
validator,
|
||||
["erofs_dirent_namelen", "erofs_dirname_order", "EINTEGRITY"],
|
||||
"block lexical validator",
|
||||
)
|
||||
require_order(
|
||||
fill,
|
||||
["erofs_dirent_name", "erofs_nid_is_valid", "bzero", "d_fileno", "erofs_uiodir"],
|
||||
"readdir publication boundary",
|
||||
)
|
||||
require_order(
|
||||
readdir,
|
||||
[
|
||||
"GENERIC_MINDIRSIZ",
|
||||
"(size_t)INT_MAX",
|
||||
"SIZE_MAX / sizeof(*cookiebuf)",
|
||||
"(int)cookie_count",
|
||||
],
|
||||
"cookie clamp",
|
||||
)
|
||||
if "uio->uio_resid / 8" in readdir or "MAX(1" in readdir:
|
||||
raise SystemExit("legacy unchecked cookie sizing remains")
|
||||
require_order(
|
||||
readdir,
|
||||
[
|
||||
"erofs_validate_dirblock",
|
||||
"erofs_previous_dirname",
|
||||
"erofs_dirname_order",
|
||||
"erofs_fill_dentries",
|
||||
],
|
||||
"linear readdir order validation",
|
||||
)
|
||||
if previous.count("erofs_read_data") != 1 or previous.count("erofs_brelse") != 1:
|
||||
raise SystemExit("readdir predecessor ownership is not one acquire/release")
|
||||
require_order(read_block, ["erofs_read_data", "erofs_validate_dirblock"], "lookup block read")
|
||||
for marker in ("block - 1", "block + 1", "erofs_dirblock_order"):
|
||||
if marker not in neighbors:
|
||||
raise SystemExit(f"bounded neighbor validation is missing: {marker}")
|
||||
if find_block.count("while (head <= back)") != 1 or re.search(r"\bfor\s*\(", find_block):
|
||||
raise SystemExit("lookup is no longer a single binary block search")
|
||||
require_order(
|
||||
find_block,
|
||||
["erofs_read_dirblock", "erofs_validate_dirblock_neighbors", "erofs_dirnamecmp"],
|
||||
"touched-block lookup validation",
|
||||
)
|
||||
require_order(namei, ["erofs_nid_is_valid", "*nid = found_nid", "*d_type"], "lookup NID publication")
|
||||
require_order(
|
||||
lookup,
|
||||
["erofs_namei", "(cnp->cn_flags & ISDOTDOT) == 0", "vn_vget_ino", "erofs_vget"],
|
||||
"FreeBSD self-NID lock boundary",
|
||||
)
|
||||
if lookup.count("vn_vget_ino") != 1 or lookup.count("erofs_vget") != 1:
|
||||
raise SystemExit("FreeBSD dotdot/ordinary vnode acquisition changed shape")
|
||||
for marker in ("vref(dvp)", "cache_enter(dvp, NULL, cnp)", "cache_enter(dvp, vp, cnp)"):
|
||||
if marker not in lookup:
|
||||
raise SystemExit(f"FreeBSD lookup behavior is missing: {marker}")
|
||||
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", dir_source + namei_source):
|
||||
raise SystemExit("negative Linux errno entered B10")
|
||||
|
||||
dir_order = [
|
||||
"erofs_validate_dirblock",
|
||||
"erofs_fill_dentries",
|
||||
"erofs_readdir_block",
|
||||
]
|
||||
if [function_position(dir_source, name) for name in dir_order] != sorted(
|
||||
function_position(dir_source, name) for name in dir_order
|
||||
):
|
||||
raise SystemExit("dir core order does not retain validator-before-Linux-core shape")
|
||||
namei_order = [
|
||||
"find_target_dirent",
|
||||
"erofs_find_target_block",
|
||||
"erofs_namei",
|
||||
"erofs_lookup",
|
||||
]
|
||||
if [function_position(namei_source, name) for name in namei_order] != sorted(
|
||||
function_position(namei_source, name) for name in namei_order
|
||||
):
|
||||
raise SystemExit("namei core order diverges from the Linux reference shape")
|
||||
|
||||
return {
|
||||
"changed_sources": sorted(changed_sources),
|
||||
"dir_core_order": dir_order,
|
||||
"namei_core_order": namei_order,
|
||||
"full_directory_state_added": False,
|
||||
"positive_errno": True,
|
||||
"vfs_cache_lookup_preserved": "vfs_cache_lookup" in current["erofs_vnops.c"],
|
||||
}
|
||||
|
||||
|
||||
def validate_block(names: list[str]) -> None:
|
||||
if not names or any(not name for name in names):
|
||||
raise ValueError(EINTEGRITY)
|
||||
encoded = [name.encode("ascii") for name in names]
|
||||
if any(left >= right for left, right in zip(encoded, encoded[1:])):
|
||||
raise ValueError(EINTEGRITY)
|
||||
|
||||
|
||||
def validate_boundary(left: list[str], right: list[str]) -> None:
|
||||
if left[-1].encode("ascii") >= right[0].encode("ascii"):
|
||||
raise ValueError(EINTEGRITY)
|
||||
|
||||
|
||||
def lookup_model(blocks: list[list[str]], target: str) -> tuple[str, int, list[int]]:
|
||||
head = 0
|
||||
back = len(blocks) - 1
|
||||
candidate = None
|
||||
reads = 0
|
||||
touched = []
|
||||
target_bytes = target.encode("ascii")
|
||||
try:
|
||||
while head <= back:
|
||||
mid = head + (back - head) // 2
|
||||
touched.append(mid)
|
||||
validate_block(blocks[mid])
|
||||
reads += 1
|
||||
if mid > 0:
|
||||
validate_block(blocks[mid - 1])
|
||||
validate_boundary(blocks[mid - 1], blocks[mid])
|
||||
reads += 1
|
||||
if mid + 1 < len(blocks):
|
||||
validate_block(blocks[mid + 1])
|
||||
validate_boundary(blocks[mid], blocks[mid + 1])
|
||||
reads += 1
|
||||
first = blocks[mid][0].encode("ascii")
|
||||
if target_bytes < first:
|
||||
back = mid - 1
|
||||
continue
|
||||
candidate = mid
|
||||
if target_bytes == first:
|
||||
return PASS, reads, touched
|
||||
head = mid + 1
|
||||
if candidate is None:
|
||||
return "ENOENT", reads, touched
|
||||
names = [name.encode("ascii") for name in blocks[candidate]]
|
||||
left = 1
|
||||
right = len(names) - 1
|
||||
while left <= right:
|
||||
mid = left + (right - left) // 2
|
||||
if names[mid] == target_bytes:
|
||||
return PASS, reads, touched
|
||||
if names[mid] < target_bytes:
|
||||
left = mid + 1
|
||||
else:
|
||||
right = mid - 1
|
||||
return "ENOENT", reads, touched
|
||||
except ValueError:
|
||||
return EINTEGRITY, reads, touched
|
||||
|
||||
|
||||
def readdir_model(blocks: list[list[str]]) -> tuple[str, list[str]]:
|
||||
output = []
|
||||
previous = None
|
||||
try:
|
||||
for block in blocks:
|
||||
validate_block(block)
|
||||
if previous is not None:
|
||||
validate_boundary(previous, block)
|
||||
output.extend(block)
|
||||
previous = block
|
||||
except ValueError:
|
||||
return EINTEGRITY, output
|
||||
return PASS, output
|
||||
|
||||
|
||||
def run_order_cases(spec: dict[str, object]) -> list[dict[str, object]]:
|
||||
results = []
|
||||
for case in spec["order_cases"]:
|
||||
lookup_status, reads, touched = lookup_model(case["blocks"], case["target"])
|
||||
readdir_status, raw = readdir_model(case["blocks"])
|
||||
if lookup_status != case["expected_lookup"]:
|
||||
raise SystemExit(f"lookup order oracle mismatch: {case['id']}")
|
||||
if readdir_status != case["expected_readdir"]:
|
||||
raise SystemExit(f"readdir order oracle mismatch: {case['id']}")
|
||||
if "expected_raw" in case and raw != case["expected_raw"]:
|
||||
raise SystemExit(f"raw directory order changed: {case['id']}")
|
||||
results.append(
|
||||
{
|
||||
"id": case["id"],
|
||||
"lookup": lookup_status,
|
||||
"readdir": readdir_status,
|
||||
"lookup_reads": reads,
|
||||
"lookup_touched": touched,
|
||||
"raw_entries": raw if readdir_status == PASS else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def nid_is_valid(config: dict[str, object], case: dict[str, object]) -> bool:
|
||||
uint64_max = (1 << 64) - 1
|
||||
metabox_bit = config["metabox_bit"]
|
||||
nid = case["nid"]
|
||||
low = nid & (metabox_bit - 1)
|
||||
if low > (uint64_max >> 5):
|
||||
return False
|
||||
offset = low << 5
|
||||
in_metabox = bool(nid & metabox_bit)
|
||||
if in_metabox:
|
||||
if not case["metabox_feature"] or case["metabox_size"] is None:
|
||||
return False
|
||||
return offset <= case["metabox_size"] and config["compact_inode_size"] <= case["metabox_size"] - offset
|
||||
primary = config["primary"]
|
||||
if primary["blocks"] > (uint64_max >> primary["blkszbits"]):
|
||||
return False
|
||||
metadata = primary["meta_blkaddr"] << primary["blkszbits"]
|
||||
if offset > uint64_max - metadata:
|
||||
return False
|
||||
offset += metadata
|
||||
image_size = primary["blocks"] << primary["blkszbits"]
|
||||
compact = config["compact_inode_size"]
|
||||
return (
|
||||
offset <= image_size
|
||||
and compact <= image_size - offset
|
||||
and offset <= primary["mediasize"]
|
||||
and compact <= primary["mediasize"] - offset
|
||||
)
|
||||
|
||||
|
||||
def run_nid_cases(config: dict[str, object]) -> list[dict[str, object]]:
|
||||
results = []
|
||||
for case in config["cases"]:
|
||||
valid = nid_is_valid(config, case)
|
||||
lookup = EINTEGRITY if not valid else PASS
|
||||
if valid and case["relation"] == "ordinary" and case["nid"] == case["current_nid"]:
|
||||
lookup = EINTEGRITY
|
||||
readdir = PASS if valid else EINTEGRITY
|
||||
if valid != case["expected_valid"] or lookup != case["expected_lookup"] or readdir != case["expected_readdir"]:
|
||||
raise SystemExit(f"NID oracle mismatch: {case['id']}")
|
||||
results.append({"id": case["id"], "valid": valid, "lookup": lookup, "readdir": readdir})
|
||||
return results
|
||||
|
||||
|
||||
def run_cookie_cases(config: dict[str, object]) -> list[dict[str, int]]:
|
||||
results = []
|
||||
allocation_limit = config["size_max"] // 8
|
||||
for case in config["cases"]:
|
||||
count = 0
|
||||
if case["resid"] > 0:
|
||||
count = case["resid"] // config["min_dirent_size"]
|
||||
count = min(count, config["int_max"], allocation_limit)
|
||||
if count != case["expected"]:
|
||||
raise SystemExit(f"cookie clamp oracle mismatch: {case['id']}")
|
||||
results.append({"id": case["id"], "resid": case["resid"], "cookies": count})
|
||||
return results
|
||||
|
||||
|
||||
def complexity_result(config: dict[str, int]) -> dict[str, object]:
|
||||
blocks = config["directory_bytes"] // config["block_size"]
|
||||
if blocks != config["expected_blocks"]:
|
||||
raise SystemExit("TC153 sparse directory block denominator changed")
|
||||
max_iterations = blocks.bit_length()
|
||||
if max_iterations != config["max_binary_iterations"]:
|
||||
raise SystemExit("TC153 binary iteration bound mismatch")
|
||||
scenarios = []
|
||||
for target in (0, blocks // 2, blocks - 1):
|
||||
head = 0
|
||||
back = blocks - 1
|
||||
iterations = 0
|
||||
reads = 0
|
||||
while head <= back:
|
||||
mid = head + (back - head) // 2
|
||||
iterations += 1
|
||||
reads += 1 + int(mid > 0) + int(mid + 1 < blocks)
|
||||
if mid == target:
|
||||
break
|
||||
if mid < target:
|
||||
head = mid + 1
|
||||
else:
|
||||
back = mid - 1
|
||||
if iterations > config["max_binary_iterations"] or reads > config["max_reads_with_neighbors"]:
|
||||
raise SystemExit("TC153 lookup exceeded the logarithmic neighbor-read bound")
|
||||
scenarios.append({"target_block": target, "iterations": iterations, "reads": reads})
|
||||
return {
|
||||
"directory_bytes": config["directory_bytes"],
|
||||
"blocks": blocks,
|
||||
"bound": "O(log n) selected blocks plus two neighbors per selection",
|
||||
"scenarios": scenarios,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B10":
|
||||
raise SystemExit("invalid B10 fixture identity")
|
||||
if spec.get("candidates") != ["P15-012", "P15-054", "P15-055", "P15-078", "P15-080"]:
|
||||
raise SystemExit("B10 candidate list changed")
|
||||
|
||||
report = {
|
||||
"schema": 1,
|
||||
"batch": "B10",
|
||||
"source": source_checks(args.root, args.dut, args.baseline),
|
||||
"order": run_order_cases(spec),
|
||||
"nid": run_nid_cases(spec["nid"]),
|
||||
"cookie": run_cookie_cases(spec["cookie"]),
|
||||
"complexity": complexity_result(spec["complexity"]),
|
||||
}
|
||||
args.report.write_text(
|
||||
json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
print(f"TC163-directory-order PASS {len(report['order'])} cases")
|
||||
print(f"TC173-readdir-boundaries PASS {len(report['nid']) + len(report['cookie'])} cases")
|
||||
print("TC153-large-directory-block-index PASS O(log n)+bounded-neighbors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"batch": "B10",
|
||||
"candidates": [
|
||||
"P15-012",
|
||||
"P15-054",
|
||||
"P15-055",
|
||||
"P15-078",
|
||||
"P15-080"
|
||||
],
|
||||
"complexity": {
|
||||
"block_size": 4096,
|
||||
"directory_bytes": 8796093022208,
|
||||
"expected_blocks": 2147483648,
|
||||
"max_binary_iterations": 32,
|
||||
"max_reads_with_neighbors": 96
|
||||
},
|
||||
"cookie": {
|
||||
"int_max": 2147483647,
|
||||
"min_dirent_size": 24,
|
||||
"size_max": 18446744073709551615,
|
||||
"cases": [
|
||||
{"id": "negative-resid", "resid": -1, "expected": 0},
|
||||
{"id": "zero-resid", "resid": 0, "expected": 0},
|
||||
{"id": "one-byte", "resid": 1, "expected": 0},
|
||||
{"id": "below-minimum", "resid": 23, "expected": 0},
|
||||
{"id": "one-dirent", "resid": 24, "expected": 1},
|
||||
{"id": "below-two-dirents", "resid": 47, "expected": 1},
|
||||
{"id": "two-dirents", "resid": 48, "expected": 2},
|
||||
{"id": "normal-buffer", "resid": 4096, "expected": 170},
|
||||
{"id": "int-clamp", "resid": 51539607552, "expected": 2147483647}
|
||||
]
|
||||
},
|
||||
"nid": {
|
||||
"compact_inode_size": 32,
|
||||
"metabox_bit": 9223372036854775808,
|
||||
"primary": {
|
||||
"blkszbits": 12,
|
||||
"blocks": 8,
|
||||
"mediasize": 32768,
|
||||
"meta_blkaddr": 1
|
||||
},
|
||||
"cases": [
|
||||
{"id": "primary-first", "nid": 0, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"},
|
||||
{"id": "primary-last", "nid": 895, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"},
|
||||
{"id": "primary-past-end", "nid": 896, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"},
|
||||
{"id": "primary-shift-overflow", "nid": 9223372036854775807, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"},
|
||||
{"id": "metabox-valid", "nid": 9223372036854775811, "current_nid": 1, "relation": "ordinary", "metabox_feature": true, "metabox_size": 128, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"},
|
||||
{"id": "metabox-no-feature", "nid": 9223372036854775808, "current_nid": 1, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"},
|
||||
{"id": "metabox-past-end", "nid": 9223372036854775812, "current_nid": 1, "relation": "ordinary", "metabox_feature": true, "metabox_size": 128, "expected_valid": false, "expected_lookup": "EINTEGRITY", "expected_readdir": "EINTEGRITY"},
|
||||
{"id": "ordinary-self", "nid": 42, "current_nid": 42, "relation": "ordinary", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "EINTEGRITY", "expected_readdir": "PASS"},
|
||||
{"id": "dot-self", "nid": 42, "current_nid": 42, "relation": "dot", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"},
|
||||
{"id": "dotdot-self", "nid": 42, "current_nid": 42, "relation": "dotdot", "metabox_feature": false, "metabox_size": null, "expected_valid": true, "expected_lookup": "PASS", "expected_readdir": "PASS"}
|
||||
]
|
||||
},
|
||||
"order_cases": [
|
||||
{
|
||||
"id": "valid-raw-order",
|
||||
"blocks": [[".", "..", "alpha", "alphabet"], ["beta", "delta"], ["omega"]],
|
||||
"target": "delta",
|
||||
"expected_lookup": "PASS",
|
||||
"expected_readdir": "PASS",
|
||||
"expected_raw": [".", "..", "alpha", "alphabet", "beta", "delta", "omega"]
|
||||
},
|
||||
{
|
||||
"id": "long-common-prefix",
|
||||
"blocks": [["prefix-0000000000000000", "prefix-0000000000000001"], ["prefix-0000000000000010", "prefix-0000000000000011"]],
|
||||
"target": "prefix-0000000000000011",
|
||||
"expected_lookup": "PASS",
|
||||
"expected_readdir": "PASS",
|
||||
"expected_raw": ["prefix-0000000000000000", "prefix-0000000000000001", "prefix-0000000000000010", "prefix-0000000000000011"]
|
||||
},
|
||||
{
|
||||
"id": "within-block-reverse",
|
||||
"blocks": [["alpha", "charlie", "bravo"]],
|
||||
"target": "bravo",
|
||||
"expected_lookup": "EINTEGRITY",
|
||||
"expected_readdir": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "within-block-duplicate",
|
||||
"blocks": [["alpha", "beta", "beta"]],
|
||||
"target": "beta",
|
||||
"expected_lookup": "EINTEGRITY",
|
||||
"expected_readdir": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "cross-block-reverse",
|
||||
"blocks": [["alpha", "delta"], ["charlie", "omega"]],
|
||||
"target": "delta",
|
||||
"expected_lookup": "EINTEGRITY",
|
||||
"expected_readdir": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "cross-block-duplicate",
|
||||
"blocks": [["alpha", "delta"], ["delta", "omega"]],
|
||||
"target": "delta",
|
||||
"expected_lookup": "EINTEGRITY",
|
||||
"expected_readdir": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "remote-corruption-not-prescanned",
|
||||
"blocks": [["a", "b"], ["c", "d"], ["e", "f"], ["g", "h"], ["i", "j"], ["k", "l"], ["z", "zz"], ["m", "n"]],
|
||||
"target": "a",
|
||||
"expected_lookup": "PASS",
|
||||
"expected_readdir": "EINTEGRITY",
|
||||
"remote_corruption_not_claimed": true
|
||||
}
|
||||
],
|
||||
"schema": 1
|
||||
}
|
||||
Executable
+396
@@ -0,0 +1,396 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
VTYPE_NUMBER = {
|
||||
"VNON": 0,
|
||||
"VREG": 1,
|
||||
"VDIR": 2,
|
||||
"VBLK": 3,
|
||||
"VCHR": 4,
|
||||
"VLNK": 5,
|
||||
"VSOCK": 6,
|
||||
"VFIFO": 7,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
parser.add_argument("--dut", type=Path, required=True)
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--gate-output", type=Path, required=True)
|
||||
parser.add_argument("--artifacts", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def committed(root: Path, commit: str, path: str) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(root), "show", f"{commit}:{path}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def replace_once(source: str, old: str, new: str, label: str) -> str:
|
||||
count = source.count(old)
|
||||
if count != 1:
|
||||
raise SystemExit(f"{label}: expected one transform site, found {count}")
|
||||
return source.replace(old, new, 1)
|
||||
|
||||
|
||||
def extract_function(source: str, name: str) -> str:
|
||||
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
|
||||
if match is None:
|
||||
raise SystemExit(f"function not found: {name}")
|
||||
start = source.rfind("\n\n", 0, match.start()) + 2
|
||||
brace = source.find("{", match.end())
|
||||
if brace < 0:
|
||||
raise SystemExit(f"function body not found: {name}")
|
||||
depth = 0
|
||||
state = "code"
|
||||
index = brace
|
||||
while index < len(source):
|
||||
char = source[index]
|
||||
following = source[index + 1] if index + 1 < len(source) else ""
|
||||
if state == "code":
|
||||
if char == "/" and following == "*":
|
||||
state = "block"
|
||||
index += 2
|
||||
continue
|
||||
if char == "/" and following == "/":
|
||||
state = "line"
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
state = "string"
|
||||
elif char == "'":
|
||||
state = "character"
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start : index + 1] + "\n"
|
||||
elif state == "block" and char == "*" and following == "/":
|
||||
state = "code"
|
||||
index += 2
|
||||
continue
|
||||
elif state == "line" and char == "\n":
|
||||
state = "code"
|
||||
elif state in {"string", "character"}:
|
||||
if char == "\\":
|
||||
index += 2
|
||||
continue
|
||||
if (state == "string" and char == '"') or (
|
||||
state == "character" and char == "'"
|
||||
):
|
||||
state = "code"
|
||||
index += 1
|
||||
raise SystemExit(f"unterminated function: {name}")
|
||||
|
||||
|
||||
def run_helper(helper: str, records: list[dict], artifacts: Path) -> list[dict]:
|
||||
harness = f'''#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define EROFS_FT_UNKNOWN 0
|
||||
#define EROFS_FT_REG_FILE 1
|
||||
#define EROFS_FT_DIR 2
|
||||
#define EROFS_FT_CHRDEV 3
|
||||
#define EROFS_FT_BLKDEV 4
|
||||
#define EROFS_FT_FIFO 5
|
||||
#define EROFS_FT_SOCK 6
|
||||
#define EROFS_FT_SYMLINK 7
|
||||
#define __enum_uint8(name) enum name
|
||||
|
||||
enum vtype {{
|
||||
VNON,
|
||||
VREG,
|
||||
VDIR,
|
||||
VBLK,
|
||||
VCHR,
|
||||
VLNK,
|
||||
VSOCK,
|
||||
VFIFO,
|
||||
}};
|
||||
|
||||
{helper}
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{{
|
||||
unsigned long file_type, vtype;
|
||||
|
||||
if (argc != 3)
|
||||
return (2);
|
||||
file_type = strtoul(argv[1], NULL, 0);
|
||||
vtype = strtoul(argv[2], NULL, 0);
|
||||
if (file_type > UINT8_MAX || vtype > VFIFO)
|
||||
return (2);
|
||||
printf("%d\\n", erofs_dirent_type_matches((uint8_t)file_type,
|
||||
(enum vtype)vtype) ? 0 : 97);
|
||||
return (0);
|
||||
}}
|
||||
'''
|
||||
source = artifacts / "B11-helper-harness.c"
|
||||
source.write_text(harness, encoding="ascii")
|
||||
with tempfile.TemporaryDirectory(prefix="b11-helper-") as temporary:
|
||||
binary = Path(temporary) / "helper"
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
"-std=c11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
str(source),
|
||||
"-o",
|
||||
str(binary),
|
||||
],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"B11 helper harness did not compile:\n{completed.stdout}")
|
||||
results = []
|
||||
for record in records:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
str(binary),
|
||||
str(record["file_type"]),
|
||||
str(VTYPE_NUMBER[record["vtype"]]),
|
||||
],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"B11 helper failed for {record['id']}")
|
||||
actual = int(completed.stdout.strip())
|
||||
expected = record["expected_errno"]
|
||||
if actual != expected:
|
||||
raise SystemExit(
|
||||
f"B11 helper result differs for {record['id']}: {actual} != {expected}"
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"actual_errno": actual,
|
||||
"expected_errno": expected,
|
||||
"id": record["id"],
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B11":
|
||||
raise SystemExit("invalid B11 dtype spec")
|
||||
args.artifacts.mkdir(parents=True, exist_ok=True)
|
||||
gate_decision = subprocess.check_output(
|
||||
["git", "-C", str(args.root), "rev-parse", spec["gate_decision"]],
|
||||
text=True,
|
||||
).strip()
|
||||
if gate_decision != spec["gate_decision"]:
|
||||
raise SystemExit("B11 gate decision identity changed")
|
||||
|
||||
gate_script = args.dut / "tests/pre15/gates/P15-081.sh"
|
||||
gate_input = args.dut / "tests/pre15/gates/P15-081-input.json"
|
||||
if sha256(gate_script) != spec["gate_script_sha256"]:
|
||||
raise SystemExit("B11 gate script identity changed")
|
||||
if sha256(gate_input) != spec["gate_input_sha256"]:
|
||||
raise SystemExit("B11 gate input identity changed")
|
||||
for name, expected in spec["gate_evidence_sha256"].items():
|
||||
path = args.gate_output / name
|
||||
if sha256(path) != expected:
|
||||
raise SystemExit(f"B11 gate replay identity changed: {name}")
|
||||
|
||||
gate_result = json.loads(
|
||||
(args.gate_output / "result.json").read_text(encoding="ascii")
|
||||
)
|
||||
gate_oracle = json.loads(
|
||||
(args.gate_output / "oracle.json").read_text(encoding="ascii")
|
||||
)
|
||||
if gate_result["status"] != "GO" or gate_result["b11"] != "AUTHORIZED":
|
||||
raise SystemExit("P15-081 gate no longer authorizes B11")
|
||||
for key, expected in spec["expected"].items():
|
||||
actual = (
|
||||
gate_result["fixture_set_sha256"]
|
||||
if key == "fixture_set_sha256"
|
||||
else gate_result[key]
|
||||
)
|
||||
if actual != expected:
|
||||
raise SystemExit(f"B11 gate count changed for {key}: {actual} != {expected}")
|
||||
|
||||
baseline = spec["baseline"]
|
||||
paths = ("inode.c", "internal.h", "namei.c", "dir.c", "erofs_vnops.c")
|
||||
before = {
|
||||
name: committed(args.root, baseline, f"repo-pre-15/src/{name}")
|
||||
for name in paths
|
||||
}
|
||||
current = {
|
||||
name: (args.dut / "src" / name).read_text(encoding="utf-8")
|
||||
for name in paths
|
||||
}
|
||||
|
||||
helper = '''bool
|
||||
erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype)
|
||||
{
|
||||
\tswitch (file_type) {
|
||||
\tcase EROFS_FT_REG_FILE:
|
||||
\t\treturn (vtype == VREG);
|
||||
\tcase EROFS_FT_DIR:
|
||||
\t\treturn (vtype == VDIR);
|
||||
\tcase EROFS_FT_CHRDEV:
|
||||
\t\treturn (vtype == VCHR);
|
||||
\tcase EROFS_FT_BLKDEV:
|
||||
\t\treturn (vtype == VBLK);
|
||||
\tcase EROFS_FT_FIFO:
|
||||
\t\treturn (vtype == VFIFO);
|
||||
\tcase EROFS_FT_SOCK:
|
||||
\t\treturn (vtype == VSOCK);
|
||||
\tcase EROFS_FT_SYMLINK:
|
||||
\t\treturn (vtype == VLNK);
|
||||
\tdefault:
|
||||
\t\treturn (true);
|
||||
\t}
|
||||
}
|
||||
|
||||
'''
|
||||
inode_marker = "/*\n * Read and decode a disk inode."
|
||||
expected_inode = replace_once(
|
||||
before["inode.c"], inode_marker, helper + inode_marker, "inode helper"
|
||||
)
|
||||
expected_internal = replace_once(
|
||||
before["internal.h"],
|
||||
"int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n",
|
||||
"bool erofs_dirent_type_matches(uint8_t file_type,\n"
|
||||
" __enum_uint8(vtype) vtype);\n"
|
||||
"int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n",
|
||||
"internal prototype",
|
||||
)
|
||||
lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n"
|
||||
lookup_replacement = (
|
||||
"\tif (error != 0)\n"
|
||||
"\t\treturn (error);\n"
|
||||
"\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n"
|
||||
"\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n"
|
||||
"\t\tvput(vp);\n"
|
||||
"\t\treturn (EINTEGRITY);\n"
|
||||
"\t}\n"
|
||||
"\t*ap->a_vpp = vp;\n"
|
||||
)
|
||||
expected_namei = replace_once(
|
||||
before["namei.c"], lookup_marker, lookup_replacement, "lookup validator"
|
||||
)
|
||||
expected = {
|
||||
"inode.c": expected_inode,
|
||||
"internal.h": expected_internal,
|
||||
"namei.c": expected_namei,
|
||||
"dir.c": before["dir.c"],
|
||||
"erofs_vnops.c": before["erofs_vnops.c"],
|
||||
}
|
||||
for name in paths:
|
||||
if current[name] != expected[name]:
|
||||
raise SystemExit(f"B11 source differs from the exact transform: {name}")
|
||||
|
||||
lookup = extract_function(current["namei.c"], "erofs_lookup")
|
||||
validator = lookup.index("erofs_dirent_type_matches")
|
||||
if lookup.index("error = erofs_vget(") > validator:
|
||||
raise SystemExit("B11 validator moved before the ordinary vnode lookup")
|
||||
if lookup.index("error = vn_vget_ino(") > validator:
|
||||
raise SystemExit("B11 validator moved before the dotdot vnode lookup")
|
||||
if validator > lookup.index("*ap->a_vpp = vp"):
|
||||
raise SystemExit("B11 validator moved after vnode publication")
|
||||
if validator > lookup.index("cache_enter(dvp, vp, cnp)"):
|
||||
raise SystemExit("B11 validator moved after namecache publication")
|
||||
validator_start = lookup.rfind(
|
||||
"if ((cnp->cn_flags & ISDOTDOT) == 0", 0, validator
|
||||
)
|
||||
if validator_start < 0:
|
||||
raise SystemExit("B11 validator condition is absent")
|
||||
validator_block = lookup[validator_start : lookup.index("*ap->a_vpp = vp")]
|
||||
if "(cnp->cn_flags & ISDOTDOT) == 0" not in validator_block:
|
||||
raise SystemExit("B11 dotdot bypass is absent")
|
||||
if "vput(vp);\n\t\treturn (EINTEGRITY);" not in validator_block:
|
||||
raise SystemExit("B11 mismatch cleanup or positive errno changed")
|
||||
for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(", "VOP_LOCK(", "vn_lock("):
|
||||
if token in validator_block:
|
||||
raise SystemExit(f"B11 validator introduced a lock/vget operation: {token}")
|
||||
for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get("):
|
||||
if token in current["dir.c"]:
|
||||
raise SystemExit(f"B11 readdir path introduced vnode loading: {token}")
|
||||
|
||||
actual_helper = extract_function(current["inode.c"], "erofs_dirent_type_matches")
|
||||
records = []
|
||||
for record in gate_oracle["known_match_records"]:
|
||||
records.append(
|
||||
{
|
||||
"expected_errno": record["expected_errno"],
|
||||
"file_type": record["file_type"],
|
||||
"id": record["case"],
|
||||
"vtype": record["vtype"],
|
||||
}
|
||||
)
|
||||
for record in gate_oracle["case_records"]:
|
||||
records.append(
|
||||
{
|
||||
"expected_errno": record["expected_errno"],
|
||||
"file_type": record["file_type"],
|
||||
"id": record["id"],
|
||||
"vtype": record["vtype"],
|
||||
}
|
||||
)
|
||||
helper_results = run_helper(actual_helper, records, args.artifacts)
|
||||
|
||||
report = {
|
||||
"batch": "B11",
|
||||
"candidate": "P15-081",
|
||||
"dir_source_unchanged": True,
|
||||
"dotdot_bypasses_validator": True,
|
||||
"gate_status": gate_result["status"],
|
||||
"helper_case_count": len(helper_results),
|
||||
"helper_results": helper_results,
|
||||
"known_mismatch_errno": "+EINTEGRITY",
|
||||
"namecache_publication_after_validator": True,
|
||||
"readdir_vget_count": 0,
|
||||
"source_transform_exact": True,
|
||||
"status": "PASS",
|
||||
"unknown_and_reserved_tolerated": True,
|
||||
"vnode_source_unchanged": True,
|
||||
}
|
||||
(args.artifacts / "B11-dtype-report.json").write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(
|
||||
"B11 dtype: PASS "
|
||||
f"helper_cases={len(helper_results)} known_mismatch=+EINTEGRITY "
|
||||
"unknown=accepted readdir_vgets=0 dotdot=bypass"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"baseline": "e2e3fb86b6fffcb01d6fd29c17dd95628ad070de",
|
||||
"batch": "B11",
|
||||
"candidate": "P15-081",
|
||||
"expected": {
|
||||
"fixture_count": 14,
|
||||
"fixture_set_sha256": "b562a7e42e139b16f3ce399d585aa7c373a66aa478a084bfed67b2b2aa9f2bd3",
|
||||
"forward_compatible_case_count": 5,
|
||||
"known_match_count": 8,
|
||||
"known_mismatch_count": 8,
|
||||
"normal_entry_count": 13,
|
||||
"prototype_case_count": 21
|
||||
},
|
||||
"gate_decision": "e2cb014fede2a8eb35387ff84f613f59a555defc",
|
||||
"gate_evidence_sha256": {
|
||||
"SHA256SUMS": "36b486617920b04ca87e88016e2fc0f519cda3a5f99dc675c9a89549615a02e3",
|
||||
"candidate.patch": "427097da9304f5bedb770be8cab5f589706d316fb18a152ca16cb9d0663713fd",
|
||||
"fixture-index.json": "1a46b8ebfec16109c5d193001dc650a1b56000045e5d9e1e9b145fe7c5df9a16",
|
||||
"lock-ledger.json": "e8c2e6ba797fbcc83d832af6583d3f74f344750f6897111e7fe501130e577c13",
|
||||
"normal-entry.tsv": "b2841448068f70633ed0c02d901177a639cfed150f0e1c58977cc6c7b1e7313e",
|
||||
"oracle.json": "932e558d3ead05572f38c89635bb10cae45393e42028031f81dfc7a56f2093f6",
|
||||
"prototype.c": "1a295d18830c61a9708d465c3b1e415efb3ff8276100e4e32139bfa715195d8b",
|
||||
"result.json": "4ec8e053c60ae75ae2fc4580a3d820a0452cf519f6b85376953135e18a7ec0ec"
|
||||
},
|
||||
"gate_input_sha256": "a887fad56927545adb48462b770100d1e64b22b32554ed8ffad14c72b8febfc3",
|
||||
"gate_script_sha256": "6215a3005214ba8d25dacd63e039320a0c63308d2334b5faff99a3927cad7d2e",
|
||||
"schema": 1
|
||||
}
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || {
|
||||
printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2
|
||||
exit 21
|
||||
}
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B12 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || {
|
||||
printf '%s\n' "B12 KLD work path exists: $work" >&2
|
||||
exit 20
|
||||
}
|
||||
mkdir -p "$work" "$(dirname -- "$output")"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
printf 'CC %s\n' "$file"
|
||||
clang $cflags -I"$sys/contrib/zstd/lib/freebsd" \
|
||||
-c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
nm -u "$output" | LC_ALL=C sort > B12-nm-u.txt
|
||||
nm -g "$output" | LC_ALL=C sort > B12-nm-global.txt
|
||||
if ! awk '$NF == "bcmp" { found = 1 } END { exit found }' B12-nm-u.txt; then
|
||||
printf '%s\n' 'B12 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q ' ZSTD_' B12-nm-u.txt; then
|
||||
printf '%s\n' 'B12 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
Executable
+228
@@ -0,0 +1,228 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
base=${3:?base image is required}
|
||||
askpass=${4:?askpass path is required}
|
||||
output=${5:?output path is required}
|
||||
protected_pid=26318
|
||||
protected_port=9222
|
||||
builder=$dut/tests/pre15/fixtures/B12-build-kld.sh
|
||||
|
||||
test ! -e "$output" || {
|
||||
printf '%s\n' "B12 QEMU output exists: $output" >&2
|
||||
exit 20
|
||||
}
|
||||
mkdir -p "$output"
|
||||
work=$(mktemp -d /tmp/pre15-b12-qemu.XXXXXX)
|
||||
overlay=$work/guest-overlay.qcow2
|
||||
module=$work/B12-erofs-zstdio0.ko
|
||||
qemu_pid=
|
||||
port=
|
||||
guest_ready=0
|
||||
module_loaded=0
|
||||
|
||||
protected_start=$(awk '{print $22}' /proc/$protected_pid/stat)
|
||||
protected_cmd=$(sha256sum /proc/$protected_pid/cmdline | awk '{print $1}')
|
||||
stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \
|
||||
>"$output/base-metadata-before.txt"
|
||||
|
||||
ssh_options()
|
||||
{
|
||||
printf '%s\n' \
|
||||
-q \
|
||||
-o BatchMode=no \
|
||||
-o PubkeyAuthentication=no \
|
||||
-o PreferredAuthentications=keyboard-interactive,password \
|
||||
-o NumberOfPasswordPrompts=1 \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 \
|
||||
-p "$port"
|
||||
}
|
||||
|
||||
guest()
|
||||
{
|
||||
timeout -k 5 "${B12_GUEST_TIMEOUT:-60}" env DISPLAY=:0 \
|
||||
SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force \
|
||||
ssh $(ssh_options) [email protected] "$@"
|
||||
}
|
||||
|
||||
copy_to_guest()
|
||||
{
|
||||
timeout -k 5 "${B12_COPY_TIMEOUT:-120}" env DISPLAY=:0 \
|
||||
SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force \
|
||||
scp -O -q \
|
||||
-o BatchMode=no \
|
||||
-o PubkeyAuthentication=no \
|
||||
-o PreferredAuthentications=keyboard-interactive,password \
|
||||
-o NumberOfPasswordPrompts=1 \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 \
|
||||
-P "$port" "$1" [email protected]:"$2"
|
||||
}
|
||||
|
||||
port_free()
|
||||
{
|
||||
python3 - "$1" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(0.2)
|
||||
raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) != 0 else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup()
|
||||
{
|
||||
set +e
|
||||
if test "$guest_ready" = 1 && test "$module_loaded" = 1; then
|
||||
guest kldunload /root/B12-erofs-zstdio0.ko \
|
||||
>>"$output/cleanup.log" 2>&1
|
||||
module_loaded=0
|
||||
fi
|
||||
if test -n "$qemu_pid" && kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
kill -TERM "$qemu_pid" 2>/dev/null
|
||||
wait_count=0
|
||||
while kill -0 "$qemu_pid" 2>/dev/null && test "$wait_count" -lt 20; do
|
||||
sleep 1
|
||||
wait_count=$((wait_count + 1))
|
||||
done
|
||||
if kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
kill -KILL "$qemu_pid" 2>/dev/null
|
||||
fi
|
||||
wait "$qemu_pid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
for tool in awk clang file nm python3 qemu-img qemu-system-x86_64 scp \
|
||||
sha256sum ssh stat timeout; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B12 QEMU tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test -d "$freebsd_src/sys"
|
||||
test -f "$base"
|
||||
test -x "$askpass"
|
||||
test -r /proc/$protected_pid/stat
|
||||
|
||||
timeout -k 10 600 /bin/sh "$builder" "$dut" "$freebsd_src" "$module" \
|
||||
"$work/kld-work" >"$output/build.stdout" 2>"$output/build.stderr"
|
||||
file "$module" >"$output/module.file"
|
||||
sha256sum "$module" >"$output/module.sha256"
|
||||
nm -g "$module" | LC_ALL=C sort >"$output/module.nm-global"
|
||||
nm -u "$module" | LC_ALL=C sort >"$output/module.nm-u"
|
||||
|
||||
qemu-img create -q -f qcow2 -F qcow2 -b "$base" "$overlay"
|
||||
port=$(python3 - "$protected_port" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
protected = int(sys.argv[1])
|
||||
while True:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
if port != protected:
|
||||
print(port)
|
||||
break
|
||||
PY
|
||||
)
|
||||
printf '%s\n' "$port" >"$output/owned-port.txt"
|
||||
qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 \
|
||||
-m 3072 -smp 2 \
|
||||
-drive "file=$overlay,if=virtio,format=qcow2,cache=none" \
|
||||
-netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \
|
||||
-device virtio-net-pci,netdev=net0 -display none \
|
||||
-serial "file:$output/serial.log" -monitor none \
|
||||
-D "$output/qemu.log" >"$output/qemu-process.log" 2>&1 &
|
||||
qemu_pid=$!
|
||||
printf '%s\n' "$qemu_pid" >"$output/owned-pid.txt"
|
||||
printf '%s\n' \
|
||||
"qemu-system-x86_64 -m 3072 -smp 2 -drive owned-overlay -port $port" \
|
||||
>"$output/qemu-argv-summary.txt"
|
||||
|
||||
boot_wait=0
|
||||
while test "$boot_wait" -lt "${B12_BOOT_TIMEOUT:-300}"; do
|
||||
if ! kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
printf '%s\n' 'owned QEMU exited before SSH readiness' >&2
|
||||
exit 21
|
||||
fi
|
||||
if B12_GUEST_TIMEOUT=10 guest true >/dev/null 2>&1; then
|
||||
guest_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
boot_wait=$((boot_wait + 2))
|
||||
done
|
||||
test "$guest_ready" = 1 || {
|
||||
printf '%s\n' 'owned QEMU SSH boot deadline expired' >&2
|
||||
exit 21
|
||||
}
|
||||
|
||||
guest uname -a >"$output/guest-uname.txt"
|
||||
guest sysctl -n kern.osreldate >"$output/guest-osreldate.txt"
|
||||
test "$(tr -d '\r\n' < "$output/guest-osreldate.txt")" = 1500068 || {
|
||||
printf '%s\n' 'guest OSREL is not exact FreeBSD 15 ABI 1500068' >&2
|
||||
exit 10
|
||||
}
|
||||
if guest kldstat -q -m erofs >/dev/null 2>&1; then
|
||||
printf '%s\n' 'guest already has an EROFS module loaded' >&2
|
||||
exit 21
|
||||
fi
|
||||
copy_to_guest "$module" /root/B12-erofs-zstdio0.ko
|
||||
guest sha256 -q /root/B12-erofs-zstdio0.ko >"$output/guest-module.sha256"
|
||||
test "$(awk '{print $1}' "$output/module.sha256")" = \
|
||||
"$(tr -d '\r\n' < "$output/guest-module.sha256")"
|
||||
guest dmesg >"$output/dmesg-before.txt"
|
||||
guest kldload /root/B12-erofs-zstdio0.ko >"$output/kldload.stdout" \
|
||||
2>"$output/kldload.stderr"
|
||||
module_loaded=1
|
||||
guest kldstat -n B12-erofs-zstdio0.ko >"$output/kldstat-loaded.txt"
|
||||
guest kldunload /root/B12-erofs-zstdio0.ko >"$output/kldunload.stdout" \
|
||||
2>"$output/kldunload.stderr"
|
||||
module_loaded=0
|
||||
guest dmesg >"$output/dmesg-after.txt"
|
||||
diff -u "$output/dmesg-before.txt" "$output/dmesg-after.txt" \
|
||||
>"$output/dmesg.diff" || true
|
||||
sed -n '/^+++ /d; /^+/s/^+//p' "$output/dmesg.diff" \
|
||||
>"$output/dmesg-added.txt"
|
||||
if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free|link_elf_obj: symbol .* undefined' \
|
||||
"$output/dmesg-added.txt"; then
|
||||
printf '%s\n' 'B12 exact-ABI load produced a kernel diagnostic' >&2
|
||||
exit 10
|
||||
fi
|
||||
|
||||
cleanup
|
||||
trap - EXIT HUP INT TERM
|
||||
stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \
|
||||
>"$output/base-metadata-after.txt"
|
||||
cmp "$output/base-metadata-before.txt" "$output/base-metadata-after.txt"
|
||||
test -r /proc/$protected_pid/stat
|
||||
test "$(awk '{print $22}' /proc/$protected_pid/stat)" = "$protected_start"
|
||||
test "$(sha256sum /proc/$protected_pid/cmdline | awk '{print $1}')" = "$protected_cmd"
|
||||
port_free "$port"
|
||||
cat >"$output/result.json" <<EOF
|
||||
{
|
||||
"candidate": "P15-087",
|
||||
"exact_abi_osreldate": 1500068,
|
||||
"kld_load": "PASS",
|
||||
"kld_unload": "PASS",
|
||||
"kernel_log": "PASS",
|
||||
"owned_overlay_removed": true,
|
||||
"owned_pid_stopped": true,
|
||||
"owned_port": $port,
|
||||
"owned_port_free_after": true,
|
||||
"protected_pid": $protected_pid,
|
||||
"protected_port": $protected_port,
|
||||
"schema": 1,
|
||||
"status": "PASS"
|
||||
}
|
||||
EOF
|
||||
printf '%s\n' 'B12 exact-ABI QEMU load/unload PASS'
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"baseline": "02cc3f4053f7f93560e2f7d9782801a7bf6548c4",
|
||||
"candidate": "P15-087",
|
||||
"gate_evidence": "planning/pre15/evidence/20260817T140853Z-G11-P15-087",
|
||||
"gate_result_sha256": "7152abbc21a377bf6e03e503e678472b3fca8212da58c77209a017fd5762b165",
|
||||
"gate_rows_sha256": "adc76429ddfb5b9aed5dee80e9ee484e3a7caa3b18025e9533fee9d617c452b0",
|
||||
"maximum_extra_provider_reads_percent": 25.0,
|
||||
"maximum_readahead_bytes": 1048576,
|
||||
"minimum_cold_median_improvement_percent": 10.0,
|
||||
"schema": 1,
|
||||
"source_sha256": {
|
||||
"src/data.c": "99aa0d083ff30e5857a2a7c4a1cd287235b9ec3873a6e1769d6bd14c53e82dfa",
|
||||
"src/dir.c": "762a91de2349b3dbb3e4d6214367fa3487899786a5eefe97d2bcb1d36debdd99",
|
||||
"src/internal.h": "0bd39e4e00efac60c598636349e4cae32137abc3811ba79af69864707e9a5946"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
{
|
||||
"candidate": "P15-018",
|
||||
"cases": [
|
||||
{
|
||||
"epoch_le": "ffffffffffffffff",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -1,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-pre1970-minus-one",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffffffffffffff",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 0,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-cross-zero",
|
||||
"mtime_delta_le": "01000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "fbffffffffffffff",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -2,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "ffc99a3b",
|
||||
"id": "compact-negative-with-max-nsec",
|
||||
"mtime_delta_le": "03000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "0000000000000080",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -9223372036854775808,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-int64-min",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffffffffffff7f",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 9223372036854775807,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-int64-max",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffffffffffff7f",
|
||||
"expected_add": "EINTEGRITY",
|
||||
"expected_nsec": "NOT_REACHED",
|
||||
"expected_seconds": null,
|
||||
"expected_time32": "NOT_REACHED",
|
||||
"expected_time64": "NOT_REACHED",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-signed-upper-overflow",
|
||||
"mtime_delta_le": "01000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffff7f00000000",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 2147483647,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-time32-max",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffff7f00000000",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 2147483648,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-time32-upper-overflow",
|
||||
"mtime_delta_le": "01000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "00000080ffffffff",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -2147483648,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-time32-min",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "ffffff7fffffffff",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -2147483649,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-time32-lower-overflow",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "00000000ffffff7f",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 9223372036854775807,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-uint32-delta-at-int64-max",
|
||||
"mtime_delta_le": "ffffffff"
|
||||
},
|
||||
{
|
||||
"epoch_le": "01000000ffffff7f",
|
||||
"expected_add": "EINTEGRITY",
|
||||
"expected_nsec": "NOT_REACHED",
|
||||
"expected_seconds": null,
|
||||
"expected_time32": "NOT_REACHED",
|
||||
"expected_time64": "NOT_REACHED",
|
||||
"fixed_nsec_le": "00000000",
|
||||
"id": "compact-uint32-delta-over-int64-max",
|
||||
"mtime_delta_le": "ffffffff"
|
||||
},
|
||||
{
|
||||
"epoch_le": "0000000000000000",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 0,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "ffc99a3b",
|
||||
"id": "compact-nsec-max",
|
||||
"mtime_delta_le": "00000000"
|
||||
},
|
||||
{
|
||||
"epoch_le": "0000000000000000",
|
||||
"expected_add": "PASS",
|
||||
"expected_nsec": "EINTEGRITY",
|
||||
"expected_seconds": 0,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"fixed_nsec_le": "00ca9a3b",
|
||||
"id": "compact-nsec-overflow",
|
||||
"mtime_delta_le": "00000000"
|
||||
}
|
||||
],
|
||||
"encoding": {
|
||||
"epoch": "signed 64-bit little-endian two's-complement seconds",
|
||||
"fixed_nsec": "unsigned 32-bit little-endian nanoseconds",
|
||||
"mtime_delta": "unsigned 32-bit little-endian seconds"
|
||||
},
|
||||
"schema": 1
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"candidate": "P15-018",
|
||||
"cases": [
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -1,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-pre1970-minus-one",
|
||||
"mtime_le": "ffffffffffffffff",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -9223372036854775808,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-int64-min",
|
||||
"mtime_le": "0000000000000080",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 9223372036854775807,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-int64-max",
|
||||
"mtime_le": "ffffffffffffff7f",
|
||||
"mtime_nsec_le": "ffc99a3b"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -2147483648,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-time32-min",
|
||||
"mtime_le": "00000080ffffffff",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": -2147483649,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-time32-lower-overflow",
|
||||
"mtime_le": "ffffff7fffffffff",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 2147483647,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-time32-max",
|
||||
"mtime_le": "ffffff7f00000000",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 2147483648,
|
||||
"expected_time32": "EINTEGRITY",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-time32-upper-overflow",
|
||||
"mtime_le": "0000008000000000",
|
||||
"mtime_nsec_le": "00000000"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "PASS",
|
||||
"expected_seconds": 0,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-nsec-max",
|
||||
"mtime_le": "0000000000000000",
|
||||
"mtime_nsec_le": "ffc99a3b"
|
||||
},
|
||||
{
|
||||
"expected_nsec": "EINTEGRITY",
|
||||
"expected_seconds": 0,
|
||||
"expected_time32": "PASS",
|
||||
"expected_time64": "PASS",
|
||||
"id": "extended-nsec-overflow",
|
||||
"mtime_le": "0000000000000000",
|
||||
"mtime_nsec_le": "00ca9a3b"
|
||||
}
|
||||
],
|
||||
"encoding": {
|
||||
"mtime": "signed 64-bit little-endian two's-complement seconds",
|
||||
"mtime_nsec": "unsigned 32-bit little-endian nanoseconds"
|
||||
},
|
||||
"schema": 1
|
||||
}
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B14 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B14 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
fi
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' 'B14 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -q ' ZSTD_'; then
|
||||
printf '%s\n' 'B14 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
Executable
+661
@@ -0,0 +1,661 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
TESTS_DIR = SCRIPT_DIR.parent.parent
|
||||
sys.path.insert(0, str(TESTS_DIR))
|
||||
|
||||
from erofs_fixture import ErofsImage, FEATURE_INCOMPAT_48BIT, SUPER
|
||||
|
||||
|
||||
BLOCK_SIZE = 4096
|
||||
CHUNK_FORMAT_INDEXES = 0x0020
|
||||
CHUNK_FORMAT_48BIT = 0x0040
|
||||
CHUNK_FORMAT_ALL = 0x007F
|
||||
FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004
|
||||
EROFS_I_DOT_OMITTED_BIT = 4
|
||||
EROFS_I_NLINK_1_BIT = 5
|
||||
EROFS_FT_REG_FILE = 1
|
||||
EROFS_FT_DIR = 2
|
||||
EROFS_FT_CHRDEV = 3
|
||||
EROFS_FT_SYMLINK = 7
|
||||
UUID = "67360174-0014-0000-0000-000000000174"
|
||||
TARGET = b"target.txt"
|
||||
TARGET_CONTENT = b"B14 target content\n"
|
||||
ENTRY_CONTENT = b"B14 directory entry\n"
|
||||
|
||||
|
||||
class FixtureError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def run(command: list[str], *, capture: bool = False) -> str:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise FixtureError(
|
||||
f"command failed ({result.returncode}): {' '.join(command)}\n{result.stdout}"
|
||||
)
|
||||
return result.stdout if capture else ""
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def align_up(value: int, alignment: int) -> int:
|
||||
return (value + alignment - 1) & ~(alignment - 1)
|
||||
|
||||
|
||||
def source_bytes(label: str, size: int) -> bytes:
|
||||
chunks = []
|
||||
for index in range(math.ceil(size / 32)):
|
||||
chunks.append(hashlib.sha256(f"B14:{label}:{index}".encode("ascii")).digest())
|
||||
return b"".join(chunks)[:size]
|
||||
|
||||
|
||||
def make_source(root: Path) -> None:
|
||||
real_dir = root / "real-dir"
|
||||
real_dir.mkdir(parents=True)
|
||||
(real_dir / "entry.txt").write_bytes(ENTRY_CONTENT)
|
||||
(root / "target.txt").write_bytes(TARGET_CONTENT)
|
||||
(root / "chunk-dir").write_bytes(source_bytes("chunk-dir", 8192))
|
||||
link_data = TARGET + source_bytes("chunk-link", 8192 - len(TARGET))
|
||||
(root / "chunk-link").write_bytes(link_data)
|
||||
(root / "chunk-char").write_bytes(source_bytes("chunk-char", 8192))
|
||||
|
||||
|
||||
def make_image(mkfs: str, primary: Path, blob: Path, source: Path) -> None:
|
||||
run(
|
||||
[
|
||||
mkfs,
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
f"-U{UUID}",
|
||||
"--chunksize=4096",
|
||||
f"--blobdev={blob}",
|
||||
str(primary),
|
||||
str(source),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def root_entry(image: ErofsImage, path: str):
|
||||
_, entry = image.resolve_root_entry(path)
|
||||
return entry
|
||||
|
||||
|
||||
def inode_data(image: ErofsImage, inode) -> bytes:
|
||||
if inode.layout == 0:
|
||||
offset = inode.start_block << image.block_bits
|
||||
elif inode.layout == 2:
|
||||
offset = inode.offset + inode.inode_size + inode.xattr_size
|
||||
else:
|
||||
raise FixtureError(f"unsupported source data layout {inode.layout}")
|
||||
end = offset + inode.size
|
||||
if end > len(image.data):
|
||||
raise FixtureError("source inode data exceeds primary image")
|
||||
return bytes(image.data[offset:end])
|
||||
|
||||
|
||||
def chunk_info(image: ErofsImage, path: str) -> dict[str, int]:
|
||||
entry = root_entry(image, path)
|
||||
inode = image.inode(entry.nid)
|
||||
if inode.layout != 4:
|
||||
raise FixtureError(f"{path}: expected chunk layout, got {inode.layout}")
|
||||
chunk_format, reserved = struct.unpack_from("<HH", image.data, inode.offset + 16)
|
||||
entry_size = 8 if chunk_format & CHUNK_FORMAT_INDEXES else 4
|
||||
index_base = align_up(inode.offset + inode.inode_size + inode.xattr_size, entry_size)
|
||||
if entry_size == 8:
|
||||
high, device_id, low = struct.unpack_from("<HHI", image.data, index_base)
|
||||
else:
|
||||
high, device_id = 0, 0
|
||||
low = struct.unpack_from("<I", image.data, index_base)[0]
|
||||
return {
|
||||
"nid": inode.nid,
|
||||
"inode_offset": inode.offset,
|
||||
"inode_size": inode.inode_size,
|
||||
"format": chunk_format,
|
||||
"reserved": reserved,
|
||||
"index_base": index_base,
|
||||
"entry_size": entry_size,
|
||||
"high": high,
|
||||
"device_id": device_id,
|
||||
"low": low,
|
||||
"size": inode.size,
|
||||
"mode": inode.mode,
|
||||
}
|
||||
|
||||
|
||||
def patch_root_type(image: ErofsImage, path: str, file_type: int) -> None:
|
||||
entry = root_entry(image, path)
|
||||
offset = entry.offset + 10
|
||||
if image.data[offset] != EROFS_FT_REG_FILE:
|
||||
raise FixtureError(f"{path}: root entry is not a regular file")
|
||||
image.data[offset] = file_type
|
||||
|
||||
|
||||
def patch_mode(image: ErofsImage, path: str, mode_type: int) -> None:
|
||||
info = chunk_info(image, path)
|
||||
if stat.S_IFMT(info["mode"]) != stat.S_IFREG:
|
||||
raise FixtureError(f"{path}: carrier inode is not regular")
|
||||
image.put_u16(info["inode_offset"] + 4, mode_type | (info["mode"] & 0o7777))
|
||||
|
||||
|
||||
def patch_size(image: ErofsImage, path: str, size: int) -> None:
|
||||
info = chunk_info(image, path)
|
||||
if info["inode_size"] == 32:
|
||||
image.put_u32(info["inode_offset"] + 8, size)
|
||||
else:
|
||||
image.put_u64(info["inode_offset"] + 8, size)
|
||||
|
||||
|
||||
def patch_directory(image: ErofsImage, blob: bytearray) -> None:
|
||||
real_entry = root_entry(image, "/real-dir")
|
||||
real_inode = image.inode(real_entry.nid)
|
||||
directory_data = bytearray(inode_data(image, real_inode))
|
||||
info = chunk_info(image, "/chunk-dir")
|
||||
if info["device_id"] != 1 or info["high"] != 0:
|
||||
raise FixtureError("chunk directory carrier is not on external slot 1")
|
||||
blob_offset = info["low"] << image.block_bits
|
||||
if blob_offset + image.block_size > len(blob):
|
||||
raise FixtureError("chunk directory carrier exceeds blob")
|
||||
first_name_offset = struct.unpack_from("<H", directory_data, 8)[0]
|
||||
for index in range(first_name_offset // 12):
|
||||
entry_offset = index * 12
|
||||
name_offset = struct.unpack_from("<H", directory_data, entry_offset + 8)[0]
|
||||
end_offset = (
|
||||
struct.unpack_from("<H", directory_data, entry_offset + 20)[0]
|
||||
if index + 1 < first_name_offset // 12
|
||||
else len(directory_data)
|
||||
)
|
||||
name = bytes(directory_data[name_offset:end_offset]).split(b"\0", 1)[0]
|
||||
if name == b".":
|
||||
struct.pack_into("<Q", directory_data, entry_offset, info["nid"])
|
||||
elif name == b"..":
|
||||
struct.pack_into("<Q", directory_data, entry_offset, image.root_nid)
|
||||
blob[blob_offset : blob_offset + image.block_size] = b"\0" * image.block_size
|
||||
blob[blob_offset : blob_offset + len(directory_data)] = directory_data
|
||||
patch_mode(image, "/chunk-dir", stat.S_IFDIR)
|
||||
patch_size(image, "/chunk-dir", len(directory_data))
|
||||
inode_offset = info["inode_offset"]
|
||||
inode_format = image.u16(inode_offset)
|
||||
inode_format &= ~((1 << EROFS_I_DOT_OMITTED_BIT) | (1 << EROFS_I_NLINK_1_BIT))
|
||||
inode_format |= real_inode.inode_format & (1 << EROFS_I_DOT_OMITTED_BIT)
|
||||
image.put_u16(inode_offset, inode_format)
|
||||
if info["inode_size"] == 32:
|
||||
image.put_u16(inode_offset + 6, 2)
|
||||
else:
|
||||
image.put_u32(inode_offset + 40, 2)
|
||||
patch_root_type(image, "/chunk-dir", EROFS_FT_DIR)
|
||||
|
||||
|
||||
def patch_symlink(image: ErofsImage) -> None:
|
||||
patch_mode(image, "/chunk-link", stat.S_IFLNK)
|
||||
patch_size(image, "/chunk-link", len(TARGET))
|
||||
patch_root_type(image, "/chunk-link", EROFS_FT_SYMLINK)
|
||||
|
||||
|
||||
def patch_character(image: ErofsImage) -> None:
|
||||
patch_mode(image, "/chunk-char", stat.S_IFCHR)
|
||||
patch_size(image, "/chunk-char", 0)
|
||||
patch_root_type(image, "/chunk-char", EROFS_FT_CHRDEV)
|
||||
|
||||
|
||||
def enable_48bit(image: ErofsImage) -> None:
|
||||
if image.feature_incompat & FEATURE_INCOMPAT_48BIT:
|
||||
raise FixtureError("48-bit feature already enabled")
|
||||
root_nid = image.root_nid
|
||||
image.put_u32(SUPER + 80, image.feature_incompat | FEATURE_INCOMPAT_48BIT)
|
||||
image.put_u16(SUPER + 14, 0)
|
||||
image.put_u64(SUPER + 112, root_nid)
|
||||
for path in ("/chunk-dir", "/chunk-link", "/chunk-char"):
|
||||
info = chunk_info(image, path)
|
||||
if info["entry_size"] != 8 or info["high"] != 0:
|
||||
raise FixtureError(f"{path}: cannot promote chunk index to 48-bit")
|
||||
image.put_u16(info["inode_offset"] + 16, info["format"] | CHUNK_FORMAT_48BIT)
|
||||
|
||||
|
||||
def save_image(image: ErofsImage, path: Path) -> None:
|
||||
if image.feature_compat & 1:
|
||||
image.update_checksum()
|
||||
image.validate_superblock()
|
||||
image.save(path)
|
||||
|
||||
|
||||
def patch_index_low(image: ErofsImage, path: str, value: int) -> None:
|
||||
info = chunk_info(image, path)
|
||||
if info["entry_size"] != 8:
|
||||
raise FixtureError("bad-index fixture requires an indexed chunk")
|
||||
image.put_u32(info["index_base"] + 4, value)
|
||||
|
||||
|
||||
def patch_index48(image: ErofsImage, path: str, high: int, low: int) -> None:
|
||||
info = chunk_info(image, path)
|
||||
if info["entry_size"] != 8 or not info["format"] & CHUNK_FORMAT_48BIT:
|
||||
raise FixtureError("bad-index48 fixture requires a 48-bit chunk index")
|
||||
image.put_u16(info["index_base"], high)
|
||||
image.put_u32(info["index_base"] + 4, low)
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict[str, Any]:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B14":
|
||||
raise FixtureError("invalid B14 fixture schema")
|
||||
if spec.get("candidate") != "P15-056" or spec.get("test") != "TC174-chunk-nonregular":
|
||||
raise FixtureError("invalid B14 fixture identity")
|
||||
cases = spec.get("cases")
|
||||
if not isinstance(cases, list) or len(cases) != 10:
|
||||
raise FixtureError("B14 fixture case denominator changed")
|
||||
identifiers = [case.get("id") for case in cases]
|
||||
if len(set(identifiers)) != len(identifiers) or not all(isinstance(value, str) for value in identifiers):
|
||||
raise FixtureError("invalid or duplicate B14 case ID")
|
||||
return spec
|
||||
|
||||
|
||||
def write_sums(output: Path, names: list[str]) -> dict[str, str]:
|
||||
hashes = {name: sha256(output / name) for name in sorted(names)}
|
||||
with (output / "SHA256SUMS").open("w", encoding="ascii") as stream:
|
||||
for name, digest in hashes.items():
|
||||
stream.write(f"{digest} {name}\n")
|
||||
return hashes
|
||||
|
||||
|
||||
def generate(args: argparse.Namespace) -> None:
|
||||
output = args.output
|
||||
if output.exists():
|
||||
raise FixtureError(f"output already exists: {output}")
|
||||
spec = load_spec(args.spec)
|
||||
output.mkdir(parents=True)
|
||||
source = output / "source"
|
||||
source.mkdir()
|
||||
make_source(source)
|
||||
raw_primary = output / "raw.erofs"
|
||||
blob_path = output / "B14-slot1.blob"
|
||||
blob_path.write_bytes(b"")
|
||||
make_image(args.mkfs, raw_primary, blob_path, source)
|
||||
|
||||
image = ErofsImage.load(raw_primary)
|
||||
image.validate_superblock()
|
||||
if not image.feature_incompat & FEATURE_INCOMPAT_CHUNKED_FILE:
|
||||
raise FixtureError("mkfs image lacks CHUNKED_FILE")
|
||||
blob = bytearray(blob_path.read_bytes())
|
||||
for path in ("/chunk-dir", "/chunk-link", "/chunk-char"):
|
||||
info = chunk_info(image, path)
|
||||
if info["entry_size"] != 8 or info["device_id"] != 1 or info["high"] != 0:
|
||||
raise FixtureError(f"{path}: unexpected mkfs chunk carrier")
|
||||
|
||||
patch_directory(image, blob)
|
||||
patch_symlink(image)
|
||||
patch_character(image)
|
||||
blob_path.write_bytes(blob)
|
||||
good32 = image.clone()
|
||||
save_image(good32, output / "B14-good32.erofs")
|
||||
|
||||
good48 = image.clone()
|
||||
enable_48bit(good48)
|
||||
save_image(good48, output / "B14-good48.erofs")
|
||||
|
||||
blob_blocks = len(blob) // BLOCK_SIZE
|
||||
bad_index = good32.clone()
|
||||
patch_index_low(bad_index, "/chunk-link", blob_blocks + 1)
|
||||
save_image(bad_index, output / "B14-bad-index.erofs")
|
||||
|
||||
bad_index48 = good48.clone()
|
||||
patch_index48(bad_index48, "/chunk-link", 0xFFFF, 0xFFFFFFFE)
|
||||
save_image(bad_index48, output / "B14-bad-index48.erofs")
|
||||
|
||||
bad_reserved = good32.clone()
|
||||
directory = chunk_info(bad_reserved, "/chunk-dir")
|
||||
bad_reserved.put_u16(directory["inode_offset"] + 18, 1)
|
||||
save_image(bad_reserved, output / "B14-bad-reserved.erofs")
|
||||
|
||||
bad_format = good32.clone()
|
||||
link = chunk_info(bad_format, "/chunk-link")
|
||||
bad_format.put_u16(link["inode_offset"] + 16, link["format"] | (CHUNK_FORMAT_ALL + 1))
|
||||
save_image(bad_format, output / "B14-bad-format.erofs")
|
||||
|
||||
bad_no_index = good48.clone()
|
||||
link48 = chunk_info(bad_no_index, "/chunk-link")
|
||||
bad_no_index.put_u16(
|
||||
link48["inode_offset"] + 16,
|
||||
(link48["format"] | CHUNK_FORMAT_48BIT) & ~CHUNK_FORMAT_INDEXES,
|
||||
)
|
||||
save_image(bad_no_index, output / "B14-bad-48-no-index.erofs")
|
||||
raw_primary.unlink()
|
||||
|
||||
names = [
|
||||
"B14-slot1.blob",
|
||||
"B14-good32.erofs",
|
||||
"B14-good48.erofs",
|
||||
"B14-bad-index.erofs",
|
||||
"B14-bad-index48.erofs",
|
||||
"B14-bad-reserved.erofs",
|
||||
"B14-bad-format.erofs",
|
||||
"B14-bad-48-no-index.erofs",
|
||||
]
|
||||
hashes = write_sums(output, names)
|
||||
manifest = {
|
||||
"schema": 1,
|
||||
"batch": "B14",
|
||||
"candidate": "P15-056",
|
||||
"test": "TC174-chunk-nonregular",
|
||||
"mkfs_version": run([args.mkfs, "-V"], capture=True).splitlines()[0],
|
||||
"hashes": hashes,
|
||||
"blob_blocks": blob_blocks,
|
||||
"target": TARGET.decode("ascii"),
|
||||
"qualification": {
|
||||
"good32": "fsck.erofs plus independent parser",
|
||||
"good48": "independent parser; erofs-utils 1.8.6 lacks 48-bit support",
|
||||
},
|
||||
"cases": [case["id"] for case in spec["cases"]],
|
||||
"good32": {
|
||||
path: chunk_info(good32, path)
|
||||
for path in ("/chunk-dir", "/chunk-link", "/chunk-char")
|
||||
},
|
||||
"good48": {
|
||||
path: chunk_info(good48, path)
|
||||
for path in ("/chunk-dir", "/chunk-link", "/chunk-char")
|
||||
},
|
||||
}
|
||||
(output / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
|
||||
|
||||
def verify(args: argparse.Namespace) -> None:
|
||||
spec = load_spec(args.spec)
|
||||
output = args.output
|
||||
manifest = json.loads((output / "manifest.json").read_text(encoding="ascii"))
|
||||
if manifest.get("cases") != [case["id"] for case in spec["cases"]]:
|
||||
raise FixtureError("manifest case list differs from B14 spec")
|
||||
for name, expected in manifest.get("hashes", {}).items():
|
||||
if sha256(output / name) != expected:
|
||||
raise FixtureError(f"fixture hash mismatch: {name}")
|
||||
|
||||
good32 = ErofsImage.load(output / "B14-good32.erofs")
|
||||
good48 = ErofsImage.load(output / "B14-good48.erofs")
|
||||
blob = (output / "B14-slot1.blob").read_bytes()
|
||||
for image, width in ((good32, 32), (good48, 48)):
|
||||
image.validate_superblock()
|
||||
directory = chunk_info(image, "/chunk-dir")
|
||||
link = chunk_info(image, "/chunk-link")
|
||||
character = chunk_info(image, "/chunk-char")
|
||||
for info in (directory, link, character):
|
||||
if info["device_id"] != 1 or info["entry_size"] != 8:
|
||||
raise FixtureError(f"{width}-bit external chunk contract changed")
|
||||
has_48bit = bool(info["format"] & CHUNK_FORMAT_48BIT)
|
||||
if has_48bit != (width == 48) or info["high"] != 0:
|
||||
raise FixtureError(f"{width}-bit chunk format contract changed")
|
||||
if stat.S_IFMT(directory["mode"]) != stat.S_IFDIR:
|
||||
raise FixtureError("chunk directory mode changed")
|
||||
if stat.S_IFMT(link["mode"]) != stat.S_IFLNK or link["size"] != len(TARGET):
|
||||
raise FixtureError("chunk symlink mode or size changed")
|
||||
if stat.S_IFMT(character["mode"]) != stat.S_IFCHR or character["size"] != 0:
|
||||
raise FixtureError("chunk character mode or size changed")
|
||||
link_offset = link["low"] << image.block_bits
|
||||
if blob[link_offset : link_offset + len(TARGET)] != TARGET:
|
||||
raise FixtureError("chunk symlink payload changed")
|
||||
directory_offset = directory["low"] << image.block_bits
|
||||
directory_data = blob[directory_offset : directory_offset + directory["size"]]
|
||||
first_name_offset = struct.unpack_from("<H", directory_data, 8)[0]
|
||||
names = []
|
||||
nids = {}
|
||||
for index in range(first_name_offset // 12):
|
||||
entry_offset = index * 12
|
||||
name_offset = struct.unpack_from("<H", directory_data, entry_offset + 8)[0]
|
||||
end_offset = (
|
||||
struct.unpack_from("<H", directory_data, entry_offset + 20)[0]
|
||||
if index + 1 < first_name_offset // 12
|
||||
else len(directory_data)
|
||||
)
|
||||
name = bytes(directory_data[name_offset:end_offset]).split(b"\0", 1)[0]
|
||||
names.append(name)
|
||||
nids[name] = struct.unpack_from("<Q", directory_data, entry_offset)[0]
|
||||
if b"entry.txt" not in names or nids.get(b".", directory["nid"]) != directory["nid"]:
|
||||
raise FixtureError("chunk directory payload identity changed")
|
||||
|
||||
bad_index = ErofsImage.load(output / "B14-bad-index.erofs")
|
||||
if chunk_info(bad_index, "/chunk-link")["low"] <= manifest["blob_blocks"]:
|
||||
raise FixtureError("32-bit bad index remains inside external device")
|
||||
bad_index48 = chunk_info(ErofsImage.load(output / "B14-bad-index48.erofs"), "/chunk-link")
|
||||
if bad_index48["high"] != 0xFFFF or bad_index48["low"] != 0xFFFFFFFE:
|
||||
raise FixtureError("48-bit bad index boundary changed")
|
||||
if chunk_info(ErofsImage.load(output / "B14-bad-reserved.erofs"), "/chunk-dir")["reserved"] != 1:
|
||||
raise FixtureError("reserved-field mutation changed")
|
||||
if not chunk_info(ErofsImage.load(output / "B14-bad-format.erofs"), "/chunk-link")["format"] & 0x80:
|
||||
raise FixtureError("unsupported-format mutation changed")
|
||||
no_index = chunk_info(ErofsImage.load(output / "B14-bad-48-no-index.erofs"), "/chunk-link")
|
||||
if not no_index["format"] & CHUNK_FORMAT_48BIT or no_index["format"] & CHUNK_FORMAT_INDEXES:
|
||||
raise FixtureError("48-bit-without-indexes mutation changed")
|
||||
|
||||
run(
|
||||
[
|
||||
args.fsck,
|
||||
f"--device={output / 'B14-slot1.blob'}",
|
||||
str(output / "B14-good32.erofs"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def extract_function(source: str, name: str) -> str:
|
||||
match = re.search(
|
||||
r"^(?:[A-Za-z_][^\n;{}]*\s+)?" + re.escape(name) + r"\s*\(",
|
||||
source,
|
||||
re.MULTILINE,
|
||||
)
|
||||
if not match:
|
||||
raise FixtureError(f"missing function: {name}")
|
||||
start = source.rfind("\n\n", 0, match.start()) + 2
|
||||
brace = source.find("{", match.end())
|
||||
depth = 0
|
||||
state = "code"
|
||||
index = brace
|
||||
while index < len(source):
|
||||
char = source[index]
|
||||
following = source[index + 1] if index + 1 < len(source) else ""
|
||||
if state == "code":
|
||||
if char == "/" and following == "*":
|
||||
state = "block"
|
||||
index += 2
|
||||
continue
|
||||
if char == "/" and following == "/":
|
||||
state = "line"
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
state = "string"
|
||||
elif char == "'":
|
||||
state = "character"
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start : index + 1]
|
||||
elif state == "block" and char == "*" and following == "/":
|
||||
state = "code"
|
||||
index += 2
|
||||
continue
|
||||
elif state == "line" and char == "\n":
|
||||
state = "code"
|
||||
elif state in {"string", "character"}:
|
||||
if char == "\\":
|
||||
index += 2
|
||||
continue
|
||||
if (state == "string" and char == '"') or (state == "character" and char == "'"):
|
||||
state = "code"
|
||||
index += 1
|
||||
raise FixtureError(f"unterminated function: {name}")
|
||||
|
||||
|
||||
def committed(root: Path, baseline: str, path: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/{path}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise FixtureError(f"cannot read B14 baseline {path}: {result.stderr}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def audit(args: argparse.Namespace) -> None:
|
||||
load_spec(args.spec)
|
||||
root = args.root
|
||||
dut = args.dut
|
||||
inode = (dut / "src/inode.c").read_text(encoding="utf-8")
|
||||
data = (dut / "src/data.c").read_text(encoding="utf-8")
|
||||
vnops = (dut / "src/erofs_vnops.c").read_text(encoding="utf-8")
|
||||
linux_inode = (root / "src-linux/inode.c").read_text(encoding="utf-8")
|
||||
base_inode = committed(root, args.baseline, "src/inode.c")
|
||||
base_data = committed(root, args.baseline, "src/data.c")
|
||||
old_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi) || vi->vtype != VREG) {"
|
||||
new_gate = "\t\tif (!erofs_sb_has_chunked_file(sbi)) {"
|
||||
if base_inode.count(old_gate) != 1:
|
||||
raise FixtureError("B14 baseline type gate changed")
|
||||
expected_inode = base_inode.replace(old_gate, new_gate, 1)
|
||||
if inode != expected_inode:
|
||||
raise FixtureError("inode.c differs from the single declared B14 transform")
|
||||
if data != base_data:
|
||||
raise FixtureError("data.c changed although the chunk mapper is already type-neutral")
|
||||
|
||||
read_inode = extract_function(inode, "erofs_read_inode")
|
||||
chunk_mapper = extract_function(data, "erofs_map_blocks_chunk")
|
||||
map_device = extract_function(data, "erofs_map_dev")
|
||||
open_vnode = extract_function(vnops, "erofs_open")
|
||||
read_vnode = extract_function(vnops, "erofs_read")
|
||||
readdir_vnode = extract_function(vnops, "erofs_readdir")
|
||||
readlink_vnode = extract_function(vnops, "erofs_readlink")
|
||||
linux_read_inode = extract_function(linux_inode, "erofs_read_inode")
|
||||
|
||||
required_inode = (
|
||||
"!erofs_sb_has_chunked_file(sbi)",
|
||||
"le16toh(chunk_info.reserved) != 0",
|
||||
"vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL",
|
||||
"EROFS_CHUNK_FORMAT_48BIT",
|
||||
"EROFS_CHUNK_FORMAT_INDEXES",
|
||||
"vi->chunkbits >= 64",
|
||||
"case VREG:",
|
||||
"case VDIR:",
|
||||
"case VLNK:",
|
||||
)
|
||||
for marker in required_inode:
|
||||
if marker not in read_inode:
|
||||
raise FixtureError(f"B14 removed inode invariant: {marker}")
|
||||
if "vi->vtype != VREG" in read_inode:
|
||||
raise FixtureError("nonregular chunk type gate remains")
|
||||
if read_inode.count("return (EINTEGRITY);") < 8 or "return (EOPNOTSUPP);" not in read_inode:
|
||||
raise FixtureError("FreeBSD inode errno contract changed")
|
||||
|
||||
required_mapper = (
|
||||
"vi->chunkbits",
|
||||
"vi->chunkformat",
|
||||
"vi->inode_off",
|
||||
"vi->inode_isize",
|
||||
"vi->xattr_isize",
|
||||
"map->m_deviceid = raw_device_id & sbi->device_id_mask",
|
||||
"map->m_flags |= EROFS_MAP_MAPPED",
|
||||
)
|
||||
for marker in required_mapper:
|
||||
if marker not in chunk_mapper:
|
||||
raise FixtureError(f"chunk mapper contract changed: {marker}")
|
||||
if "vtype" in chunk_mapper or "VREG" in chunk_mapper or "VLNK" in chunk_mapper or "VDIR" in chunk_mapper:
|
||||
raise FixtureError("chunk mapper acquired an OS vnode type dependency")
|
||||
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", chunk_mapper + map_device):
|
||||
raise FixtureError("negative Linux errno entered the FreeBSD map path")
|
||||
for marker in ("ENODEV", "dif->devvp", "dif->cp", "erofs_check_device_range"):
|
||||
if marker not in map_device:
|
||||
raise FixtureError(f"FreeBSD device/GEOM boundary changed: {marker}")
|
||||
|
||||
if "VN_ISDEV(vp)" not in open_vnode or "return (EOPNOTSUPP);" not in open_vnode:
|
||||
raise FixtureError("special vnode open safety changed")
|
||||
if "case VREG:" not in read_vnode or "case VDIR:" not in read_vnode:
|
||||
raise FixtureError("FreeBSD VOP_READ type boundary changed")
|
||||
if "v_type != VDIR" not in readdir_vnode or "v_type != VLNK" not in readlink_vnode:
|
||||
raise FixtureError("FreeBSD readdir/readlink type boundary changed")
|
||||
|
||||
for marker in ("case S_IFDIR:", "case S_IFREG:", "case S_IFLNK:"):
|
||||
if marker not in linux_read_inode:
|
||||
raise FixtureError(f"Linux common data-layout type marker changed: {marker}")
|
||||
linux_chunk = linux_read_inode[linux_read_inode.index("if (vi->datalayout == EROFS_INODE_CHUNK_BASED)") :]
|
||||
if "vi->chunkformat" not in linux_chunk or "S_ISREG" in linux_chunk.split("inode_set_atime", 1)[0]:
|
||||
raise FixtureError("Linux chunk summary is no longer type-neutral")
|
||||
|
||||
report = {
|
||||
"schema": 1,
|
||||
"batch": "B14",
|
||||
"candidate": "P15-056",
|
||||
"baseline": args.baseline,
|
||||
"source_transform_count": 1,
|
||||
"data_c_unchanged": True,
|
||||
"linux_common_types": ["S_IFDIR", "S_IFREG", "S_IFLNK"],
|
||||
"freebsd_vnode_types": ["VDIR", "VREG", "VLNK"],
|
||||
"freebsd_boundaries": [
|
||||
"positive errno",
|
||||
"explicit GEOM device slots",
|
||||
"special vnode open EOPNOTSUPP",
|
||||
"typed readdir/readlink VOPs",
|
||||
],
|
||||
}
|
||||
args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
generate_parser = subparsers.add_parser("generate")
|
||||
generate_parser.add_argument("--output", type=Path, required=True)
|
||||
generate_parser.add_argument("--spec", type=Path, required=True)
|
||||
generate_parser.add_argument("--mkfs", default="mkfs.erofs")
|
||||
verify_parser = subparsers.add_parser("verify")
|
||||
verify_parser.add_argument("--output", type=Path, required=True)
|
||||
verify_parser.add_argument("--spec", type=Path, required=True)
|
||||
verify_parser.add_argument("--fsck", default="fsck.erofs")
|
||||
audit_parser = subparsers.add_parser("audit")
|
||||
audit_parser.add_argument("--root", type=Path, required=True)
|
||||
audit_parser.add_argument("--dut", type=Path, required=True)
|
||||
audit_parser.add_argument("--baseline", required=True)
|
||||
audit_parser.add_argument("--spec", type=Path, required=True)
|
||||
audit_parser.add_argument("--report", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.command == "generate":
|
||||
generate(args)
|
||||
elif args.command == "verify":
|
||||
verify(args)
|
||||
else:
|
||||
audit(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "%s\n", message);
|
||||
return (1);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
char target[256];
|
||||
ssize_t length;
|
||||
int descriptor;
|
||||
|
||||
if (argc < 3)
|
||||
return (fail("usage: B14-chunk-probe OP PATH [TARGET]"));
|
||||
if (strcmp(argv[1], "readlink-pass") == 0) {
|
||||
if (argc != 4)
|
||||
return (fail("readlink-pass requires an expected target"));
|
||||
length = readlink(argv[2], target, sizeof(target));
|
||||
if (length < 0)
|
||||
return (fail("readlink-pass unexpectedly failed"));
|
||||
if ((size_t)length != strlen(argv[3]) ||
|
||||
memcmp(target, argv[3], (size_t)length) != 0)
|
||||
return (fail("readlink-pass returned the wrong target"));
|
||||
return (0);
|
||||
}
|
||||
if (strcmp(argv[1], "readlink-integrity") == 0) {
|
||||
errno = 0;
|
||||
if (readlink(argv[2], target, sizeof(target)) >= 0 ||
|
||||
errno != EINTEGRITY)
|
||||
return (fail("readlink did not return EINTEGRITY"));
|
||||
return (0);
|
||||
}
|
||||
if (strcmp(argv[1], "open-unsupported") == 0) {
|
||||
errno = 0;
|
||||
descriptor = open(argv[2], O_RDONLY | O_NONBLOCK);
|
||||
if (descriptor >= 0) {
|
||||
close(descriptor);
|
||||
return (fail("special vnode unexpectedly opened"));
|
||||
}
|
||||
if (errno != EOPNOTSUPP)
|
||||
return (fail("special vnode did not return EOPNOTSUPP"));
|
||||
return (0);
|
||||
}
|
||||
if (strcmp(argv[1], "stat-integrity") == 0 ||
|
||||
strcmp(argv[1], "stat-unsupported") == 0) {
|
||||
struct stat status;
|
||||
int expected;
|
||||
|
||||
expected = strcmp(argv[1], "stat-integrity") == 0 ?
|
||||
EINTEGRITY : EOPNOTSUPP;
|
||||
errno = 0;
|
||||
if (lstat(argv[2], &status) == 0 || errno != expected)
|
||||
return (fail("stat did not return the expected errno"));
|
||||
return (0);
|
||||
}
|
||||
return (fail("unknown B14 probe operation"));
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"batch": "B14",
|
||||
"candidate": "P15-056",
|
||||
"test": "TC174-chunk-nonregular",
|
||||
"cases": [
|
||||
{
|
||||
"id": "directory-indexed32-external",
|
||||
"fixture": "B14-good32.erofs",
|
||||
"path": "/chunk-dir",
|
||||
"vnode": "VDIR",
|
||||
"operation": "lookup-readdir",
|
||||
"device_id": 1,
|
||||
"expected": "PASS"
|
||||
},
|
||||
{
|
||||
"id": "symlink-indexed32-external",
|
||||
"fixture": "B14-good32.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "follow-readlink",
|
||||
"device_id": 1,
|
||||
"expected": "PASS"
|
||||
},
|
||||
{
|
||||
"id": "character-vnode-not-operational",
|
||||
"fixture": "B14-good32.erofs",
|
||||
"path": "/chunk-char",
|
||||
"vnode": "VCHR",
|
||||
"operation": "stat-open",
|
||||
"device_id": 1,
|
||||
"expected": "STAT_PASS_OPEN_EOPNOTSUPP"
|
||||
},
|
||||
{
|
||||
"id": "directory-indexed48-zero-high",
|
||||
"fixture": "B14-good48.erofs",
|
||||
"path": "/chunk-dir",
|
||||
"vnode": "VDIR",
|
||||
"operation": "lookup-readdir",
|
||||
"device_id": 1,
|
||||
"expected": "PASS"
|
||||
},
|
||||
{
|
||||
"id": "symlink-indexed48-zero-high",
|
||||
"fixture": "B14-good48.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "follow-readlink",
|
||||
"device_id": 1,
|
||||
"expected": "PASS"
|
||||
},
|
||||
{
|
||||
"id": "symlink-index-outside-device",
|
||||
"fixture": "B14-bad-index.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "readlink",
|
||||
"device_id": 1,
|
||||
"expected": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "symlink-index48-outside-device",
|
||||
"fixture": "B14-bad-index48.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "readlink",
|
||||
"device_id": 1,
|
||||
"expected": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "directory-reserved-nonzero",
|
||||
"fixture": "B14-bad-reserved.erofs",
|
||||
"path": "/chunk-dir",
|
||||
"vnode": "VDIR",
|
||||
"operation": "stat",
|
||||
"device_id": 1,
|
||||
"expected": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"id": "symlink-unsupported-format",
|
||||
"fixture": "B14-bad-format.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "stat",
|
||||
"device_id": 1,
|
||||
"expected": "EOPNOTSUPP"
|
||||
},
|
||||
{
|
||||
"id": "symlink-48bit-without-indexes",
|
||||
"fixture": "B14-bad-48-no-index.erofs",
|
||||
"path": "/chunk-link",
|
||||
"vnode": "VLNK",
|
||||
"operation": "stat",
|
||||
"device_id": 0,
|
||||
"expected": "EINTEGRITY"
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B16 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B16 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
fi
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' 'B16 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -q ' ZSTD_'; then
|
||||
printf '%s\n' 'B16 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,158 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define PROBE_BUFFER_SIZE 2048
|
||||
|
||||
static const char pattern[] = "p15-019-safe-target/";
|
||||
|
||||
static int
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "%s\n", message);
|
||||
return (1);
|
||||
}
|
||||
|
||||
static int
|
||||
parse_positive(const char *text, long minimum, long maximum, long *value)
|
||||
{
|
||||
char *end;
|
||||
long parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtol(text, &end, 10);
|
||||
if (errno != 0 || *text == '\0' || *end != '\0' || parsed < minimum ||
|
||||
parsed > maximum)
|
||||
return (-1);
|
||||
*value = parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static void
|
||||
fill_expected(char *buffer, size_t length)
|
||||
{
|
||||
size_t index;
|
||||
|
||||
for (index = 0; index < length; index++)
|
||||
buffer[index] = pattern[index % (sizeof(pattern) - 1)];
|
||||
}
|
||||
|
||||
static int
|
||||
readlink_pass(const char *path, size_t expected_length)
|
||||
{
|
||||
char actual[PROBE_BUFFER_SIZE];
|
||||
char expected[PROBE_BUFFER_SIZE];
|
||||
ssize_t length;
|
||||
|
||||
if (expected_length + 1 > sizeof(actual))
|
||||
return (fail("expected length exceeds probe buffer"));
|
||||
memset(actual, 0xa5, sizeof(actual));
|
||||
fill_expected(expected, expected_length);
|
||||
errno = 0;
|
||||
length = readlink(path, actual, sizeof(actual));
|
||||
if (length < 0)
|
||||
return (fail("readlink-pass returned an error"));
|
||||
if ((size_t)length != expected_length)
|
||||
return (fail("readlink-pass returned the wrong length"));
|
||||
if (memcmp(actual, expected, expected_length) != 0)
|
||||
return (fail("readlink-pass returned the wrong target"));
|
||||
if ((unsigned char)actual[expected_length] != 0xa5)
|
||||
return (fail("readlink-pass appended or copied an extra byte"));
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
readlink_errno(const char *path, int expected_errno)
|
||||
{
|
||||
char actual[PROBE_BUFFER_SIZE];
|
||||
ssize_t length;
|
||||
size_t index;
|
||||
|
||||
memset(actual, 0xa5, sizeof(actual));
|
||||
errno = 0;
|
||||
length = readlink(path, actual, sizeof(actual));
|
||||
if (length != -1 || errno != expected_errno)
|
||||
return (fail("readlink-errno returned the wrong result"));
|
||||
for (index = 0; index < sizeof(actual); index++) {
|
||||
if ((unsigned char)actual[index] != 0xa5)
|
||||
return (fail("failed readlink modified the caller buffer"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
follow_errno(const char *path, int expected_errno)
|
||||
{
|
||||
struct stat status;
|
||||
|
||||
errno = 0;
|
||||
if (stat(path, &status) != -1 || errno != expected_errno)
|
||||
return (fail("follow-errno returned the wrong result"));
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
concurrent_readlink(const char *path, size_t length, long workers, long loops)
|
||||
{
|
||||
long worker;
|
||||
int status;
|
||||
pid_t child;
|
||||
|
||||
for (worker = 0; worker < workers; worker++) {
|
||||
child = fork();
|
||||
if (child < 0)
|
||||
return (fail("fork failed"));
|
||||
if (child == 0) {
|
||||
for (long iteration = 0; iteration < loops; iteration++) {
|
||||
if (readlink_pass(path, length) != 0)
|
||||
_exit(1);
|
||||
}
|
||||
_exit(0);
|
||||
}
|
||||
}
|
||||
for (worker = 0; worker < workers; worker++) {
|
||||
if (wait(&status) < 0 || !WIFEXITED(status) ||
|
||||
WEXITSTATUS(status) != 0)
|
||||
return (fail("concurrent readlink worker failed"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
long first, second, third;
|
||||
|
||||
if (argc == 4 && strcmp(argv[1], "readlink-pass") == 0) {
|
||||
if (parse_positive(argv[3], 1, PROBE_BUFFER_SIZE - 1, &first) != 0)
|
||||
return (fail("invalid readlink-pass length"));
|
||||
return (readlink_pass(argv[2], (size_t)first));
|
||||
}
|
||||
if (argc == 4 && strcmp(argv[1], "readlink-errno") == 0) {
|
||||
if (parse_positive(argv[3], 1, 255, &first) != 0)
|
||||
return (fail("invalid expected errno"));
|
||||
return (readlink_errno(argv[2], (int)first));
|
||||
}
|
||||
if (argc == 4 && strcmp(argv[1], "follow-errno") == 0) {
|
||||
if (parse_positive(argv[3], 1, 255, &first) != 0)
|
||||
return (fail("invalid expected errno"));
|
||||
return (follow_errno(argv[2], (int)first));
|
||||
}
|
||||
if (argc == 6 && strcmp(argv[1], "concurrent") == 0) {
|
||||
if (parse_positive(argv[3], 1, PROBE_BUFFER_SIZE - 1, &first) != 0 ||
|
||||
parse_positive(argv[4], 1, 64, &second) != 0 ||
|
||||
parse_positive(argv[5], 1, 1000, &third) != 0)
|
||||
return (fail("invalid concurrent arguments"));
|
||||
return (concurrent_readlink(argv[2], (size_t)first, second, third));
|
||||
}
|
||||
return (fail("usage: B16-symlink-probe MODE PATH ARG..."));
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"batch": "B16",
|
||||
"candidate": "P15-019",
|
||||
"cases": [
|
||||
"normal-short",
|
||||
"normal-max",
|
||||
"empty",
|
||||
"nul-first",
|
||||
"nul-middle",
|
||||
"nul-last",
|
||||
"too-long"
|
||||
],
|
||||
"errno": {
|
||||
"EINTEGRITY": 97,
|
||||
"ENAMETOOLONG": 63
|
||||
},
|
||||
"fixture_set_sha256": "5e99e5ad9112508991e78ae6e65928973d6b0da78285a059894bed06010cf373",
|
||||
"gate": {
|
||||
"base": "d645feb720c7022d2138d2a62eb72c022eb75351",
|
||||
"commit": "aea34aba38a298d493d0a584f3985cf3589f358e",
|
||||
"input_sha256": "291bf12838981ef650e16bbf7381eaa9b2921de957e97dec46e6a573750a19b8",
|
||||
"script_sha256": "afa20cbd28ca9dfc8064596a312248756ed7cde54398efb5ef713818cc04ea54"
|
||||
},
|
||||
"layouts": {
|
||||
"chunk": {
|
||||
"layout": 4,
|
||||
"short_length": 31
|
||||
},
|
||||
"compressed": {
|
||||
"layout": 3,
|
||||
"short_length": 64
|
||||
},
|
||||
"fragment": {
|
||||
"layout": 1,
|
||||
"short_length": 31
|
||||
},
|
||||
"inline": {
|
||||
"layout": 2,
|
||||
"short_length": 31
|
||||
},
|
||||
"plain": {
|
||||
"layout": 0,
|
||||
"short_length": 31
|
||||
}
|
||||
},
|
||||
"maxpathlen": 1024,
|
||||
"qemu_cases": [
|
||||
"normal-short",
|
||||
"normal-max",
|
||||
"empty",
|
||||
"nul-middle",
|
||||
"too-long"
|
||||
],
|
||||
"schema": 1,
|
||||
"test": "TC166-symlink-layouts"
|
||||
}
|
||||
Executable
+529
@@ -0,0 +1,529 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic B17 EROFS xattr images from one audited seed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER_OFFSET = 1024
|
||||
FEATURE_COMPAT_XATTR_FILTER = 0x00000004
|
||||
ACL_FILTER_BITS = (1 << 21) | (1 << 30)
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER_OFFSET + 144:
|
||||
raise ValueError("seed is shorter than the EROFS superblock")
|
||||
if self.u32(SUPER_OFFSET) != 0xE0F5E1E2:
|
||||
raise ValueError("seed has the wrong EROFS magic")
|
||||
self.block_bits = self.data[SUPER_OFFSET + 12]
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER_OFFSET + 40)
|
||||
self.xattr_blkaddr = self.u32(SUPER_OFFSET + 44)
|
||||
self.root_nid = self.u16(SUPER_OFFSET + 14)
|
||||
self.packed_nid = self.u64(SUPER_OFFSET + 96)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def put_u16(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<H", self.data, offset, value)
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_count": xattr_count,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inline_data_offset(self, inode: dict[str, int]) -> int:
|
||||
if inode["layout"] != 2:
|
||||
raise ValueError("fixture seed expected flat-inline metadata")
|
||||
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
start = self.inline_data_offset(inode)
|
||||
data = self.data[start : start + inode["size"]]
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name == 0 or first_name % 12 != 0 or first_name > len(data):
|
||||
raise ValueError("seed root directory is malformed")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
name_start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
if index + 1 < count:
|
||||
name_end = struct.unpack_from("<H", data, entry + 20)[0]
|
||||
else:
|
||||
name_end = len(data)
|
||||
name = bytes(data[name_start:name_end]).split(b"\0", 1)[0]
|
||||
entries.append((name, nid))
|
||||
return entries
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
nid = next(
|
||||
(candidate for name, candidate in self.directory_entries(inode)
|
||||
if name == component),
|
||||
None,
|
||||
)
|
||||
if nid is None:
|
||||
raise ValueError(f"seed path is absent: {path}")
|
||||
inode = self.inode(nid)
|
||||
return inode
|
||||
|
||||
def body_offset(self, inode: dict[str, int]) -> int:
|
||||
return inode["offset"] + inode["inode_size"]
|
||||
|
||||
def first_inline_entry(self, path: str) -> tuple[dict[str, int], int]:
|
||||
inode = self.resolve(path)
|
||||
body = self.body_offset(inode)
|
||||
shared_count = self.data[body + 4]
|
||||
entry = body + 12 + shared_count * 4
|
||||
if entry + 4 > body + inode["xattr_size"]:
|
||||
raise ValueError(f"seed path has no inline xattr entry: {path}")
|
||||
return inode, entry
|
||||
|
||||
def first_shared_entry(self, path: str) -> tuple[dict[str, int], int, int]:
|
||||
inode = self.resolve(path)
|
||||
body = self.body_offset(inode)
|
||||
if self.data[body + 4] == 0:
|
||||
raise ValueError(f"seed path has no shared xattr id: {path}")
|
||||
shared_id = self.u32(body + 12)
|
||||
entry = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
|
||||
return inode, body, entry
|
||||
|
||||
def prefix_record_offset(self) -> int:
|
||||
if self.packed_nid == 0:
|
||||
raise ValueError("seed has no packed prefix carrier")
|
||||
packed = self.inode(self.packed_nid)
|
||||
logical = self.u32(SUPER_OFFSET + 92) << 2
|
||||
if logical + 2 > packed["size"]:
|
||||
raise ValueError("seed prefix header is outside packed data")
|
||||
return self.inline_data_offset(packed) + logical
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
checksum = SUPER_OFFSET + 4
|
||||
self.put_u32(checksum, 0)
|
||||
end = SUPER_OFFSET + self.block_size - SUPER_OFFSET
|
||||
if end > len(self.data):
|
||||
raise ValueError("seed does not contain the checksummed block")
|
||||
self.put_u32(checksum, crc32c(self.data[SUPER_OFFSET:end]))
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
value = json.loads(path.read_text(encoding="ascii"))
|
||||
if value.get("schema") != 1 or value.get("batch") != "B17":
|
||||
raise SystemExit("invalid B17 fixture spec identity")
|
||||
cases = value.get("cases")
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise SystemExit("B17 fixture spec has no cases")
|
||||
identifiers = [case.get("id") for case in cases]
|
||||
if any(not isinstance(item, str) for item in identifiers):
|
||||
raise SystemExit("B17 fixture case has an invalid id")
|
||||
if len(identifiers) != len(set(identifiers)):
|
||||
raise SystemExit("B17 fixture case ids are not unique")
|
||||
return value
|
||||
|
||||
|
||||
def create_source(root: Path) -> None:
|
||||
root.mkdir(parents=True)
|
||||
root.chmod(0o755)
|
||||
files = {
|
||||
"acl.bin": b"acl\n",
|
||||
"inline.bin": b"inline\n",
|
||||
"prefix.bin": b"prefix\n",
|
||||
"shared-0.bin": b"shared-0\n",
|
||||
"shared-1.bin": b"shared-1\n",
|
||||
"shared-2.bin": b"shared-2\n",
|
||||
"shared-3.bin": b"shared-3\n",
|
||||
}
|
||||
for name, payload in files.items():
|
||||
path = root / name
|
||||
path.write_bytes(payload)
|
||||
path.chmod(0o644)
|
||||
os.setxattr(root / "acl.bin", b"user.aclx", bytes([0xA5]) * 24)
|
||||
os.setxattr(root / "inline.bin", b"user.alpha", b"value\0binary\xff")
|
||||
os.setxattr(
|
||||
root / "prefix.bin", b"user.company.branch.leaf", b"prefix-value"
|
||||
)
|
||||
for index in range(4):
|
||||
os.setxattr(
|
||||
root / f"shared-{index}.bin", b"user.shared", b"shared-value"
|
||||
)
|
||||
for path in sorted(root.iterdir()):
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
os.utime(root, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def build_seed(spec: dict, output: Path, work: Path) -> dict[str, object]:
|
||||
mkfs = shutil.which("mkfs.erofs")
|
||||
if mkfs is None:
|
||||
raise SystemExit("mkfs.erofs is required to rebuild the B17 seed")
|
||||
version_run = subprocess.run(
|
||||
[mkfs, "-V"], check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
version = version_run.stdout.splitlines()[0] if version_run.stdout else ""
|
||||
if version != spec["seed"]["mkfs_version"]:
|
||||
raise SystemExit(
|
||||
f"mkfs.erofs version differs: expected {spec['seed']['mkfs_version']!r}, "
|
||||
f"got {version!r}"
|
||||
)
|
||||
source = work / "source"
|
||||
create_source(source)
|
||||
command = [
|
||||
mkfs,
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"--sort=path",
|
||||
f"-U{spec['seed']['uuid']}",
|
||||
"-x2",
|
||||
"-Exattr-name-filter,force-inode-extended",
|
||||
"--xattr-prefix=user.company.",
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command, check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
f"mkfs.erofs failed with {completed.returncode}:\n{completed.stdout}"
|
||||
)
|
||||
image = Image.load(output)
|
||||
packed = image.inode(image.packed_nid)
|
||||
image.put_u16(packed["offset"] + 4, 0o100644)
|
||||
image.update_checksum()
|
||||
output.write_bytes(image.data)
|
||||
return {"version": version, "command": command, "stdout": completed.stdout}
|
||||
|
||||
|
||||
def configure_acl(image: Image, direct: list[dict[str, object]]) -> None:
|
||||
inode, entry = image.first_inline_entry("/acl.bin")
|
||||
if image.data[entry] != 4 or image.data[entry + 1] != 1:
|
||||
raise ValueError("seed ACL staging entry shape changed")
|
||||
if image.u16(entry + 2) != 24:
|
||||
raise ValueError("seed ACL staging value shape changed")
|
||||
acl = b"".join(
|
||||
[
|
||||
struct.pack("<I", 2),
|
||||
struct.pack("<HHI", 1, 6, 0xFFFFFFFF),
|
||||
struct.pack("<HHI", 4, 0, 0xFFFFFFFF),
|
||||
struct.pack("<HHI", 32, 0, 0xFFFFFFFF),
|
||||
]
|
||||
)
|
||||
image.data[entry] = 0
|
||||
image.data[entry + 1] = 2
|
||||
image.put_u16(entry + 2, len(acl))
|
||||
image.data[entry + 4 : entry + 4 + len(acl)] = acl
|
||||
body = image.body_offset(inode)
|
||||
image.put_u32(body, image.u32(body) & ~ACL_FILTER_BITS)
|
||||
direct.extend(
|
||||
[
|
||||
{"field": "acl.empty-suffix-entry", "offset": entry, "bytes": 32},
|
||||
{"field": "acl.h_name_filter", "offset": body, "bytes": 4},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def apply_mutation(image: Image, name: str) -> list[dict[str, object]]:
|
||||
direct: list[dict[str, object]] = []
|
||||
if name == "none":
|
||||
return direct
|
||||
if name == "acl-empty-suffix":
|
||||
configure_acl(image, direct)
|
||||
elif name in {"filter-reserved-one", "filter-reserved-255"}:
|
||||
value = 1 if name.endswith("one") else 255
|
||||
offset = SUPER_OFFSET + 104
|
||||
image.data[offset] = value
|
||||
direct.append({"field": "super.xattr_filter_reserved", "offset": offset, "bytes": 1})
|
||||
elif name in {"filter-feature-off", "filter-feature-off-reserved-one"}:
|
||||
offset = SUPER_OFFSET + 8
|
||||
image.put_u32(offset, image.u32(offset) & ~FEATURE_COMPAT_XATTR_FILTER)
|
||||
direct.append({"field": "super.feature_compat", "offset": offset, "bytes": 4})
|
||||
if name.endswith("reserved-one"):
|
||||
reserved = SUPER_OFFSET + 104
|
||||
image.data[reserved] = 1
|
||||
direct.append({"field": "super.xattr_filter_reserved", "offset": reserved, "bytes": 1})
|
||||
elif name == "unknown-filter-acl-present":
|
||||
configure_acl(image, direct)
|
||||
inode = image.resolve("/acl.bin")
|
||||
body = image.body_offset(inode)
|
||||
image.put_u32(body, image.u32(body) | ACL_FILTER_BITS)
|
||||
image.data[SUPER_OFFSET + 104] = 1
|
||||
direct.extend(
|
||||
[
|
||||
{"field": "ibody.h_name_filter", "offset": body, "bytes": 4},
|
||||
{"field": "super.xattr_filter_reserved", "offset": SUPER_OFFSET + 104, "bytes": 1},
|
||||
]
|
||||
)
|
||||
elif name in {"inline-name-nul", "short-name-index", "inline-value-size"}:
|
||||
_, entry = image.first_inline_entry("/inline.bin")
|
||||
if name == "inline-name-nul":
|
||||
offset = entry + 4 + image.data[entry] // 2
|
||||
image.data[offset] = 0
|
||||
direct.append({"field": "inline.e_name", "offset": offset, "bytes": 1})
|
||||
elif name == "short-name-index":
|
||||
image.data[entry + 1] = 7
|
||||
direct.append({"field": "inline.e_name_index", "offset": entry + 1, "bytes": 1})
|
||||
else:
|
||||
image.put_u16(entry + 2, 0xFFFF)
|
||||
direct.append({"field": "inline.e_value_size", "offset": entry + 2, "bytes": 2})
|
||||
elif name in {"shared-name-nul", "shared-value-size"}:
|
||||
_, _, entry = image.first_shared_entry("/shared-0.bin")
|
||||
if name == "shared-name-nul":
|
||||
offset = entry + 4 + image.data[entry] // 2
|
||||
image.data[offset] = 0
|
||||
direct.append({"field": "shared.e_name", "offset": offset, "bytes": 1})
|
||||
else:
|
||||
image.put_u16(entry + 2, 0xFFFF)
|
||||
direct.append({"field": "shared.e_value_size", "offset": entry + 2, "bytes": 2})
|
||||
elif name == "shared-id-offset":
|
||||
_, body, _ = image.first_shared_entry("/shared-0.bin")
|
||||
image.put_u32(body + 12, 0xFFFFFFFF)
|
||||
direct.append({"field": "ibody.h_shared_xattrs[0]", "offset": body + 12, "bytes": 4})
|
||||
elif name in {"long-prefix-id",}:
|
||||
_, entry = image.first_inline_entry("/prefix.bin")
|
||||
image.data[entry + 1] = 0xFF
|
||||
direct.append({"field": "inline.e_name_index", "offset": entry + 1, "bytes": 1})
|
||||
elif name in {"prefix-infix-nul", "prefix-base-index", "prefix-record-truncated"}:
|
||||
prefix = image.prefix_record_offset()
|
||||
if name == "prefix-infix-nul":
|
||||
offset = prefix + 4
|
||||
image.data[offset] = 0
|
||||
direct.append({"field": "prefix.infix", "offset": offset, "bytes": 1})
|
||||
elif name == "prefix-base-index":
|
||||
image.data[prefix + 2] = 0
|
||||
direct.append({"field": "prefix.base_index", "offset": prefix + 2, "bytes": 1})
|
||||
else:
|
||||
image.put_u16(prefix, 255)
|
||||
direct.append({"field": "prefix.record_length", "offset": prefix, "bytes": 2})
|
||||
elif name == "prefix-offset-bounds":
|
||||
offset = SUPER_OFFSET + 92
|
||||
image.put_u32(offset, 0x100)
|
||||
direct.append({"field": "super.xattr_prefix_start", "offset": offset, "bytes": 4})
|
||||
elif name in {"header-shared-count", "header-only", "ibody-bounds", "inline-entry-truncated"}:
|
||||
inode = image.resolve("/inline.bin")
|
||||
body = image.body_offset(inode)
|
||||
if name == "header-shared-count":
|
||||
image.data[body + 4] = (inode["xattr_size"] - 12) // 4 + 1
|
||||
direct.append({"field": "ibody.h_shared_count", "offset": body + 4, "bytes": 1})
|
||||
else:
|
||||
count = {"header-only": 1, "ibody-bounds": 0xFFFF, "inline-entry-truncated": 3}[name]
|
||||
image.put_u16(inode["offset"] + 2, count)
|
||||
direct.append({"field": "inode.i_xattr_icount", "offset": inode["offset"] + 2, "bytes": 2})
|
||||
else:
|
||||
raise ValueError(f"unknown B17 mutation: {name}")
|
||||
return direct
|
||||
|
||||
|
||||
def validate_seed_shape(image: Image) -> None:
|
||||
feature = image.u32(SUPER_OFFSET + 8)
|
||||
if not feature & FEATURE_COMPAT_XATTR_FILTER:
|
||||
raise ValueError("seed does not declare the xattr filter feature")
|
||||
if image.data[SUPER_OFFSET + 104] != 0:
|
||||
raise ValueError("seed xattr filter reserved byte is not zero")
|
||||
_, inline = image.first_inline_entry("/inline.bin")
|
||||
if bytes(image.data[inline + 4 : inline + 4 + image.data[inline]]) != b"alpha":
|
||||
raise ValueError("seed inline xattr changed")
|
||||
shared_inode, shared_body, shared = image.first_shared_entry("/shared-0.bin")
|
||||
if image.data[shared_body + 4] != 1 or shared_inode["xattr_size"] < 16:
|
||||
raise ValueError("seed shared xattr header changed")
|
||||
if bytes(image.data[shared + 4 : shared + 4 + image.data[shared]]) != b"shared":
|
||||
raise ValueError("seed shared xattr entry changed")
|
||||
_, long_entry = image.first_inline_entry("/prefix.bin")
|
||||
if not image.data[long_entry + 1] & 0x80:
|
||||
raise ValueError("seed long-prefix entry changed")
|
||||
prefix = image.prefix_record_offset()
|
||||
length = image.u16(prefix)
|
||||
if image.data[prefix + 2] != 1 or bytes(image.data[prefix + 3 : prefix + 2 + length]) != b"company.":
|
||||
raise ValueError("seed long-prefix record changed")
|
||||
_, acl = image.first_inline_entry("/acl.bin")
|
||||
if image.data[acl] != 4 or image.u16(acl + 2) != 24:
|
||||
raise ValueError("seed ACL staging entry changed")
|
||||
|
||||
|
||||
def command_rebuild(spec_path: Path, seed_path: Path, output: Path, work: Path) -> int:
|
||||
spec = load_spec(spec_path)
|
||||
if output.exists() or work.exists():
|
||||
raise SystemExit("rebuild output and work paths must not exist")
|
||||
work.mkdir(parents=True)
|
||||
details = build_seed(spec, output, work)
|
||||
actual = sha256(output)
|
||||
expected = spec["seed"]["sha256"]
|
||||
tracked = sha256(seed_path)
|
||||
if actual != expected or tracked != expected:
|
||||
raise SystemExit(
|
||||
f"B17 seed reproducibility failed: generated={actual} tracked={tracked} expected={expected}"
|
||||
)
|
||||
validate_seed_shape(Image.load(output))
|
||||
print(json.dumps({"status": "PASS", "seed_sha256": actual, **details}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def command_create_seed(spec_path: Path, output: Path, work: Path) -> int:
|
||||
spec = load_spec(spec_path)
|
||||
if output.exists() or work.exists():
|
||||
raise SystemExit("seed output and work paths must not exist")
|
||||
work.mkdir(parents=True)
|
||||
details = build_seed(spec, output, work)
|
||||
validate_seed_shape(Image.load(output))
|
||||
actual = sha256(output)
|
||||
print(json.dumps({"status": "PASS", "seed_sha256": actual, **details}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def command_generate(spec_path: Path, seed_path: Path, output: Path) -> int:
|
||||
spec = load_spec(spec_path)
|
||||
if output.exists():
|
||||
raise SystemExit(f"refusing to overwrite fixture output: {output}")
|
||||
expected_seed = spec["seed"]["sha256"]
|
||||
actual_seed = sha256(seed_path)
|
||||
if actual_seed != expected_seed:
|
||||
raise SystemExit(
|
||||
f"B17 seed hash differs: expected {expected_seed}, got {actual_seed}"
|
||||
)
|
||||
seed = Image.load(seed_path)
|
||||
validate_seed_shape(seed)
|
||||
output.mkdir(parents=True)
|
||||
records = []
|
||||
aggregate = hashlib.sha256()
|
||||
for case in spec["cases"]:
|
||||
image = Image(seed.data)
|
||||
before = bytes(image.data)
|
||||
direct = apply_mutation(image, case["mutation"])
|
||||
image.update_checksum()
|
||||
destination = output / f"{case['id']}.erofs"
|
||||
destination.write_bytes(image.data)
|
||||
changed = [index for index, pair in enumerate(zip(before, image.data)) if pair[0] != pair[1]]
|
||||
record = {
|
||||
"id": case["id"],
|
||||
"class": case["class"],
|
||||
"mutation": case["mutation"],
|
||||
"path": destination.name,
|
||||
"bytes": destination.stat().st_size,
|
||||
"sha256": sha256(destination),
|
||||
"direct_mutations": direct,
|
||||
"changed_byte_count_with_checksum": len(changed),
|
||||
}
|
||||
records.append(record)
|
||||
aggregate.update(case["id"].encode("ascii"))
|
||||
aggregate.update(b"\0")
|
||||
aggregate.update(record["sha256"].encode("ascii"))
|
||||
aggregate.update(b"\n")
|
||||
manifest = {
|
||||
"schema": 1,
|
||||
"batch": "B17",
|
||||
"seed_sha256": actual_seed,
|
||||
"fixture_count": len(records),
|
||||
"legal_count": sum(record["class"] == "legal" for record in records),
|
||||
"damaged_count": sum(record["class"] == "damaged" for record in records),
|
||||
"fixture_set_sha256": aggregate.hexdigest(),
|
||||
"fixtures": records,
|
||||
}
|
||||
write_json(output / "fixture-index.json", manifest)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"fixture_count": manifest["fixture_count"],
|
||||
"fixture_set_sha256": manifest["fixture_set_sha256"],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
create = subparsers.add_parser("create-seed")
|
||||
create.add_argument("--spec", type=Path, required=True)
|
||||
create.add_argument("--output", type=Path, required=True)
|
||||
create.add_argument("--work", type=Path, required=True)
|
||||
rebuild = subparsers.add_parser("rebuild-seed")
|
||||
rebuild.add_argument("--spec", type=Path, required=True)
|
||||
rebuild.add_argument("--seed", type=Path, required=True)
|
||||
rebuild.add_argument("--output", type=Path, required=True)
|
||||
rebuild.add_argument("--work", type=Path, required=True)
|
||||
generate = subparsers.add_parser("generate")
|
||||
generate.add_argument("--spec", type=Path, required=True)
|
||||
generate.add_argument("--seed", type=Path, required=True)
|
||||
generate.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.command == "create-seed":
|
||||
return command_create_seed(args.spec, args.output, args.work)
|
||||
if args.command == "rebuild-seed":
|
||||
return command_rebuild(args.spec, args.seed, args.output, args.work)
|
||||
return command_generate(args.spec, args.seed, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+482
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently parse B17 images and compare frozen FreeBSD expectations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER_OFFSET = 1024
|
||||
FEATURE_COMPAT_XATTR_FILTER = 0x00000004
|
||||
FEATURE_COMPAT_PLAIN_XATTR_PFX = 0x00000010
|
||||
FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040
|
||||
ACL_FILTER_BITS = (1 << 21) | (1 << 30)
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
class Reject(Exception):
|
||||
def __init__(self, errno_name: str, point: str):
|
||||
super().__init__(f"{errno_name} at {point}")
|
||||
self.errno_name = errno_name
|
||||
self.point = point
|
||||
|
||||
|
||||
def crc32c(data: bytes, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.data = path.read_bytes()
|
||||
if len(self.data) < SUPER_OFFSET + 144:
|
||||
raise Reject("EINTEGRITY", "super.bounds")
|
||||
if self.u32(SUPER_OFFSET) != 0xE0F5E1E2:
|
||||
raise Reject("EINTEGRITY", "super.magic")
|
||||
self.block_bits = self.data[SUPER_OFFSET + 12]
|
||||
if self.block_bits < 9 or self.block_bits > 16:
|
||||
raise Reject("EINTEGRITY", "super.block-size")
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.blocks = self.u32(SUPER_OFFSET + 36)
|
||||
self.limit = self.blocks << self.block_bits
|
||||
if self.limit > len(self.data):
|
||||
raise Reject("EINTEGRITY", "super.image-bounds")
|
||||
self.feature_compat = self.u32(SUPER_OFFSET + 8)
|
||||
self.feature_incompat = self.u32(SUPER_OFFSET + 80)
|
||||
self.filter_reserved = self.data[SUPER_OFFSET + 104]
|
||||
self.meta_blkaddr = self.u32(SUPER_OFFSET + 40)
|
||||
self.xattr_blkaddr = self.u32(SUPER_OFFSET + 44)
|
||||
self.root_nid = self.u16(SUPER_OFFSET + 14)
|
||||
self.packed_nid = self.u64(SUPER_OFFSET + 96)
|
||||
self.prefix_count = self.data[SUPER_OFFSET + 91]
|
||||
self.prefix_start = self.u32(SUPER_OFFSET + 92)
|
||||
self.verify_checksum()
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 2 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u16-bounds")
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 4 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u32-bounds")
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 8 > len(self.data):
|
||||
raise Reject("EINTEGRITY", "raw.u64-bounds")
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def verify_checksum(self) -> None:
|
||||
if not self.feature_compat & 1:
|
||||
return
|
||||
end = self.block_size
|
||||
if end > len(self.data):
|
||||
raise Reject("EINTEGRITY", "super.checksum-bounds")
|
||||
expected = self.u32(SUPER_OFFSET + 4)
|
||||
block = bytearray(self.data[SUPER_OFFSET:end])
|
||||
block[4:8] = bytes(4)
|
||||
if crc32c(bytes(block)) != expected:
|
||||
raise Reject("EINTEGRITY", "super.checksum")
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
if offset + 32 > self.limit:
|
||||
raise Reject("EINTEGRITY", "inode.bounds")
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
if offset + inode_size > self.limit:
|
||||
raise Reject("EINTEGRITY", "inode.bounds")
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_count": xattr_count,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_range(self, inode: dict[str, int], logical: int, length: int, point: str) -> bytes:
|
||||
if logical > inode["size"] or length > inode["size"] - logical:
|
||||
raise Reject("EINTEGRITY", point)
|
||||
if inode["layout"] == 2:
|
||||
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
|
||||
elif inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
raise Reject("EOPNOTSUPP", "inode.layout")
|
||||
if physical > self.limit or length > self.limit - physical:
|
||||
raise Reject("EINTEGRITY", point)
|
||||
return self.data[physical : physical + length]
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
data = self.inode_range(inode, 0, inode["size"], "directory.bounds")
|
||||
if len(data) < 12:
|
||||
raise Reject("EINTEGRITY", "directory.header")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name == 0 or first_name % 12 != 0 or first_name > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-offset")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
name_start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
name_end = (
|
||||
struct.unpack_from("<H", data, entry + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(data)
|
||||
)
|
||||
if name_start > name_end or name_end > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-bounds")
|
||||
name = data[name_start:name_end].split(b"\0", 1)[0]
|
||||
entries.append((name, nid))
|
||||
return entries
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
nid = next(
|
||||
(candidate for name, candidate in self.directory_entries(inode)
|
||||
if name == component),
|
||||
None,
|
||||
)
|
||||
if nid is None:
|
||||
raise Reject("ENOATTR", "path.lookup")
|
||||
inode = self.inode(nid)
|
||||
return inode
|
||||
|
||||
def load_prefixes(self) -> list[tuple[int, bytes]]:
|
||||
if not self.feature_incompat & FEATURE_INCOMPAT_XATTR_PREFIXES or self.prefix_count == 0:
|
||||
return []
|
||||
if self.feature_compat & FEATURE_COMPAT_PLAIN_XATTR_PFX:
|
||||
raise Reject("EOPNOTSUPP", "prefix.plain-unexpected")
|
||||
if self.packed_nid == 0:
|
||||
raise Reject("EINTEGRITY", "prefix.carrier")
|
||||
packed = self.inode(self.packed_nid)
|
||||
logical = self.prefix_start << 2
|
||||
prefixes = []
|
||||
for _ in range(self.prefix_count):
|
||||
while logical % 4:
|
||||
logical += 1
|
||||
header = self.inode_range(packed, logical, 2, "prefix.header-bounds")
|
||||
length = struct.unpack("<H", header)[0]
|
||||
if length < 1 or length > 256:
|
||||
raise Reject("EINTEGRITY", "prefix.length")
|
||||
payload = self.inode_range(
|
||||
packed, logical + 2, length, "prefix.payload-bounds"
|
||||
)
|
||||
base_index = payload[0]
|
||||
infix = payload[1:]
|
||||
if b"\0" in infix:
|
||||
raise Reject("EINTEGRITY", "prefix.infix-nul")
|
||||
prefixes.append((base_index, infix))
|
||||
logical += 2 + length
|
||||
return prefixes
|
||||
|
||||
def load_body(self, inode: dict[str, int]) -> tuple[int, int, list[int]]:
|
||||
size = inode["xattr_size"]
|
||||
if size == 0:
|
||||
return 0, 0, []
|
||||
body = inode["offset"] + inode["inode_size"]
|
||||
if body > self.limit or size > self.limit - body:
|
||||
raise Reject("EINTEGRITY", "ibody.bounds")
|
||||
if size < 12:
|
||||
raise Reject("EINTEGRITY", "ibody.header-bounds")
|
||||
if size == 12:
|
||||
raise Reject("EOPNOTSUPP", "ibody.header-only")
|
||||
shared_count = self.data[body + 4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > size:
|
||||
raise Reject("EINTEGRITY", "ibody.shared-count")
|
||||
shared = [self.u32(body + 12 + index * 4) for index in range(shared_count)]
|
||||
return body, header_size, shared
|
||||
|
||||
def entry(self, offset: int, limit: int, kind: str) -> dict[str, object]:
|
||||
if offset > limit or 4 > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{kind}.entry-header")
|
||||
name_length = self.data[offset]
|
||||
name_index = self.data[offset + 1]
|
||||
value_length = self.u16(offset + 2)
|
||||
name_end = offset + 4 + name_length
|
||||
if name_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.name-bounds")
|
||||
value_end = name_end + value_length
|
||||
if value_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.value-bounds")
|
||||
aligned_end = (value_end + 3) & ~3
|
||||
if aligned_end > limit:
|
||||
raise Reject("EINTEGRITY", f"{kind}.padding-bounds")
|
||||
name = self.data[offset + 4 : name_end]
|
||||
if b"\0" in name:
|
||||
raise Reject("EINTEGRITY", f"{kind}.name-nul")
|
||||
return {
|
||||
"offset": offset,
|
||||
"next": aligned_end,
|
||||
"name_index": name_index,
|
||||
"name": name,
|
||||
"value": self.data[name_end:value_end],
|
||||
}
|
||||
|
||||
def entries(self, inode: dict[str, int]) -> tuple[list[dict[str, object]], int]:
|
||||
body, header_size, shared_ids = self.load_body(inode)
|
||||
if body == 0:
|
||||
return [], 0
|
||||
body_end = body + inode["xattr_size"]
|
||||
cursor = body + header_size
|
||||
entries = []
|
||||
while cursor < body_end:
|
||||
item = self.entry(cursor, body_end, "inline")
|
||||
entries.append(item)
|
||||
cursor = int(item["next"])
|
||||
for shared_id in shared_ids:
|
||||
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
|
||||
if offset > self.limit or 4 > self.limit - offset:
|
||||
raise Reject("EINTEGRITY", "shared.offset-bounds")
|
||||
entries.append(self.entry(offset, self.limit, "shared"))
|
||||
return entries, len(shared_ids)
|
||||
|
||||
def resolved_name(
|
||||
self, entry: dict[str, object], prefixes: list[tuple[int, bytes]]
|
||||
) -> tuple[str, bytes]:
|
||||
index = int(entry["name_index"])
|
||||
infix = b""
|
||||
from_prefix = False
|
||||
if index & 0x80:
|
||||
prefix_id = index & 0x7F
|
||||
if prefix_id >= len(prefixes):
|
||||
raise Reject("ENOATTR", "name.long-prefix-id")
|
||||
index, infix = prefixes[prefix_id]
|
||||
from_prefix = True
|
||||
mapping = {
|
||||
1: ("user", b""),
|
||||
2: ("system", b"posix_acl_access"),
|
||||
3: ("system", b"posix_acl_default"),
|
||||
4: ("system", b"trusted."),
|
||||
6: ("system", b"security."),
|
||||
}
|
||||
if index not in mapping:
|
||||
point = "name.prefix-base-index" if from_prefix else "name.short-index"
|
||||
raise Reject("ENOATTR", point)
|
||||
namespace, fixed = mapping[index]
|
||||
return namespace, fixed + infix + bytes(entry["name"])
|
||||
|
||||
def getxattr(self, inode: dict[str, int], namespace: str, name: bytes,
|
||||
prefixes: list[tuple[int, bytes]]) -> tuple[bytes, int]:
|
||||
entries, shared_count = self.entries(inode)
|
||||
deferred: Reject | None = None
|
||||
for entry in entries:
|
||||
try:
|
||||
actual_namespace, actual_name = self.resolved_name(entry, prefixes)
|
||||
except Reject as error:
|
||||
deferred = error
|
||||
continue
|
||||
if actual_namespace == namespace and actual_name == name:
|
||||
return bytes(entry["value"]), shared_count
|
||||
if deferred is not None:
|
||||
raise deferred
|
||||
raise Reject("ENOATTR", "name.not-found")
|
||||
|
||||
def listxattr(self, inode: dict[str, int], namespace: str,
|
||||
prefixes: list[tuple[int, bytes]]) -> tuple[list[bytes], int]:
|
||||
entries, shared_count = self.entries(inode)
|
||||
names = []
|
||||
deferred: Reject | None = None
|
||||
for entry in entries:
|
||||
try:
|
||||
actual_namespace, actual_name = self.resolved_name(entry, prefixes)
|
||||
except Reject as error:
|
||||
deferred = error
|
||||
continue
|
||||
if actual_namespace == namespace:
|
||||
names.append(actual_name)
|
||||
if deferred is not None and not names:
|
||||
raise deferred
|
||||
return names, shared_count
|
||||
|
||||
|
||||
def parse_acl(value: bytes) -> list[list[int]]:
|
||||
if len(value) < 4 or (len(value) - 4) % 8:
|
||||
raise Reject("EINTEGRITY", "acl.value-size")
|
||||
if struct.unpack_from("<I", value)[0] != 2:
|
||||
raise Reject("EINTEGRITY", "acl.version")
|
||||
entries = []
|
||||
for offset in range(4, len(value), 8):
|
||||
entries.append(list(struct.unpack_from("<HHI", value, offset)))
|
||||
return entries
|
||||
|
||||
|
||||
def execute(case: dict, image_path: Path) -> dict[str, object]:
|
||||
reader = Reader(image_path)
|
||||
prefixes = reader.load_prefixes()
|
||||
if "expected_feature_filter" in case:
|
||||
actual_feature = bool(reader.feature_compat & FEATURE_COMPAT_XATTR_FILTER)
|
||||
if actual_feature != case["expected_feature_filter"]:
|
||||
raise AssertionError("raw xattr filter feature differs")
|
||||
if reader.filter_reserved != case["expected_filter_reserved"]:
|
||||
raise AssertionError("raw xattr filter reserved byte differs")
|
||||
usable = actual_feature and reader.filter_reserved == 0
|
||||
if usable != case["expected_filter_usable"]:
|
||||
raise AssertionError("xattr filter use-site gate differs")
|
||||
operation = case["operation"]
|
||||
result: dict[str, object] = {}
|
||||
if operation == "mount":
|
||||
return result
|
||||
inode = reader.resolve(case["target"])
|
||||
if "expected_acl_filter_negative" in case:
|
||||
body, _, _ = reader.load_body(inode)
|
||||
actual_negative = (
|
||||
reader.u32(body) & ACL_FILTER_BITS
|
||||
) == ACL_FILTER_BITS
|
||||
if actual_negative != case["expected_acl_filter_negative"]:
|
||||
raise AssertionError("ACL name-filter declaration differs")
|
||||
if operation == "get":
|
||||
value, shared_count = reader.getxattr(
|
||||
inode, case["namespace"], case["name"].encode("ascii"), prefixes
|
||||
)
|
||||
result["value_hex"] = value.hex()
|
||||
result["shared_count"] = shared_count
|
||||
if "expected_value_hex" in case and value.hex() != case["expected_value_hex"]:
|
||||
raise AssertionError("xattr value differs")
|
||||
if "expected_shared_count" in case and shared_count != case["expected_shared_count"]:
|
||||
raise AssertionError("xattr shared count differs")
|
||||
elif operation == "list":
|
||||
names, shared_count = reader.listxattr(inode, case["namespace"], prefixes)
|
||||
result["names"] = [name.decode("ascii") for name in names]
|
||||
result["shared_count"] = shared_count
|
||||
elif operation == "acl":
|
||||
value, _ = reader.getxattr(
|
||||
inode, "system", b"posix_acl_access", prefixes
|
||||
)
|
||||
acl = parse_acl(value)
|
||||
result["acl"] = acl
|
||||
if acl != case["expected_acl"]:
|
||||
raise AssertionError("ACL empty-suffix value differs")
|
||||
else:
|
||||
raise AssertionError(f"unknown operation: {operation}")
|
||||
return result
|
||||
|
||||
|
||||
def fsck_accept(path: Path) -> dict[str, object]:
|
||||
fsck = shutil.which("fsck.erofs")
|
||||
if fsck is None:
|
||||
raise SystemExit("fsck.erofs is required for legal B17 fixture baseline")
|
||||
completed = subprocess.run(
|
||||
[fsck, "-d0", "--xattrs", str(path)], check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
return {"exit": completed.returncode, "stdout": completed.stdout}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--fixtures", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
index = json.loads(
|
||||
(args.fixtures / "fixture-index.json").read_text(encoding="ascii")
|
||||
)
|
||||
indexed = {item["id"]: item for item in index["fixtures"]}
|
||||
errnos = spec["freebsd_errno"]
|
||||
results = []
|
||||
failures = []
|
||||
for case in spec["cases"]:
|
||||
image_path = args.fixtures / indexed[case["id"]]["path"]
|
||||
actual_errno = 0
|
||||
actual_errno_name = None
|
||||
actual_reject = "accepted"
|
||||
details: dict[str, object] = {}
|
||||
try:
|
||||
details = execute(case, image_path)
|
||||
except Reject as error:
|
||||
actual_errno_name = error.errno_name
|
||||
actual_errno = errnos[error.errno_name]
|
||||
actual_reject = error.point
|
||||
except Exception as error:
|
||||
failures.append({"id": case["id"], "reason": repr(error)})
|
||||
results.append({"id": case["id"], "status": "FAIL", "exception": repr(error)})
|
||||
continue
|
||||
expected_name = case.get("expected_errno_name")
|
||||
matched = (
|
||||
actual_errno == case["expected_errno"]
|
||||
and actual_errno_name == expected_name
|
||||
and actual_reject == case["expected_reject"]
|
||||
)
|
||||
fsck = None
|
||||
if case.get("fsck_accept"):
|
||||
fsck = fsck_accept(image_path)
|
||||
matched = matched and fsck["exit"] == 0
|
||||
result = {
|
||||
"id": case["id"],
|
||||
"class": case["class"],
|
||||
"status": "PASS" if matched else "FAIL",
|
||||
"expected_errno": case["expected_errno"],
|
||||
"actual_errno": actual_errno,
|
||||
"actual_errno_name": actual_errno_name,
|
||||
"expected_reject": case["expected_reject"],
|
||||
"actual_reject": actual_reject,
|
||||
"details": details,
|
||||
}
|
||||
if fsck is not None:
|
||||
result["fsck_exit"] = fsck["exit"]
|
||||
result["fsck_stdout_sha256"] = hashlib.sha256(
|
||||
fsck["stdout"].encode("utf-8")
|
||||
).hexdigest()
|
||||
results.append(result)
|
||||
if not matched:
|
||||
failures.append(result)
|
||||
report = {
|
||||
"schema": 1,
|
||||
"batch": "B17",
|
||||
"status": "PASS" if not failures else "FAIL",
|
||||
"fixture_count": len(results),
|
||||
"legal_passed": sum(
|
||||
item["status"] == "PASS" and item["class"] == "legal"
|
||||
for item in results
|
||||
),
|
||||
"damaged_passed": sum(
|
||||
item["status"] == "PASS" and item["class"] == "damaged"
|
||||
for item in results
|
||||
),
|
||||
"failures": failures,
|
||||
"results": results,
|
||||
}
|
||||
args.report.write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": report["status"],
|
||||
"fixtures": report["fixture_count"],
|
||||
"legal_passed": report["legal_passed"],
|
||||
"damaged_passed": report["damaged_passed"],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Binary file not shown.
@@ -0,0 +1,324 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"batch": "B17",
|
||||
"seed": {
|
||||
"path": "B17-xattr-seed.erofs",
|
||||
"sha256": "8137e7966a9dc5fba85dc63750d065bd54f6d08dc25b13f1e8c4f0deea934041",
|
||||
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
|
||||
"uuid": "00000000-0000-0000-0000-000000000317"
|
||||
},
|
||||
"freebsd_errno": {
|
||||
"EINTEGRITY": 97,
|
||||
"ENOATTR": 87,
|
||||
"EOPNOTSUPP": 45
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"id": "valid-inline-binary",
|
||||
"class": "legal",
|
||||
"mutation": "none",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_value_hex": "76616c75650062696e617279ff",
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-shared-header-entry",
|
||||
"class": "legal",
|
||||
"mutation": "none",
|
||||
"operation": "get",
|
||||
"target": "/shared-0.bin",
|
||||
"namespace": "user",
|
||||
"name": "shared",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_value_hex": "7368617265642d76616c7565",
|
||||
"expected_shared_count": 1,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-long-prefix",
|
||||
"class": "legal",
|
||||
"mutation": "none",
|
||||
"operation": "get",
|
||||
"target": "/prefix.bin",
|
||||
"namespace": "user",
|
||||
"name": "company.branch.leaf",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_value_hex": "7072656669782d76616c7565",
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-acl-empty-suffix",
|
||||
"class": "legal",
|
||||
"mutation": "acl-empty-suffix",
|
||||
"operation": "acl",
|
||||
"target": "/acl.bin",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": true,
|
||||
"expected_filter_reserved": 0,
|
||||
"expected_filter_usable": true,
|
||||
"expected_acl_filter_negative": false,
|
||||
"expected_acl": [[1, 6, 4294967295], [4, 0, 4294967295], [32, 0, 4294967295]],
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-filter-reserved-zero",
|
||||
"class": "legal",
|
||||
"mutation": "none",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": true,
|
||||
"expected_filter_reserved": 0,
|
||||
"expected_filter_usable": true,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-filter-reserved-one",
|
||||
"class": "legal",
|
||||
"mutation": "filter-reserved-one",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": true,
|
||||
"expected_filter_reserved": 1,
|
||||
"expected_filter_usable": false,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-filter-reserved-255",
|
||||
"class": "legal",
|
||||
"mutation": "filter-reserved-255",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": true,
|
||||
"expected_filter_reserved": 255,
|
||||
"expected_filter_usable": false,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-filter-feature-off",
|
||||
"class": "legal",
|
||||
"mutation": "filter-feature-off",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": false,
|
||||
"expected_filter_reserved": 0,
|
||||
"expected_filter_usable": false,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-filter-feature-off-reserved-one",
|
||||
"class": "legal",
|
||||
"mutation": "filter-feature-off-reserved-one",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": false,
|
||||
"expected_filter_reserved": 1,
|
||||
"expected_filter_usable": false,
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "valid-unknown-filter-acl-present",
|
||||
"class": "legal",
|
||||
"mutation": "unknown-filter-acl-present",
|
||||
"operation": "acl",
|
||||
"target": "/acl.bin",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"expected_feature_filter": true,
|
||||
"expected_filter_reserved": 1,
|
||||
"expected_filter_usable": false,
|
||||
"expected_acl_filter_negative": true,
|
||||
"expected_acl": [[1, 6, 4294967295], [4, 0, 4294967295], [32, 0, 4294967295]],
|
||||
"fsck_accept": true
|
||||
},
|
||||
{
|
||||
"id": "invalid-inline-name-nul",
|
||||
"class": "damaged",
|
||||
"mutation": "inline-name-nul",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "inline.name-nul"
|
||||
},
|
||||
{
|
||||
"id": "invalid-shared-name-nul",
|
||||
"class": "damaged",
|
||||
"mutation": "shared-name-nul",
|
||||
"operation": "list",
|
||||
"target": "/shared-0.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "shared.name-nul"
|
||||
},
|
||||
{
|
||||
"id": "invalid-prefix-infix-nul",
|
||||
"class": "damaged",
|
||||
"mutation": "prefix-infix-nul",
|
||||
"operation": "mount",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "prefix.infix-nul"
|
||||
},
|
||||
{
|
||||
"id": "invalid-short-name-index",
|
||||
"class": "damaged",
|
||||
"mutation": "short-name-index",
|
||||
"operation": "get",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"name": "alpha",
|
||||
"expected_errno": 87,
|
||||
"expected_errno_name": "ENOATTR",
|
||||
"expected_reject": "name.short-index"
|
||||
},
|
||||
{
|
||||
"id": "invalid-long-prefix-id",
|
||||
"class": "damaged",
|
||||
"mutation": "long-prefix-id",
|
||||
"operation": "get",
|
||||
"target": "/prefix.bin",
|
||||
"namespace": "user",
|
||||
"name": "company.branch.leaf",
|
||||
"expected_errno": 87,
|
||||
"expected_errno_name": "ENOATTR",
|
||||
"expected_reject": "name.long-prefix-id"
|
||||
},
|
||||
{
|
||||
"id": "invalid-prefix-base-index",
|
||||
"class": "damaged",
|
||||
"mutation": "prefix-base-index",
|
||||
"operation": "get",
|
||||
"target": "/prefix.bin",
|
||||
"namespace": "user",
|
||||
"name": "company.branch.leaf",
|
||||
"expected_errno": 87,
|
||||
"expected_errno_name": "ENOATTR",
|
||||
"expected_reject": "name.prefix-base-index"
|
||||
},
|
||||
{
|
||||
"id": "invalid-inline-value-size",
|
||||
"class": "damaged",
|
||||
"mutation": "inline-value-size",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "inline.value-bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-shared-value-size",
|
||||
"class": "damaged",
|
||||
"mutation": "shared-value-size",
|
||||
"operation": "list",
|
||||
"target": "/shared-0.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "shared.value-bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-shared-id-offset",
|
||||
"class": "damaged",
|
||||
"mutation": "shared-id-offset",
|
||||
"operation": "list",
|
||||
"target": "/shared-0.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "shared.offset-bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-header-shared-count",
|
||||
"class": "damaged",
|
||||
"mutation": "header-shared-count",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "ibody.shared-count"
|
||||
},
|
||||
{
|
||||
"id": "invalid-header-only",
|
||||
"class": "damaged",
|
||||
"mutation": "header-only",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 45,
|
||||
"expected_errno_name": "EOPNOTSUPP",
|
||||
"expected_reject": "ibody.header-only"
|
||||
},
|
||||
{
|
||||
"id": "invalid-ibody-bounds",
|
||||
"class": "damaged",
|
||||
"mutation": "ibody-bounds",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "ibody.bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-inline-entry-truncated",
|
||||
"class": "damaged",
|
||||
"mutation": "inline-entry-truncated",
|
||||
"operation": "list",
|
||||
"target": "/inline.bin",
|
||||
"namespace": "user",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "inline.name-bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-prefix-record-truncated",
|
||||
"class": "damaged",
|
||||
"mutation": "prefix-record-truncated",
|
||||
"operation": "mount",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "prefix.payload-bounds"
|
||||
},
|
||||
{
|
||||
"id": "invalid-prefix-offset-bounds",
|
||||
"class": "damaged",
|
||||
"mutation": "prefix-offset-bounds",
|
||||
"operation": "mount",
|
||||
"expected_errno": 97,
|
||||
"expected_errno_name": "EINTEGRITY",
|
||||
"expected_reject": "prefix.header-bounds"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#define B19A_WORKERS 64
|
||||
#define B19A_BODY_SIZE 256
|
||||
#define B19A_ENTRY_LIMIT 65536
|
||||
#define B19A_MOUNT_BUDGET 1048576
|
||||
#define B19A_EINTEGRITY 97
|
||||
|
||||
enum cache_state {
|
||||
CACHE_EMPTY,
|
||||
CACHE_INFLIGHT,
|
||||
CACHE_READY,
|
||||
CACHE_FAILED,
|
||||
};
|
||||
|
||||
enum cache_lookup {
|
||||
CACHE_BYPASS,
|
||||
CACHE_HIT,
|
||||
CACHE_OWNER,
|
||||
CACHE_ERROR,
|
||||
};
|
||||
|
||||
struct mount_budget {
|
||||
pthread_mutex_t lock;
|
||||
size_t resident;
|
||||
};
|
||||
|
||||
struct cache {
|
||||
pthread_mutex_t lock;
|
||||
pthread_cond_t cv;
|
||||
struct mount_budget *budget;
|
||||
unsigned char *body;
|
||||
size_t body_size;
|
||||
size_t charged;
|
||||
unsigned int waiters;
|
||||
int error;
|
||||
enum cache_state state;
|
||||
bool closing;
|
||||
};
|
||||
|
||||
struct wave {
|
||||
struct cache *cache;
|
||||
pthread_barrier_t barrier;
|
||||
unsigned char payload[B19A_BODY_SIZE];
|
||||
atomic_uint loads;
|
||||
atomic_uint bypasses;
|
||||
int expected_error;
|
||||
int results[B19A_WORKERS];
|
||||
};
|
||||
|
||||
struct worker_arg {
|
||||
struct wave *wave;
|
||||
unsigned int index;
|
||||
};
|
||||
|
||||
struct close_arg {
|
||||
struct cache *cache;
|
||||
atomic_bool done;
|
||||
};
|
||||
|
||||
static void
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "B19a cache model failure: %s\n", message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void
|
||||
check_pthread(int error, const char *operation)
|
||||
{
|
||||
if (error != 0) {
|
||||
errno = error;
|
||||
perror(operation);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
budget_init(struct mount_budget *budget)
|
||||
{
|
||||
memset(budget, 0, sizeof(*budget));
|
||||
check_pthread(pthread_mutex_init(&budget->lock, NULL),
|
||||
"pthread_mutex_init budget");
|
||||
}
|
||||
|
||||
static bool
|
||||
budget_reserve(struct mount_budget *budget, size_t amount)
|
||||
{
|
||||
bool reserved;
|
||||
|
||||
check_pthread(pthread_mutex_lock(&budget->lock),
|
||||
"pthread_mutex_lock budget");
|
||||
reserved = amount <= B19A_MOUNT_BUDGET - budget->resident;
|
||||
if (reserved)
|
||||
budget->resident += amount;
|
||||
check_pthread(pthread_mutex_unlock(&budget->lock),
|
||||
"pthread_mutex_unlock budget");
|
||||
return (reserved);
|
||||
}
|
||||
|
||||
static void
|
||||
budget_release(struct mount_budget *budget, size_t amount)
|
||||
{
|
||||
check_pthread(pthread_mutex_lock(&budget->lock),
|
||||
"pthread_mutex_lock budget");
|
||||
if (amount > budget->resident)
|
||||
fail("mount budget underflow");
|
||||
budget->resident -= amount;
|
||||
check_pthread(pthread_mutex_unlock(&budget->lock),
|
||||
"pthread_mutex_unlock budget");
|
||||
}
|
||||
|
||||
static size_t
|
||||
budget_resident(struct mount_budget *budget)
|
||||
{
|
||||
size_t resident;
|
||||
|
||||
check_pthread(pthread_mutex_lock(&budget->lock),
|
||||
"pthread_mutex_lock budget");
|
||||
resident = budget->resident;
|
||||
check_pthread(pthread_mutex_unlock(&budget->lock),
|
||||
"pthread_mutex_unlock budget");
|
||||
return (resident);
|
||||
}
|
||||
|
||||
static void
|
||||
budget_destroy(struct mount_budget *budget)
|
||||
{
|
||||
if (budget_resident(budget) != 0)
|
||||
fail("mount budget remained charged at destroy");
|
||||
check_pthread(pthread_mutex_destroy(&budget->lock),
|
||||
"pthread_mutex_destroy budget");
|
||||
}
|
||||
|
||||
static void
|
||||
cache_init(struct cache *cache, struct mount_budget *budget)
|
||||
{
|
||||
memset(cache, 0, sizeof(*cache));
|
||||
cache->budget = budget;
|
||||
check_pthread(pthread_mutex_init(&cache->lock, NULL),
|
||||
"pthread_mutex_init cache");
|
||||
check_pthread(pthread_cond_init(&cache->cv, NULL),
|
||||
"pthread_cond_init cache");
|
||||
}
|
||||
|
||||
static enum cache_lookup
|
||||
cache_claim(struct cache *cache, size_t body_size, unsigned char *output,
|
||||
int *errorp)
|
||||
{
|
||||
enum cache_lookup result;
|
||||
|
||||
*errorp = 0;
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock cache");
|
||||
if (cache->closing) {
|
||||
*errorp = ENXIO;
|
||||
goto bypass;
|
||||
}
|
||||
if (cache->state == CACHE_READY) {
|
||||
if (cache->body == NULL || cache->body_size != body_size)
|
||||
fail("READY cache body is inconsistent");
|
||||
memcpy(output, cache->body, body_size);
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
return (CACHE_HIT);
|
||||
}
|
||||
if (cache->state == CACHE_INFLIGHT) {
|
||||
++cache->waiters;
|
||||
do {
|
||||
check_pthread(pthread_cond_wait(&cache->cv, &cache->lock),
|
||||
"pthread_cond_wait cache");
|
||||
} while (cache->state == CACHE_INFLIGHT);
|
||||
if (cache->state == CACHE_READY) {
|
||||
if (cache->body == NULL || cache->body_size != body_size)
|
||||
fail("published cache body is inconsistent");
|
||||
memcpy(output, cache->body, body_size);
|
||||
result = CACHE_HIT;
|
||||
} else {
|
||||
if (cache->state != CACHE_FAILED || cache->error <= 0)
|
||||
fail("waiter observed an untyped failure");
|
||||
*errorp = cache->error;
|
||||
result = CACHE_ERROR;
|
||||
}
|
||||
--cache->waiters;
|
||||
if (cache->state == CACHE_FAILED && cache->waiters == 0) {
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
if (cache->waiters == 0)
|
||||
check_pthread(pthread_cond_broadcast(&cache->cv),
|
||||
"pthread_cond_broadcast cache");
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
return (result);
|
||||
}
|
||||
if (cache->state == CACHE_FAILED) {
|
||||
if (cache->waiters != 0)
|
||||
goto bypass;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
if (body_size > B19A_ENTRY_LIMIT ||
|
||||
!budget_reserve(cache->budget, body_size))
|
||||
goto bypass;
|
||||
cache->body_size = body_size;
|
||||
cache->charged = body_size;
|
||||
cache->state = CACHE_INFLIGHT;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
return (CACHE_OWNER);
|
||||
|
||||
bypass:
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
return (CACHE_BYPASS);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_complete(struct cache *cache, unsigned char *body, size_t body_size,
|
||||
int error)
|
||||
{
|
||||
if ((error == 0) != (body != NULL))
|
||||
fail("owner completion is not typed");
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock cache");
|
||||
if (cache->state != CACHE_INFLIGHT || cache->body_size != body_size)
|
||||
fail("cache owner lost its inflight body");
|
||||
if (error == 0) {
|
||||
cache->body = body;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_READY;
|
||||
} else {
|
||||
if (error < 1)
|
||||
fail("owner published a non-positive errno");
|
||||
budget_release(cache->budget, cache->charged);
|
||||
cache->charged = 0;
|
||||
cache->body_size = 0;
|
||||
cache->error = error;
|
||||
cache->state = CACHE_FAILED;
|
||||
if (cache->waiters == 0) {
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
}
|
||||
check_pthread(pthread_cond_broadcast(&cache->cv),
|
||||
"pthread_cond_broadcast cache");
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
}
|
||||
|
||||
static unsigned char *
|
||||
body_copy(const unsigned char *payload, size_t body_size)
|
||||
{
|
||||
unsigned char *copy;
|
||||
|
||||
copy = malloc(body_size);
|
||||
if (copy == NULL)
|
||||
fail("body allocation failed");
|
||||
memcpy(copy, payload, body_size);
|
||||
return (copy);
|
||||
}
|
||||
|
||||
static void
|
||||
wait_for_waiters(struct cache *cache, unsigned int expected)
|
||||
{
|
||||
struct timespec delay = { .tv_sec = 0, .tv_nsec = 1000000 };
|
||||
unsigned int attempt, waiters;
|
||||
|
||||
for (attempt = 0; attempt < 5000; ++attempt) {
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock cache");
|
||||
waiters = cache->waiters;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
if (waiters == expected)
|
||||
return;
|
||||
nanosleep(&delay, NULL);
|
||||
}
|
||||
fail("workers did not register as same-vnode waiters");
|
||||
}
|
||||
|
||||
static void *
|
||||
wave_worker(void *opaque)
|
||||
{
|
||||
struct worker_arg *arg;
|
||||
struct wave *wave;
|
||||
unsigned char output[B19A_BODY_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int barrier_error, error;
|
||||
|
||||
arg = opaque;
|
||||
wave = arg->wave;
|
||||
barrier_error = pthread_barrier_wait(&wave->barrier);
|
||||
if (barrier_error != 0 && barrier_error != PTHREAD_BARRIER_SERIAL_THREAD)
|
||||
check_pthread(barrier_error, "pthread_barrier_wait");
|
||||
lookup = cache_claim(wave->cache, sizeof(output), output, &error);
|
||||
if (lookup == CACHE_OWNER) {
|
||||
atomic_fetch_add_explicit(&wave->loads, 1, memory_order_relaxed);
|
||||
wait_for_waiters(wave->cache, B19A_WORKERS - 1);
|
||||
if (wave->expected_error != 0) {
|
||||
cache_complete(wave->cache, NULL, sizeof(output),
|
||||
wave->expected_error);
|
||||
wave->results[arg->index] = wave->expected_error;
|
||||
} else {
|
||||
memcpy(output, wave->payload, sizeof(output));
|
||||
cache_complete(wave->cache,
|
||||
body_copy(wave->payload, sizeof(wave->payload)),
|
||||
sizeof(output), 0);
|
||||
wave->results[arg->index] =
|
||||
memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO;
|
||||
}
|
||||
} else if (lookup == CACHE_HIT) {
|
||||
wave->results[arg->index] =
|
||||
memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO;
|
||||
} else if (lookup == CACHE_ERROR) {
|
||||
wave->results[arg->index] = error;
|
||||
} else {
|
||||
atomic_fetch_add_explicit(&wave->bypasses, 1, memory_order_relaxed);
|
||||
wave->results[arg->index] = error != 0 ? error : EBUSY;
|
||||
}
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
run_wave(struct cache *cache, int expected_error)
|
||||
{
|
||||
struct wave wave;
|
||||
struct worker_arg args[B19A_WORKERS];
|
||||
pthread_t threads[B19A_WORKERS];
|
||||
unsigned int index;
|
||||
|
||||
memset(&wave, 0, sizeof(wave));
|
||||
wave.cache = cache;
|
||||
wave.expected_error = expected_error;
|
||||
for (index = 0; index < sizeof(wave.payload); ++index)
|
||||
wave.payload[index] = (unsigned char)(index ^ 0x5a);
|
||||
check_pthread(pthread_barrier_init(&wave.barrier, NULL, B19A_WORKERS),
|
||||
"pthread_barrier_init");
|
||||
for (index = 0; index < B19A_WORKERS; ++index) {
|
||||
args[index].wave = &wave;
|
||||
args[index].index = index;
|
||||
check_pthread(pthread_create(&threads[index], NULL, wave_worker,
|
||||
&args[index]), "pthread_create");
|
||||
}
|
||||
for (index = 0; index < B19A_WORKERS; ++index)
|
||||
check_pthread(pthread_join(threads[index], NULL), "pthread_join");
|
||||
check_pthread(pthread_barrier_destroy(&wave.barrier),
|
||||
"pthread_barrier_destroy");
|
||||
if (atomic_load_explicit(&wave.loads, memory_order_relaxed) != 1)
|
||||
fail("same-vnode wave did not have exactly one owner");
|
||||
if (atomic_load_explicit(&wave.bypasses, memory_order_relaxed) != 0)
|
||||
fail("same-vnode wave bypassed the cache");
|
||||
for (index = 0; index < B19A_WORKERS; ++index) {
|
||||
if (wave.results[index] != expected_error)
|
||||
fail("same-vnode waiter observed a different result");
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
cache_invalidate(struct cache *cache)
|
||||
{
|
||||
unsigned char *body;
|
||||
size_t charged;
|
||||
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock cache");
|
||||
while (cache->state == CACHE_INFLIGHT || cache->waiters != 0)
|
||||
check_pthread(pthread_cond_wait(&cache->cv, &cache->lock),
|
||||
"pthread_cond_wait cache");
|
||||
body = cache->body;
|
||||
charged = cache->charged;
|
||||
cache->body = NULL;
|
||||
cache->body_size = 0;
|
||||
cache->charged = 0;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
free(body);
|
||||
if (charged != 0)
|
||||
budget_release(cache->budget, charged);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_close(struct cache *cache)
|
||||
{
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock cache");
|
||||
cache->closing = true;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock cache");
|
||||
cache_invalidate(cache);
|
||||
check_pthread(pthread_cond_destroy(&cache->cv),
|
||||
"pthread_cond_destroy cache");
|
||||
check_pthread(pthread_mutex_destroy(&cache->lock),
|
||||
"pthread_mutex_destroy cache");
|
||||
}
|
||||
|
||||
static void *
|
||||
close_worker(void *opaque)
|
||||
{
|
||||
struct close_arg *arg;
|
||||
|
||||
arg = opaque;
|
||||
cache_close(arg->cache);
|
||||
atomic_store_explicit(&arg->done, true, memory_order_release);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
test_hits_and_failure(struct mount_budget *budget)
|
||||
{
|
||||
struct cache cache;
|
||||
unsigned char output[B19A_BODY_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
unsigned int index;
|
||||
int error;
|
||||
|
||||
cache_init(&cache, budget);
|
||||
run_wave(&cache, 0);
|
||||
if (budget_resident(budget) != B19A_BODY_SIZE)
|
||||
fail("success wave did not charge one body");
|
||||
for (index = 0; index < 100; ++index) {
|
||||
lookup = cache_claim(&cache, sizeof(output), output, &error);
|
||||
if (lookup != CACHE_HIT || error != 0)
|
||||
fail("ready cache did not hit");
|
||||
}
|
||||
cache_invalidate(&cache);
|
||||
if (budget_resident(budget) != 0 || cache.state != CACHE_EMPTY)
|
||||
fail("cache invalidation did not release its body");
|
||||
run_wave(&cache, B19A_EINTEGRITY);
|
||||
if (budget_resident(budget) != 0 || cache.state != CACHE_EMPTY)
|
||||
fail("failed publication leaked body state");
|
||||
run_wave(&cache, 0);
|
||||
cache_close(&cache);
|
||||
if (budget_resident(budget) != 0)
|
||||
fail("cache close did not release the retry body");
|
||||
}
|
||||
|
||||
static void
|
||||
test_limits(struct mount_budget *budget)
|
||||
{
|
||||
struct cache caches[17];
|
||||
unsigned char *body, output[B19A_ENTRY_LIMIT + 1];
|
||||
enum cache_lookup lookup;
|
||||
unsigned int index;
|
||||
int error;
|
||||
|
||||
cache_init(&caches[0], budget);
|
||||
lookup = cache_claim(&caches[0], B19A_ENTRY_LIMIT + 1, output, &error);
|
||||
if (lookup != CACHE_BYPASS || budget_resident(budget) != 0)
|
||||
fail("oversized body did not bypass without charge");
|
||||
cache_close(&caches[0]);
|
||||
|
||||
for (index = 0; index < 17; ++index)
|
||||
cache_init(&caches[index], budget);
|
||||
for (index = 0; index < 16; ++index) {
|
||||
lookup = cache_claim(&caches[index], B19A_ENTRY_LIMIT, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("mount budget rejected an in-budget body");
|
||||
body = calloc(1, B19A_ENTRY_LIMIT);
|
||||
if (body == NULL)
|
||||
fail("budget body allocation failed");
|
||||
cache_complete(&caches[index], body, B19A_ENTRY_LIMIT, 0);
|
||||
}
|
||||
if (budget_resident(budget) != B19A_MOUNT_BUDGET)
|
||||
fail("mount budget did not reach its exact hard limit");
|
||||
lookup = cache_claim(&caches[16], B19A_ENTRY_LIMIT, output, &error);
|
||||
if (lookup != CACHE_BYPASS || budget_resident(budget) != B19A_MOUNT_BUDGET)
|
||||
fail("mount budget exhaustion did not bypass");
|
||||
for (index = 0; index < 17; ++index)
|
||||
cache_close(&caches[index]);
|
||||
if (budget_resident(budget) != 0)
|
||||
fail("mount budget did not drain after reclaim");
|
||||
}
|
||||
|
||||
static void
|
||||
test_close_inflight(struct mount_budget *budget)
|
||||
{
|
||||
struct cache cache;
|
||||
struct close_arg close_arg;
|
||||
pthread_t closer;
|
||||
unsigned char output[B19A_BODY_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
struct timespec delay = { .tv_sec = 0, .tv_nsec = 10000000 };
|
||||
int error;
|
||||
|
||||
cache_init(&cache, budget);
|
||||
lookup = cache_claim(&cache, sizeof(output), output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("close test did not establish an inflight owner");
|
||||
memset(&close_arg, 0, sizeof(close_arg));
|
||||
close_arg.cache = &cache;
|
||||
check_pthread(pthread_create(&closer, NULL, close_worker, &close_arg),
|
||||
"pthread_create close");
|
||||
nanosleep(&delay, NULL);
|
||||
if (atomic_load_explicit(&close_arg.done, memory_order_acquire))
|
||||
fail("cache close did not wait for inflight owner");
|
||||
cache_complete(&cache, body_copy(output, sizeof(output)), sizeof(output), 0);
|
||||
check_pthread(pthread_join(closer, NULL), "pthread_join close");
|
||||
if (!atomic_load_explicit(&close_arg.done, memory_order_acquire) ||
|
||||
budget_resident(budget) != 0)
|
||||
fail("cache close did not drain inflight completion");
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
struct mount_budget budget;
|
||||
|
||||
budget_init(&budget);
|
||||
test_hits_and_failure(&budget);
|
||||
test_limits(&budget);
|
||||
test_close_inflight(&budget);
|
||||
budget_destroy(&budget);
|
||||
printf("{\"body_limit\":%u,\"failure_publication\":\"PASS\","
|
||||
"\"inflight_close\":\"PASS\",\"mount_budget\":%u,"
|
||||
"\"owner_waiter\":\"PASS\",\"reclaim\":\"PASS\","
|
||||
"\"resident_after\":0,\"status\":\"PASS\"}\n",
|
||||
B19A_ENTRY_LIMIT, B19A_MOUNT_BUDGET);
|
||||
return (0);
|
||||
}
|
||||
Executable
+418
@@ -0,0 +1,418 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic real EROFS fixtures for B19a."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def deterministic_value(label: str, length: int) -> bytes:
|
||||
seed = (label + "|").encode("ascii")
|
||||
return (seed * (length // len(seed) + 1))[:length]
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B19a"
|
||||
or spec.get("candidate") != "P15-022"
|
||||
):
|
||||
raise SystemExit("invalid B19a fixture spec identity")
|
||||
return spec
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
||||
raise ValueError("invalid EROFS image")
|
||||
self.block_bits = self.data[SUPER + 12]
|
||||
if self.block_bits < 9 or self.block_bits > 16:
|
||||
raise ValueError("invalid EROFS block size")
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.blocks = self.u32(SUPER + 36)
|
||||
self.limit = self.blocks << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER + 40)
|
||||
self.xattr_blkaddr = self.u32(SUPER + 44)
|
||||
self.root_nid = self.u16(SUPER + 14)
|
||||
if self.limit > len(self.data):
|
||||
raise ValueError("EROFS image is truncated")
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 2 > len(self.data):
|
||||
raise ValueError("u16 read is out of bounds")
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
if offset < 0 or offset + 4 > len(self.data):
|
||||
raise ValueError("u32 read is out of bounds")
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
if offset > self.limit or 64 > self.limit - offset:
|
||||
raise ValueError("inode is out of bounds")
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = struct.unpack_from(
|
||||
"<Q" if inode_size == 64 else "<I", self.data, offset + 8
|
||||
)[0]
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
||||
if logical > inode["size"] or length > inode["size"] - logical:
|
||||
raise ValueError("inode data range is out of bounds")
|
||||
if inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
elif inode["layout"] == 2:
|
||||
tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size
|
||||
if logical < tail_start:
|
||||
if logical + length > tail_start:
|
||||
raise ValueError("inode data read crosses inline tail")
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
physical = (
|
||||
inode["offset"]
|
||||
+ inode["inode_size"]
|
||||
+ inode["xattr_size"]
|
||||
+ logical
|
||||
- tail_start
|
||||
)
|
||||
else:
|
||||
raise ValueError("unsupported directory layout")
|
||||
if physical > self.limit or length > self.limit - physical:
|
||||
raise ValueError("inode data is outside the image")
|
||||
return bytes(self.data[physical : physical + length])
|
||||
|
||||
@staticmethod
|
||||
def parse_dirblock(data: bytes) -> list[tuple[bytes, int]]:
|
||||
if len(data) < 12:
|
||||
raise ValueError("directory block is short")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name < 12 or first_name >= len(data) or first_name % 12:
|
||||
raise ValueError("directory first name offset is invalid")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
previous_offset = 0
|
||||
previous_name: bytes | None = None
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
end = (
|
||||
struct.unpack_from("<H", data, entry + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(data)
|
||||
)
|
||||
if (
|
||||
start < first_name
|
||||
or start >= len(data)
|
||||
or (index == 0 and start != first_name)
|
||||
or (index != 0 and start <= previous_offset)
|
||||
or end <= start
|
||||
or end > len(data)
|
||||
):
|
||||
raise ValueError("directory name bounds are invalid")
|
||||
span = data[start:end]
|
||||
if index + 1 < count:
|
||||
if b"\0" in span:
|
||||
raise ValueError("non-trailing directory name contains NUL")
|
||||
name = span
|
||||
else:
|
||||
name = span.split(b"\0", 1)[0]
|
||||
if not name or len(name) > 255 or b"/" in name:
|
||||
raise ValueError("directory name is invalid")
|
||||
if previous_name is not None and previous_name >= name:
|
||||
raise ValueError("directory names are not strictly ordered")
|
||||
entries.append((name, nid))
|
||||
previous_offset = start
|
||||
previous_name = name
|
||||
return entries
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
result = []
|
||||
previous_name: bytes | None = None
|
||||
logical = 0
|
||||
while logical < inode["size"]:
|
||||
length = min(self.block_size, inode["size"] - logical)
|
||||
block = self.parse_dirblock(self.inode_data(inode, logical, length))
|
||||
if previous_name is not None and previous_name >= block[0][0]:
|
||||
raise ValueError("directory block boundary is not ordered")
|
||||
result.extend(block)
|
||||
previous_name = block[-1][0]
|
||||
logical += length
|
||||
return result
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"fixture path is absent or ambiguous: {path}")
|
||||
inode = self.inode(matches[0])
|
||||
return inode
|
||||
|
||||
def body_offset(self, inode: dict[str, int]) -> int:
|
||||
return inode["offset"] + inode["inode_size"]
|
||||
|
||||
def body_shape(self, inode: dict[str, int]) -> tuple[int, list[int], list[int]]:
|
||||
body = self.body_offset(inode)
|
||||
size = inode["xattr_size"]
|
||||
if size <= 12 or body > self.limit or size > self.limit - body:
|
||||
raise ValueError("target xattr body is invalid")
|
||||
shared_count = self.data[body + 4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > size:
|
||||
raise ValueError("target shared count is invalid")
|
||||
shared_ids = [self.u32(body + 12 + index * 4) for index in range(shared_count)]
|
||||
inline_entries = []
|
||||
cursor = body + header_size
|
||||
end = body + size
|
||||
while cursor < end:
|
||||
if cursor + 4 > end:
|
||||
raise ValueError("target inline xattr header is truncated")
|
||||
name_length = self.data[cursor]
|
||||
value_length = self.u16(cursor + 2)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
if total > end - cursor:
|
||||
raise ValueError("target inline xattr entry is truncated")
|
||||
inline_entries.append(cursor)
|
||||
cursor += total
|
||||
return header_size, shared_ids, inline_entries
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
if self.u32(SUPER + 8) & 1:
|
||||
struct.pack_into("<I", self.data, SUPER + 4, 0)
|
||||
struct.pack_into(
|
||||
"<I", self.data, SUPER + 4, crc32c(self.data[SUPER : self.block_size])
|
||||
)
|
||||
|
||||
def write(self, path: Path) -> None:
|
||||
path.write_bytes(self.data)
|
||||
|
||||
|
||||
def set_epoch(path: Path) -> None:
|
||||
for entry in sorted(path.rglob("*"), reverse=True):
|
||||
os.utime(entry, (0, 0), follow_symlinks=False)
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def create_source(path: Path, spec: dict) -> None:
|
||||
path.mkdir(parents=True)
|
||||
path.chmod(0o755)
|
||||
shared_value = deterministic_value(
|
||||
"shared-value-p15-022", spec["fixture"]["shared_value_bytes"]
|
||||
)
|
||||
inline_value = deterministic_value(
|
||||
"inline-value-p15-022", spec["fixture"]["inline_value_bytes"]
|
||||
)
|
||||
files = []
|
||||
for index in range(spec["fixture"]["peer_count"]):
|
||||
prefix = f"peer-{index:03d}-"
|
||||
suffix = "x" * (spec["fixture"]["peer_name_bytes"] - len(prefix) - 4)
|
||||
files.append(path / f"{prefix}{suffix}.bin")
|
||||
target = path / "target.bin"
|
||||
files.append(target)
|
||||
for index, entry in enumerate(files):
|
||||
entry.write_bytes(f"P15-022 file {index:03d}\n".encode("ascii"))
|
||||
entry.chmod(0o644)
|
||||
os.setxattr(entry, b"user.shared", shared_value)
|
||||
if entry == target:
|
||||
os.setxattr(entry, b"user.inline", inline_value)
|
||||
else:
|
||||
os.setxattr(
|
||||
entry,
|
||||
b"user.peer",
|
||||
deterministic_value(f"peer-{index:03d}", 31),
|
||||
)
|
||||
|
||||
budget = path / "budget"
|
||||
budget.mkdir()
|
||||
budget.chmod(0o755)
|
||||
for index in range(spec["fixture"]["budget_file_count"]):
|
||||
entry = budget / f"budget-{index:03d}.bin"
|
||||
entry.write_bytes(b"B19a budget\n")
|
||||
entry.chmod(0o644)
|
||||
value = bytearray(
|
||||
deterministic_value(
|
||||
f"budget-value-{index:03d}",
|
||||
spec["fixture"]["budget_value_bytes"],
|
||||
)
|
||||
)
|
||||
struct.pack_into("<I", value, 0, index)
|
||||
os.setxattr(entry, b"user.budget", bytes(value))
|
||||
set_epoch(path)
|
||||
|
||||
|
||||
def build_image(source: Path, output: Path, spec: dict) -> list[str]:
|
||||
mkfs = shutil.which("mkfs.erofs")
|
||||
if mkfs is None:
|
||||
raise SystemExit("mkfs.erofs is required")
|
||||
command = [
|
||||
mkfs,
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"--sort=path",
|
||||
f"-U{spec['fixture']['uuid']}",
|
||||
"-x2",
|
||||
"-Eforce-inode-extended",
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
"mkfs.erofs failed: " + completed.stdout.decode("utf-8", "replace")
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
|
||||
image = Image.load(valid)
|
||||
target = image.resolve(spec["fixture"]["target"])
|
||||
body = image.body_offset(target)
|
||||
_, shared_ids, inline_entries = image.body_shape(target)
|
||||
if mutation == "corrupt-shared-count":
|
||||
image.data[body + 4] = 255
|
||||
elif mutation == "corrupt-shared-id":
|
||||
if not shared_ids:
|
||||
raise SystemExit("valid fixture has no shared xattr ID")
|
||||
struct.pack_into("<I", image.data, body + 12, 0xFFFFFFFF)
|
||||
elif mutation == "corrupt-inline-name":
|
||||
if not inline_entries or image.data[inline_entries[0]] == 0:
|
||||
raise SystemExit("valid fixture has no inline xattr entry")
|
||||
image.data[inline_entries[0] + 4] = 0
|
||||
else:
|
||||
raise SystemExit(f"unknown B19a mutation: {mutation}")
|
||||
image.update_checksum()
|
||||
image.write(output)
|
||||
|
||||
|
||||
def generate(output: Path, work: Path, spec: dict) -> dict:
|
||||
if output.exists() or work.exists():
|
||||
raise SystemExit("B19a output and work paths must be absent")
|
||||
output.mkdir(parents=True)
|
||||
source = work / "source"
|
||||
create_source(source, spec)
|
||||
valid = output / "valid.erofs"
|
||||
command = build_image(source, valid, spec)
|
||||
image = Image.load(valid)
|
||||
root = image.inode(image.root_nid)
|
||||
root_entries = image.directory_entries(root)
|
||||
target = image.resolve(spec["fixture"]["target"])
|
||||
_, shared_ids, inline_entries = image.body_shape(target)
|
||||
if root["size"] <= image.block_size:
|
||||
raise SystemExit("B19a root directory did not span multiple blocks")
|
||||
if not shared_ids or not inline_entries:
|
||||
raise SystemExit("B19a target lacks both shared and inline xattrs")
|
||||
target_positions = [
|
||||
index for index, (name, _) in enumerate(root_entries) if name == b"target.bin"
|
||||
]
|
||||
if target_positions != [len(root_entries) - 1]:
|
||||
raise SystemExit("B19a target is not uniquely sorted after all peers")
|
||||
for mutation in (
|
||||
"corrupt-shared-count",
|
||||
"corrupt-shared-id",
|
||||
"corrupt-inline-name",
|
||||
):
|
||||
mutate(valid, output / f"{mutation}.erofs", mutation, spec)
|
||||
fsck = shutil.which("fsck.erofs")
|
||||
if fsck is None:
|
||||
raise SystemExit("fsck.erofs is required")
|
||||
completed = subprocess.run(
|
||||
[fsck, str(valid)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
"fsck.erofs rejected B19a valid fixture: "
|
||||
+ completed.stdout.decode("utf-8", "replace")
|
||||
)
|
||||
hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
|
||||
frozen = spec.get("fixture_sha256", {})
|
||||
if frozen and hashes != frozen:
|
||||
raise SystemExit(f"B19a fixture hashes differ: {hashes}")
|
||||
report = {
|
||||
"block_size": image.block_size,
|
||||
"command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
|
||||
"fixture_sha256": hashes,
|
||||
"inline_entries": len(inline_entries),
|
||||
"root_directory_blocks": (root["size"] + image.block_size - 1) // image.block_size,
|
||||
"root_entry_count": len(root_entries),
|
||||
"shared_count": len(shared_ids),
|
||||
"status": "PASS",
|
||||
"target_entry_index": target_positions[0],
|
||||
}
|
||||
(output / "manifest.json").write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--work", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = load_spec(args.spec)
|
||||
try:
|
||||
report = generate(args.output, args.work, spec)
|
||||
finally:
|
||||
shutil.rmtree(args.work, ignore_errors=True)
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+853
@@ -0,0 +1,853 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent multi-block directory and xattr oracle for B19a."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
class Reject(RuntimeError):
|
||||
def __init__(self, errno_name: str, point: str):
|
||||
super().__init__(f"{errno_name} at {point}")
|
||||
self.errno_name = errno_name
|
||||
self.point = point
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def crc32c(data: bytes, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B19a"
|
||||
or spec.get("candidate") != "P15-022"
|
||||
):
|
||||
raise SystemExit("invalid B19a oracle spec identity")
|
||||
return spec
|
||||
|
||||
|
||||
class ReadStats:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.calls = 0
|
||||
self.bytes = 0
|
||||
self.block_reads = 0
|
||||
|
||||
def add(self, offset: int, length: int, block_size: int) -> None:
|
||||
with self.lock:
|
||||
self.calls += 1
|
||||
self.bytes += length
|
||||
if length:
|
||||
self.block_reads += (
|
||||
(offset + length - 1) // block_size - offset // block_size + 1
|
||||
)
|
||||
|
||||
def reset(self) -> None:
|
||||
with self.lock:
|
||||
self.calls = 0
|
||||
self.bytes = 0
|
||||
self.block_reads = 0
|
||||
|
||||
def snapshot(self) -> dict[str, int]:
|
||||
with self.lock:
|
||||
return {
|
||||
"bytes": self.bytes,
|
||||
"calls": self.calls,
|
||||
"provider_block_reads": self.block_reads,
|
||||
}
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.fd = os.open(path, os.O_RDONLY)
|
||||
self.size = os.fstat(self.fd).st_size
|
||||
self.stats = ReadStats()
|
||||
header = self.read(SUPER, 144, "super")
|
||||
if struct.unpack_from("<I", header)[0] != MAGIC:
|
||||
raise Reject("EINTEGRITY", "super.magic")
|
||||
self.feature_compat = struct.unpack_from("<I", header, 8)[0]
|
||||
self.block_bits = header[12]
|
||||
if self.block_bits < 9 or self.block_bits > 16:
|
||||
raise Reject("EINTEGRITY", "super.block-size")
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.root_nid = struct.unpack_from("<H", header, 14)[0]
|
||||
self.blocks = struct.unpack_from("<I", header, 36)[0]
|
||||
self.limit = self.blocks << self.block_bits
|
||||
self.meta_blkaddr = struct.unpack_from("<I", header, 40)[0]
|
||||
self.xattr_blkaddr = struct.unpack_from("<I", header, 44)[0]
|
||||
if self.limit > self.size:
|
||||
raise Reject("EINTEGRITY", "super.bounds")
|
||||
if self.feature_compat & 1:
|
||||
expected = struct.unpack_from("<I", header, 4)[0]
|
||||
raw = bytearray(self.read(SUPER, self.block_size - SUPER, "super.checksum"))
|
||||
raw[4:8] = bytes(4)
|
||||
if crc32c(bytes(raw)) != expected:
|
||||
raise Reject("EINTEGRITY", "super.checksum")
|
||||
self.stats.reset()
|
||||
|
||||
def close(self) -> None:
|
||||
os.close(self.fd)
|
||||
|
||||
def __enter__(self) -> "Reader":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.close()
|
||||
|
||||
def read(self, offset: int, length: int, point: str) -> bytes:
|
||||
limit = self.limit if hasattr(self, "limit") else self.size
|
||||
if offset < 0 or length < 0 or offset > limit or length > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{point}.bounds")
|
||||
data = os.pread(self.fd, length, offset)
|
||||
if len(data) != length:
|
||||
raise Reject("EINTEGRITY", f"{point}.short")
|
||||
block_size = self.block_size if hasattr(self, "block_size") else 4096
|
||||
self.stats.add(offset, length, block_size)
|
||||
return data
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
raw = self.read(offset, 64, "inode")
|
||||
inode_format, xattr_count = struct.unpack_from("<HH", raw)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", raw, 8)[0]
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": struct.unpack_from("<I", raw, 16)[0],
|
||||
}
|
||||
|
||||
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
||||
if logical > inode["size"] or length > inode["size"] - logical:
|
||||
raise Reject("EINTEGRITY", "inode.data-range")
|
||||
if inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
elif inode["layout"] == 2:
|
||||
tail_start = ((inode["size"] + self.block_size - 1) // self.block_size - 1) * self.block_size
|
||||
if logical < tail_start:
|
||||
if logical + length > tail_start:
|
||||
raise Reject("EINTEGRITY", "inode.inline-crossing")
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
physical = (
|
||||
inode["offset"]
|
||||
+ inode["inode_size"]
|
||||
+ inode["xattr_size"]
|
||||
+ logical
|
||||
- tail_start
|
||||
)
|
||||
else:
|
||||
raise Reject("EOPNOTSUPP", "inode.layout")
|
||||
return self.read(physical, length, "inode.data")
|
||||
|
||||
@staticmethod
|
||||
def parse_dirblock(data: bytes, block_index: int) -> list[dict[str, object]]:
|
||||
if len(data) < 12:
|
||||
raise Reject("EINTEGRITY", "directory.header")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name < 12 or first_name >= len(data) or first_name % 12:
|
||||
raise Reject("EINTEGRITY", "directory.name-offset")
|
||||
count = first_name // 12
|
||||
entries = []
|
||||
previous_offset = 0
|
||||
previous_name: bytes | None = None
|
||||
for index in range(count):
|
||||
offset = index * 12
|
||||
nid, start, file_type = struct.unpack_from("<QHB", data, offset)
|
||||
end = (
|
||||
struct.unpack_from("<H", data, offset + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(data)
|
||||
)
|
||||
if (
|
||||
start < first_name
|
||||
or start >= len(data)
|
||||
or (index == 0 and start != first_name)
|
||||
or (index != 0 and start <= previous_offset)
|
||||
or end <= start
|
||||
or end > len(data)
|
||||
):
|
||||
raise Reject("EINTEGRITY", "directory.name-bounds")
|
||||
span = data[start:end]
|
||||
if index + 1 < count:
|
||||
if b"\0" in span:
|
||||
raise Reject("EINTEGRITY", "directory.name-nul")
|
||||
name = span
|
||||
else:
|
||||
name = span.split(b"\0", 1)[0]
|
||||
if not name or len(name) > 255 or b"/" in name:
|
||||
raise Reject("EINTEGRITY", "directory.name")
|
||||
if previous_name is not None and previous_name >= name:
|
||||
raise Reject("EINTEGRITY", "directory.order")
|
||||
entries.append(
|
||||
{
|
||||
"block": block_index,
|
||||
"entry": index,
|
||||
"file_type": file_type,
|
||||
"name": name,
|
||||
"nid": nid,
|
||||
}
|
||||
)
|
||||
previous_offset = start
|
||||
previous_name = name
|
||||
return entries
|
||||
|
||||
@staticmethod
|
||||
def validate_boundary(
|
||||
left: list[dict[str, object]], right: list[dict[str, object]]
|
||||
) -> None:
|
||||
if left[-1]["name"] >= right[0]["name"]:
|
||||
raise Reject("EINTEGRITY", "directory.boundary-order")
|
||||
|
||||
@staticmethod
|
||||
def resolve_entries(entries: list[dict[str, object]], name: bytes) -> int:
|
||||
matches = [int(entry["nid"]) for entry in entries if entry["name"] == name]
|
||||
if not matches:
|
||||
raise Reject("ENOENT", "path.missing")
|
||||
if len(matches) != 1:
|
||||
raise Reject("EINTEGRITY", "path.duplicate")
|
||||
return matches[0]
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[dict[str, object]]:
|
||||
entries = []
|
||||
previous_name: bytes | None = None
|
||||
logical = 0
|
||||
block_index = 0
|
||||
while logical < inode["size"]:
|
||||
length = min(self.block_size, inode["size"] - logical)
|
||||
block_entries = self.parse_dirblock(
|
||||
self.inode_data(inode, logical, length), block_index
|
||||
)
|
||||
if entries:
|
||||
self.validate_boundary(entries, block_entries)
|
||||
entries.extend(block_entries)
|
||||
previous_name = block_entries[-1]["name"]
|
||||
logical += length
|
||||
block_index += 1
|
||||
return entries
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode("ascii").split(b"/"):
|
||||
if not component:
|
||||
continue
|
||||
inode = self.inode(
|
||||
self.resolve_entries(self.directory_entries(inode), component)
|
||||
)
|
||||
return inode
|
||||
|
||||
def body_offset(self, inode: dict[str, int]) -> int:
|
||||
return inode["offset"] + inode["inode_size"]
|
||||
|
||||
@staticmethod
|
||||
def parse_entry(raw: bytes, offset: int, limit: int, point: str) -> tuple[dict, int]:
|
||||
if offset > limit or 4 > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{point}.header")
|
||||
name_length, name_index, value_length = struct.unpack_from("<BBH", raw, offset)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
if total > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{point}.bounds")
|
||||
name_start = offset + 4
|
||||
name_end = name_start + name_length
|
||||
value_end = name_end + value_length
|
||||
name = raw[name_start:name_end]
|
||||
if b"\0" in name:
|
||||
raise Reject("EINTEGRITY", f"{point}.name-nul")
|
||||
return (
|
||||
{
|
||||
"index": name_index,
|
||||
"name": name,
|
||||
"value": raw[name_end:value_end],
|
||||
},
|
||||
total,
|
||||
)
|
||||
|
||||
def shared_entry(self, shared_id: int) -> dict:
|
||||
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
|
||||
header = self.read(offset, 4, "shared.header")
|
||||
name_length, _, value_length = struct.unpack("<BBH", header)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
raw = self.read(offset, total, "shared.entry")
|
||||
entry, parsed = self.parse_entry(raw, 0, len(raw), "shared")
|
||||
if parsed != total:
|
||||
raise Reject("EINTEGRITY", "shared.size")
|
||||
return entry
|
||||
|
||||
def load_body(self, inode: dict[str, int], validate_shared: bool) -> dict:
|
||||
size = inode["xattr_size"]
|
||||
if size < 12:
|
||||
raise Reject("EINTEGRITY", "ibody.header")
|
||||
if size == 12:
|
||||
raise Reject("EOPNOTSUPP", "ibody.header-only")
|
||||
raw = self.read(self.body_offset(inode), size, "ibody")
|
||||
shared_count = raw[4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > size:
|
||||
raise Reject("EINTEGRITY", "ibody.shared-count")
|
||||
shared_ids = [
|
||||
struct.unpack_from("<I", raw, 12 + index * 4)[0]
|
||||
for index in range(shared_count)
|
||||
]
|
||||
inline = []
|
||||
cursor = header_size
|
||||
while cursor < size:
|
||||
entry, total = self.parse_entry(raw, cursor, size, "inline")
|
||||
inline.append(entry)
|
||||
cursor += total
|
||||
if validate_shared:
|
||||
for shared_id in shared_ids:
|
||||
self.shared_entry(shared_id)
|
||||
return {
|
||||
"header_size": header_size,
|
||||
"inline": inline,
|
||||
"raw": raw,
|
||||
"shared_ids": shared_ids,
|
||||
}
|
||||
|
||||
|
||||
class MountBudget:
|
||||
def __init__(self, limit: int):
|
||||
self.limit = limit
|
||||
self.resident = 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def reserve(self, amount: int) -> bool:
|
||||
with self.lock:
|
||||
if amount > self.limit - self.resident:
|
||||
return False
|
||||
self.resident += amount
|
||||
return True
|
||||
|
||||
def release(self, amount: int) -> None:
|
||||
with self.lock:
|
||||
if amount > self.resident:
|
||||
raise RuntimeError("cache budget underflow")
|
||||
self.resident -= amount
|
||||
|
||||
|
||||
class BodyCache:
|
||||
EMPTY = "EMPTY"
|
||||
INFLIGHT = "INFLIGHT"
|
||||
READY = "READY"
|
||||
FAILED = "FAILED"
|
||||
|
||||
def __init__(self, spec: dict, budget: MountBudget):
|
||||
self.limit = spec["model"]["entry_body_limit_bytes"]
|
||||
self.budget = budget
|
||||
self.cv = threading.Condition()
|
||||
self.state = self.EMPTY
|
||||
self.body: dict | None = None
|
||||
self.error: Reject | None = None
|
||||
self.waiters = 0
|
||||
self.loads = 0
|
||||
self.charged = 0
|
||||
self.closing = False
|
||||
self.wait_for_waiters = 0
|
||||
|
||||
def acquire(self, reader: Reader, inode: dict[str, int]) -> dict:
|
||||
owner = False
|
||||
reserved = False
|
||||
with self.cv:
|
||||
if self.closing:
|
||||
raise Reject("ENXIO", "cache.closing")
|
||||
if self.state == self.READY:
|
||||
if self.body is None:
|
||||
raise RuntimeError("READY cache has no body")
|
||||
return self.body
|
||||
if self.state == self.INFLIGHT:
|
||||
self.waiters += 1
|
||||
try:
|
||||
while self.state == self.INFLIGHT:
|
||||
self.cv.wait()
|
||||
if self.state == self.READY:
|
||||
if self.body is None:
|
||||
raise RuntimeError("published cache has no body")
|
||||
return self.body
|
||||
if self.state != self.FAILED or self.error is None:
|
||||
raise RuntimeError("cache waiter has no typed result")
|
||||
raise Reject(self.error.errno_name, self.error.point)
|
||||
finally:
|
||||
self.waiters -= 1
|
||||
if self.state == self.FAILED and self.waiters == 0:
|
||||
self.error = None
|
||||
self.state = self.EMPTY
|
||||
if self.waiters == 0:
|
||||
self.cv.notify_all()
|
||||
if self.state == self.FAILED:
|
||||
if self.waiters:
|
||||
return reader.load_body(inode, validate_shared=True)
|
||||
self.error = None
|
||||
self.state = self.EMPTY
|
||||
if inode["xattr_size"] <= self.limit:
|
||||
reserved = self.budget.reserve(inode["xattr_size"])
|
||||
if not reserved:
|
||||
return reader.load_body(inode, validate_shared=True)
|
||||
self.state = self.INFLIGHT
|
||||
self.charged = inode["xattr_size"]
|
||||
owner = True
|
||||
if not owner:
|
||||
raise RuntimeError("cache claim lost owner state")
|
||||
try:
|
||||
if self.wait_for_waiters:
|
||||
deadline = time.monotonic() + 5
|
||||
while True:
|
||||
with self.cv:
|
||||
if self.waiters >= self.wait_for_waiters:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("cache waiters did not reach owner barrier")
|
||||
time.sleep(0.001)
|
||||
body = reader.load_body(inode, validate_shared=True)
|
||||
with self.cv:
|
||||
self.loads += 1
|
||||
self.body = body
|
||||
self.error = None
|
||||
self.state = self.READY
|
||||
self.cv.notify_all()
|
||||
return body
|
||||
except Reject as error:
|
||||
with self.cv:
|
||||
self.loads += 1
|
||||
self.budget.release(self.charged)
|
||||
self.charged = 0
|
||||
self.error = error
|
||||
self.state = self.FAILED
|
||||
if self.waiters == 0:
|
||||
self.error = None
|
||||
self.state = self.EMPTY
|
||||
self.cv.notify_all()
|
||||
raise
|
||||
|
||||
def invalidate(self) -> None:
|
||||
with self.cv:
|
||||
while self.state == self.INFLIGHT or self.waiters:
|
||||
self.cv.wait()
|
||||
if self.state == self.READY:
|
||||
self.body = None
|
||||
self.budget.release(self.charged)
|
||||
self.charged = 0
|
||||
self.error = None
|
||||
self.state = self.EMPTY
|
||||
|
||||
def close(self) -> None:
|
||||
with self.cv:
|
||||
self.closing = True
|
||||
while self.state == self.INFLIGHT or self.waiters:
|
||||
self.cv.wait()
|
||||
if self.state == self.READY:
|
||||
self.body = None
|
||||
self.budget.release(self.charged)
|
||||
self.charged = 0
|
||||
self.error = None
|
||||
self.state = self.EMPTY
|
||||
|
||||
|
||||
def body_for_operation(
|
||||
reader: Reader, inode: dict[str, int], variant: str, cache: BodyCache | None
|
||||
) -> dict:
|
||||
if variant == "baseline":
|
||||
return reader.load_body(inode, validate_shared=False)
|
||||
if variant != "candidate" or cache is None:
|
||||
raise RuntimeError("invalid B19a oracle variant")
|
||||
return cache.acquire(reader, inode)
|
||||
|
||||
|
||||
def entries_for_body(reader: Reader, body: dict) -> list[dict]:
|
||||
entries = list(body["inline"])
|
||||
entries.extend(reader.shared_entry(shared_id) for shared_id in body["shared_ids"])
|
||||
return entries
|
||||
|
||||
|
||||
def getxattr(
|
||||
reader: Reader,
|
||||
inode: dict[str, int],
|
||||
name: bytes,
|
||||
variant: str,
|
||||
cache: BodyCache | None,
|
||||
) -> bytes:
|
||||
body = body_for_operation(reader, inode, variant, cache)
|
||||
for entry in body["inline"]:
|
||||
if entry["index"] == 1 and entry["name"] == name:
|
||||
return bytes(entry["value"])
|
||||
for shared_id in body["shared_ids"]:
|
||||
entry = reader.shared_entry(shared_id)
|
||||
if entry["index"] == 1 and entry["name"] == name:
|
||||
return bytes(entry["value"])
|
||||
raise Reject("ENOATTR", "xattr.not-found")
|
||||
|
||||
|
||||
def listxattr(
|
||||
reader: Reader,
|
||||
inode: dict[str, int],
|
||||
variant: str,
|
||||
cache: BodyCache | None,
|
||||
) -> list[bytes]:
|
||||
return [
|
||||
bytes(entry["name"])
|
||||
for entry in entries_for_body(
|
||||
reader, body_for_operation(reader, inode, variant, cache)
|
||||
)
|
||||
if entry["index"] == 1
|
||||
]
|
||||
|
||||
|
||||
def expected_value(label: str, length: int) -> bytes:
|
||||
seed = (label + "|").encode("ascii")
|
||||
return (seed * (length // len(seed) + 1))[:length]
|
||||
|
||||
|
||||
def encode_dirblock(names: list[bytes], size: int = 128) -> bytes:
|
||||
first_name = len(names) * 12
|
||||
payload = bytearray(size)
|
||||
cursor = first_name
|
||||
for index, name in enumerate(names):
|
||||
if cursor + len(name) > size:
|
||||
raise ValueError("synthetic directory block is too small")
|
||||
struct.pack_into("<QHBB", payload, index * 12, index + 1, cursor, 1, 0)
|
||||
payload[cursor : cursor + len(name)] = name
|
||||
cursor += len(name)
|
||||
return bytes(payload)
|
||||
|
||||
|
||||
def expect_reject(function, errno_name: str, point: str) -> dict[str, str]:
|
||||
try:
|
||||
function()
|
||||
except Reject as error:
|
||||
if error.errno_name != errno_name or error.point != point:
|
||||
raise SystemExit(
|
||||
f"oracle control expected {errno_name} at {point}, got {error}"
|
||||
)
|
||||
return {"errno": error.errno_name, "point": error.point}
|
||||
raise SystemExit(f"oracle control unexpectedly passed: {errno_name} at {point}")
|
||||
|
||||
|
||||
def run_selftest(output: Path) -> dict:
|
||||
controls = {}
|
||||
first = Reader.parse_dirblock(
|
||||
encode_dirblock([b"alpha", b"boundary-last"], 96),
|
||||
0,
|
||||
)
|
||||
second = Reader.parse_dirblock(
|
||||
encode_dirblock([b"target", b"zeta"], 80), 1
|
||||
)
|
||||
Reader.validate_boundary(first, second)
|
||||
combined = first + second
|
||||
target_nid = Reader.resolve_entries(combined, b"target")
|
||||
matches = [entry for entry in combined if entry["name"] == b"target"]
|
||||
if (
|
||||
len(matches) != 1
|
||||
or matches[0]["block"] != 1
|
||||
or int(matches[0]["nid"]) != target_nid
|
||||
):
|
||||
raise SystemExit("target was not resolved after a block boundary")
|
||||
controls["target-after-boundary"] = "PASS"
|
||||
controls["short-final-block"] = "PASS"
|
||||
|
||||
controls["duplicate-within-block"] = expect_reject(
|
||||
lambda: Reader.parse_dirblock(encode_dirblock([b"dup", b"dup"]), 0),
|
||||
"EINTEGRITY",
|
||||
"directory.order",
|
||||
)
|
||||
left = Reader.parse_dirblock(encode_dirblock([b"alpha", b"dup"]), 0)
|
||||
right = Reader.parse_dirblock(encode_dirblock([b"dup", b"zeta"]), 1)
|
||||
controls["duplicate-across-blocks"] = expect_reject(
|
||||
lambda: Reader.validate_boundary(left, right),
|
||||
"EINTEGRITY",
|
||||
"directory.boundary-order",
|
||||
)
|
||||
controls["missing-target"] = expect_reject(
|
||||
lambda: Reader.resolve_entries(combined, b"missing"),
|
||||
"ENOENT",
|
||||
"path.missing",
|
||||
)
|
||||
duplicate_entries = combined + [{**combined[-1], "nid": 999, "name": b"target"}]
|
||||
controls["duplicate-resolution"] = expect_reject(
|
||||
lambda: Reader.resolve_entries(duplicate_entries, b"target"),
|
||||
"EINTEGRITY",
|
||||
"path.duplicate",
|
||||
)
|
||||
backwards = Reader.parse_dirblock(encode_dirblock([b"aardvark", b"beta"]), 1)
|
||||
controls["backwards-boundary"] = expect_reject(
|
||||
lambda: Reader.validate_boundary(first, backwards),
|
||||
"EINTEGRITY",
|
||||
"directory.boundary-order",
|
||||
)
|
||||
malformed = bytearray(encode_dirblock([b"alpha", b"bravo"]))
|
||||
struct.pack_into("<H", malformed, 20, len(malformed) + 1)
|
||||
controls["crossing-name"] = expect_reject(
|
||||
lambda: Reader.parse_dirblock(bytes(malformed), 0),
|
||||
"EINTEGRITY",
|
||||
"directory.name-bounds",
|
||||
)
|
||||
report = {"controls": controls, "status": "PASS"}
|
||||
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
return report
|
||||
|
||||
|
||||
def operation_result(
|
||||
image: Path, spec: dict, variant: str, operation: str
|
||||
) -> dict[str, object]:
|
||||
budget = MountBudget(spec["model"]["cache_budget_bytes"])
|
||||
cache = BodyCache(spec, budget) if variant == "candidate" else None
|
||||
try:
|
||||
with Reader(image) as reader:
|
||||
inode = reader.resolve(spec["fixture"]["target"])
|
||||
reader.stats.reset()
|
||||
if operation == "get-inline":
|
||||
value = getxattr(reader, inode, b"inline", variant, cache)
|
||||
return {"status": "PASS", "value_sha256": hashlib.sha256(value).hexdigest()}
|
||||
if operation == "get-shared":
|
||||
value = getxattr(reader, inode, b"shared", variant, cache)
|
||||
return {"status": "PASS", "value_sha256": hashlib.sha256(value).hexdigest()}
|
||||
if operation == "list-user":
|
||||
names = listxattr(reader, inode, variant, cache)
|
||||
return {"names": [name.decode("ascii") for name in names], "status": "PASS"}
|
||||
raise RuntimeError(f"unknown operation: {operation}")
|
||||
except Reject as error:
|
||||
return {"errno": spec["errno"].get(error.errno_name, -1), "point": error.point, "status": error.errno_name}
|
||||
finally:
|
||||
if cache is not None:
|
||||
cache.close()
|
||||
if budget.resident != 0:
|
||||
raise RuntimeError("operation cache budget leaked")
|
||||
|
||||
|
||||
def run_correctness(fixtures: Path, spec: dict, output: Path) -> dict:
|
||||
valid = fixtures / "valid.erofs"
|
||||
with Reader(valid) as reader:
|
||||
root = reader.inode(reader.root_nid)
|
||||
root_entries = reader.directory_entries(root)
|
||||
target_entries = [entry for entry in root_entries if entry["name"] == b"target.bin"]
|
||||
if len(target_entries) != 1 or int(target_entries[0]["block"]) == 0:
|
||||
raise SystemExit("real target did not resolve after the first block")
|
||||
if [entry["name"] for entry in root_entries] != sorted(
|
||||
entry["name"] for entry in root_entries
|
||||
):
|
||||
raise SystemExit("real directory order is unstable")
|
||||
boundary = []
|
||||
for left, right in zip(root_entries, root_entries[1:]):
|
||||
if left["block"] != right["block"]:
|
||||
boundary.append(
|
||||
{
|
||||
"left": bytes(left["name"]).decode("ascii"),
|
||||
"right": bytes(right["name"]).decode("ascii"),
|
||||
}
|
||||
)
|
||||
inode = reader.resolve(spec["fixture"]["target"])
|
||||
body = reader.load_body(inode, validate_shared=True)
|
||||
if not body["inline"] or not body["shared_ids"]:
|
||||
raise SystemExit("real target lacks inline/shared xattr coverage")
|
||||
directory = {
|
||||
"block_count": (root["size"] + reader.block_size - 1) // reader.block_size,
|
||||
"boundaries": boundary,
|
||||
"entry_count": len(root_entries),
|
||||
"target_block": target_entries[0]["block"],
|
||||
"target_entry": target_entries[0]["entry"],
|
||||
}
|
||||
|
||||
expected_inline = expected_value(
|
||||
"inline-value-p15-022", spec["fixture"]["inline_value_bytes"]
|
||||
)
|
||||
expected_shared = expected_value(
|
||||
"shared-value-p15-022", spec["fixture"]["shared_value_bytes"]
|
||||
)
|
||||
cases = []
|
||||
for variant in ("baseline", "candidate"):
|
||||
inline = operation_result(valid, spec, variant, "get-inline")
|
||||
shared = operation_result(valid, spec, variant, "get-shared")
|
||||
listed = operation_result(valid, spec, variant, "list-user")
|
||||
if inline.get("value_sha256") != hashlib.sha256(expected_inline).hexdigest():
|
||||
raise SystemExit(f"{variant} inline value mismatch")
|
||||
if shared.get("value_sha256") != hashlib.sha256(expected_shared).hexdigest():
|
||||
raise SystemExit(f"{variant} shared value mismatch")
|
||||
if sorted(listed.get("names", [])) != ["inline", "shared"]:
|
||||
raise SystemExit(f"{variant} list result mismatch: {listed}")
|
||||
cases.extend(
|
||||
[
|
||||
{"id": "valid-inline", "variant": variant, "result": inline},
|
||||
{"id": "valid-shared", "variant": variant, "result": shared},
|
||||
{"id": "valid-list", "variant": variant, "result": listed},
|
||||
]
|
||||
)
|
||||
|
||||
damaged = [
|
||||
("corrupt-shared-count.erofs", "get-inline", "EINTEGRITY"),
|
||||
("corrupt-shared-id.erofs", "get-shared", "EINTEGRITY"),
|
||||
("corrupt-inline-name.erofs", "list-user", "EINTEGRITY"),
|
||||
]
|
||||
for filename, operation, expected in damaged:
|
||||
for variant in ("baseline", "candidate"):
|
||||
result = operation_result(fixtures / filename, spec, variant, operation)
|
||||
if result["status"] != expected or int(result.get("errno", -1)) < 0:
|
||||
raise SystemExit(f"{filename} {variant} mismatch: {result}")
|
||||
cases.append({"id": filename, "variant": variant, "result": result})
|
||||
|
||||
budget = MountBudget(spec["model"]["cache_budget_bytes"])
|
||||
cache = BodyCache(spec, budget)
|
||||
cache.wait_for_waiters = spec["concurrency"]["workers"] - 1
|
||||
with Reader(valid) as reader:
|
||||
inode = reader.resolve(spec["fixture"]["target"])
|
||||
barrier = threading.Barrier(spec["concurrency"]["workers"])
|
||||
|
||||
def worker(_: int) -> str:
|
||||
barrier.wait()
|
||||
value = getxattr(reader, inode, b"inline", "candidate", cache)
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=spec["concurrency"]["workers"]) as executor:
|
||||
hashes = list(executor.map(worker, range(spec["concurrency"]["workers"])))
|
||||
if len(set(hashes)) != 1 or cache.loads != 1 or cache.state != BodyCache.READY:
|
||||
raise SystemExit("candidate owner/waiter publication mismatch")
|
||||
first_charge = budget.resident
|
||||
cache.invalidate()
|
||||
if budget.resident != 0 or cache.state != BodyCache.EMPTY:
|
||||
raise SystemExit("candidate invalidation did not release cache")
|
||||
cache.wait_for_waiters = 0
|
||||
getxattr(reader, inode, b"inline", "candidate", cache)
|
||||
if cache.loads != 2 or budget.resident != first_charge:
|
||||
raise SystemExit("candidate re-establishment mismatch")
|
||||
cache.close()
|
||||
if budget.resident != 0:
|
||||
raise SystemExit("candidate reclaim/close leaked budget")
|
||||
|
||||
report = {
|
||||
"cases": cases,
|
||||
"concurrency": {
|
||||
"body_loads": 1,
|
||||
"completed": spec["concurrency"]["workers"],
|
||||
"workers": spec["concurrency"]["workers"],
|
||||
},
|
||||
"directory": directory,
|
||||
"fixture_sha256": {
|
||||
path.name: sha256(path) for path in sorted(fixtures.glob("*.erofs"))
|
||||
},
|
||||
"lifecycle": {
|
||||
"establish": "PASS",
|
||||
"hit": "PASS",
|
||||
"invalidate": "PASS",
|
||||
"reclaim": "PASS",
|
||||
"resident_after": budget.resident,
|
||||
},
|
||||
"status": "PASS",
|
||||
}
|
||||
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
return report
|
||||
|
||||
|
||||
def run_sample(
|
||||
image: Path, spec: dict, variant: str, sample: int, output: Path
|
||||
) -> dict:
|
||||
budget = MountBudget(spec["model"]["cache_budget_bytes"])
|
||||
cache = BodyCache(spec, budget) if variant == "candidate" else None
|
||||
operations = spec["benchmark"]["operations"]
|
||||
with Reader(image) as reader:
|
||||
inode = reader.resolve(spec["fixture"]["target"])
|
||||
|
||||
def one_loop() -> None:
|
||||
for operation in operations:
|
||||
if operation == "get-inline":
|
||||
getxattr(reader, inode, b"inline", variant, cache)
|
||||
elif operation == "get-shared":
|
||||
getxattr(reader, inode, b"shared", variant, cache)
|
||||
elif operation == "list-user":
|
||||
listxattr(reader, inode, variant, cache)
|
||||
else:
|
||||
raise RuntimeError(f"unknown benchmark operation: {operation}")
|
||||
|
||||
for _ in range(spec["benchmark"]["warmup_loops"]):
|
||||
one_loop()
|
||||
reader.stats.reset()
|
||||
started = time.monotonic_ns()
|
||||
for _ in range(spec["benchmark"]["loops_per_sample"]):
|
||||
one_loop()
|
||||
elapsed = time.monotonic_ns() - started
|
||||
reads = reader.stats.snapshot()
|
||||
loads = cache.loads if cache is not None else 0
|
||||
resident_before_close = budget.resident
|
||||
if cache is not None:
|
||||
cache.close()
|
||||
if budget.resident != 0:
|
||||
raise SystemExit("benchmark cache budget leaked")
|
||||
report = {
|
||||
"cache_body_loads": loads,
|
||||
"cache_resident_before_close": resident_before_close,
|
||||
"elapsed_ns": elapsed,
|
||||
"fixture_sha256": sha256(image),
|
||||
"host": os.uname().sysname + " " + os.uname().release,
|
||||
"loops": spec["benchmark"]["loops_per_sample"],
|
||||
"operations": operations,
|
||||
"reads": reads,
|
||||
"sample": sample,
|
||||
"status": "PASS",
|
||||
"variant": variant,
|
||||
"warmup_loops": spec["benchmark"]["warmup_loops"],
|
||||
}
|
||||
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
selftest = subparsers.add_parser("selftest")
|
||||
selftest.add_argument("--output", type=Path, required=True)
|
||||
correctness = subparsers.add_parser("correctness")
|
||||
correctness.add_argument("--fixtures", type=Path, required=True)
|
||||
correctness.add_argument("--output", type=Path, required=True)
|
||||
sample = subparsers.add_parser("sample")
|
||||
sample.add_argument("--image", type=Path, required=True)
|
||||
sample.add_argument("--variant", choices=("baseline", "candidate"), required=True)
|
||||
sample.add_argument("--sample", type=int, required=True)
|
||||
sample.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = load_spec(args.spec)
|
||||
if args.command == "selftest":
|
||||
report = run_selftest(args.output)
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
elif args.command == "correctness":
|
||||
report = run_correctness(args.fixtures, spec, args.output)
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
else:
|
||||
run_sample(args.image, spec, args.variant, args.sample, args.output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,189 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/extattr.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "%s\n", message);
|
||||
return (1);
|
||||
}
|
||||
|
||||
static int
|
||||
get_repeat(const char *path, const char *name, size_t expected_size)
|
||||
{
|
||||
uint8_t first[1024], second[1024];
|
||||
ssize_t first_size, second_size;
|
||||
|
||||
if (expected_size > sizeof(first))
|
||||
return (fail("xattr size exceeds probe buffer"));
|
||||
memset(first, 0xa5, sizeof(first));
|
||||
memset(second, 0x5a, sizeof(second));
|
||||
errno = 0;
|
||||
first_size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name,
|
||||
first, sizeof(first));
|
||||
if (first_size != (ssize_t)expected_size || errno != 0)
|
||||
return (fail("first xattr read returned the wrong size or errno"));
|
||||
errno = 0;
|
||||
second_size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name,
|
||||
second, sizeof(second));
|
||||
if (second_size != first_size || errno != 0 ||
|
||||
memcmp(first, second, expected_size) != 0)
|
||||
return (fail("repeated xattr read changed its result"));
|
||||
for (size_t offset = expected_size; offset < sizeof(first); offset++) {
|
||||
if (first[offset] != 0xa5 || second[offset] != 0x5a)
|
||||
return (fail("xattr read wrote beyond its value"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
get_errno(const char *path, const char *name, int expected_errno)
|
||||
{
|
||||
uint8_t value[32];
|
||||
ssize_t size;
|
||||
|
||||
memset(value, 0xa5, sizeof(value));
|
||||
errno = 0;
|
||||
size = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, value,
|
||||
sizeof(value));
|
||||
if (size != -1 || errno != expected_errno) {
|
||||
fprintf(stderr, "xattr errno mismatch: size=%zd errno=%d expected=%d\n",
|
||||
size, errno, expected_errno);
|
||||
return (fail("xattr returned the wrong positive errno"));
|
||||
}
|
||||
for (size_t offset = 0; offset < sizeof(value); offset++) {
|
||||
if (value[offset] != 0xa5)
|
||||
return (fail("failed xattr lookup modified the output"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
list_user(const char *path)
|
||||
{
|
||||
uint8_t *buffer;
|
||||
ssize_t size;
|
||||
size_t offset;
|
||||
unsigned int seen;
|
||||
|
||||
errno = 0;
|
||||
size = extattr_list_file(path, EXTATTR_NAMESPACE_USER, NULL, 0);
|
||||
if (size <= 0 || errno != 0)
|
||||
return (fail("xattr list sizing failed"));
|
||||
buffer = malloc((size_t)size);
|
||||
if (buffer == NULL)
|
||||
return (fail("xattr list allocation failed"));
|
||||
if (extattr_list_file(path, EXTATTR_NAMESPACE_USER, buffer,
|
||||
(size_t)size) != size) {
|
||||
free(buffer);
|
||||
return (fail("xattr list transfer failed"));
|
||||
}
|
||||
seen = 0;
|
||||
for (offset = 0; offset < (size_t)size;) {
|
||||
const char *name;
|
||||
uint8_t name_size;
|
||||
|
||||
name_size = buffer[offset++];
|
||||
if (name_size == 0 || name_size > (size_t)size - offset) {
|
||||
free(buffer);
|
||||
return (fail("xattr list record is malformed"));
|
||||
}
|
||||
name = (const char *)(buffer + offset);
|
||||
if (name_size == sizeof("inline") - 1 &&
|
||||
memcmp(name, "inline", name_size) == 0) {
|
||||
if ((seen & 1U) != 0) {
|
||||
free(buffer);
|
||||
return (fail("xattr list duplicated inline"));
|
||||
}
|
||||
seen |= 1U;
|
||||
} else if (name_size == sizeof("shared") - 1 &&
|
||||
memcmp(name, "shared", name_size) == 0) {
|
||||
if ((seen & 2U) != 0) {
|
||||
free(buffer);
|
||||
return (fail("xattr list duplicated shared"));
|
||||
}
|
||||
seen |= 2U;
|
||||
} else {
|
||||
free(buffer);
|
||||
return (fail("xattr list returned an unexpected name"));
|
||||
}
|
||||
offset += name_size;
|
||||
}
|
||||
free(buffer);
|
||||
return (seen == 3U ? 0 : fail("xattr list omitted a name"));
|
||||
}
|
||||
|
||||
static int
|
||||
valid(const char *path)
|
||||
{
|
||||
if (get_repeat(path, "inline", 127) != 0 ||
|
||||
get_repeat(path, "shared", 511) != 0 ||
|
||||
list_user(path) != 0 ||
|
||||
get_errno(path, "missing", ENOATTR) != 0)
|
||||
return (1);
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
corrupt(const char *path)
|
||||
{
|
||||
return (get_errno(path, "inline", EINTEGRITY));
|
||||
}
|
||||
|
||||
static int
|
||||
concurrent(const char *path, long workers, long loops)
|
||||
{
|
||||
int status;
|
||||
pid_t child;
|
||||
|
||||
for (long worker = 0; worker < workers; worker++) {
|
||||
child = fork();
|
||||
if (child < 0)
|
||||
return (fail("xattr worker fork failed"));
|
||||
if (child == 0) {
|
||||
for (long iteration = 0; iteration < loops; iteration++) {
|
||||
if (valid(path) != 0)
|
||||
_exit(1);
|
||||
}
|
||||
_exit(0);
|
||||
}
|
||||
}
|
||||
for (long worker = 0; worker < workers; worker++) {
|
||||
if (wait(&status) < 0 || !WIFEXITED(status) ||
|
||||
WEXITSTATUS(status) != 0)
|
||||
return (fail("xattr worker failed"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
char *end;
|
||||
long workers, loops;
|
||||
|
||||
if (argc == 3 && strcmp(argv[1], "valid") == 0)
|
||||
return (valid(argv[2]));
|
||||
if (argc == 3 && strcmp(argv[1], "corrupt") == 0)
|
||||
return (corrupt(argv[2]));
|
||||
if (argc == 5 && strcmp(argv[1], "concurrent") == 0) {
|
||||
errno = 0;
|
||||
workers = strtol(argv[3], &end, 10);
|
||||
if (errno != 0 || end[0] != '\0' || workers < 1 || workers > 64)
|
||||
return (fail("invalid worker count"));
|
||||
errno = 0;
|
||||
loops = strtol(argv[4], &end, 10);
|
||||
if (errno != 0 || end[0] != '\0' || loops < 1 || loops > 100)
|
||||
return (fail("invalid loop count"));
|
||||
return (concurrent(argv[2], workers, loops));
|
||||
}
|
||||
return (fail("usage: B19a-xattr-probe valid|corrupt|concurrent ..."));
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"batch": "B19a",
|
||||
"benchmark": {
|
||||
"loops_per_sample": 200,
|
||||
"operations": [
|
||||
"get-inline",
|
||||
"get-shared",
|
||||
"list-user"
|
||||
],
|
||||
"sample_timeout_seconds": 30,
|
||||
"samples": 5,
|
||||
"warmup_loops": 20
|
||||
},
|
||||
"candidate": "P15-022",
|
||||
"concurrency": {
|
||||
"loops_per_worker": 10,
|
||||
"workers": 64
|
||||
},
|
||||
"errno": {
|
||||
"EINTEGRITY": 97,
|
||||
"ENOATTR": 87,
|
||||
"ENOENT": 2,
|
||||
"ENOMEM": 12,
|
||||
"ENXIO": 6
|
||||
},
|
||||
"fixture": {
|
||||
"budget_file_count": 400,
|
||||
"budget_value_bytes": 3000,
|
||||
"inline_name": "inline",
|
||||
"inline_value_bytes": 127,
|
||||
"peer_count": 64,
|
||||
"peer_name_bytes": 72,
|
||||
"shared_name": "shared",
|
||||
"shared_value_bytes": 511,
|
||||
"target": "/target.bin",
|
||||
"uuid": "00000000-0000-0000-0000-000000000022"
|
||||
},
|
||||
"fixture_sha256": {
|
||||
"corrupt-inline-name.erofs": "6b41a7ea8beb359c8ddf36bc1e700915fd594666a9d2a814df847fbae9f854f5",
|
||||
"corrupt-shared-count.erofs": "ce3f29a56201e9b0f3bfe7720fa9cd8293f7eee2169c02f8c58c2ccf7e984cd9",
|
||||
"corrupt-shared-id.erofs": "09ed7ea43f304d9ed9e38a29e0b7b66b6ef063a1ae3906a3b86a455af19985af",
|
||||
"valid.erofs": "1970bf1625077be62f1dff08bd5303010e1da5c8a820da1317bfdd645812d5c6"
|
||||
},
|
||||
"model": {
|
||||
"cache_budget_bytes": 1048576,
|
||||
"entry_body_limit_bytes": 65536
|
||||
},
|
||||
"schema": 1,
|
||||
"thresholds": {
|
||||
"minimum_provider_metadata_read_reduction_percent": 25.0
|
||||
}
|
||||
}
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B19b KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B19b KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
fi
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' 'B19b KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -qi 'xxh'; then
|
||||
printf '%s\n' 'B19b KLD contains an external xxh symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -q ' ZSTD_'; then
|
||||
printf '%s\n' 'B19b KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic real EROFS Bloom fixtures for B19b."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B19b" or spec.get("candidate") != "P15-021":
|
||||
raise SystemExit("invalid B19b fixture spec identity")
|
||||
return spec
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
||||
raise ValueError("invalid EROFS image")
|
||||
self.block_bits = self.data[SUPER + 12]
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.blocks = self.u32(SUPER + 36)
|
||||
self.limit = self.blocks << self.block_bits
|
||||
self.meta_blkaddr = self.u32(SUPER + 40)
|
||||
self.root_nid = self.u16(SUPER + 14)
|
||||
if self.limit > len(self.data):
|
||||
raise ValueError("EROFS image is truncated")
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_count = self.u16(offset + 2)
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", self.data, offset + 8)[0]
|
||||
return {
|
||||
"nid": nid,
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": self.u32(offset + 16),
|
||||
}
|
||||
|
||||
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
||||
if inode["layout"] == 2:
|
||||
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
|
||||
elif inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
raise ValueError("fixture directory has an unsupported layout")
|
||||
if physical > self.limit or length > self.limit - physical:
|
||||
raise ValueError("fixture directory is out of bounds")
|
||||
return bytes(self.data[physical:physical + length])
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
result = []
|
||||
logical = 0
|
||||
while logical < inode["size"]:
|
||||
length = min(self.block_size, inode["size"] - logical)
|
||||
data = self.inode_data(inode, logical, length)
|
||||
if len(data) < 12:
|
||||
raise ValueError("fixture directory block is short")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name == 0 or first_name % 12 or first_name > len(data):
|
||||
raise ValueError("fixture directory name offset is invalid")
|
||||
count = first_name // 12
|
||||
for index in range(count):
|
||||
entry = index * 12
|
||||
nid = struct.unpack_from("<Q", data, entry)[0]
|
||||
start = struct.unpack_from("<H", data, entry + 8)[0]
|
||||
end = struct.unpack_from("<H", data, entry + 20)[0] if index + 1 < count else len(data)
|
||||
if start > end or end > len(data):
|
||||
raise ValueError("fixture directory name is out of bounds")
|
||||
result.append((data[start:end].split(b"\0", 1)[0], nid))
|
||||
logical += length
|
||||
return result
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode().split(b"/"):
|
||||
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"fixture path is absent or ambiguous: {path}")
|
||||
inode = self.inode(matches[0])
|
||||
return inode
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
compat = self.u32(SUPER + 8)
|
||||
if compat & 1:
|
||||
self.put_u32(SUPER + 4, 0)
|
||||
self.put_u32(SUPER + 4, crc32c(self.data[SUPER:self.block_size]))
|
||||
|
||||
def write(self, path: Path) -> None:
|
||||
path.write_bytes(self.data)
|
||||
|
||||
|
||||
def create_source(path: Path, spec: dict) -> None:
|
||||
path.mkdir(parents=True)
|
||||
path.chmod(0o755)
|
||||
files = [path / f"peer-{index:03d}.bin" for index in range(spec["fixture"]["peer_count"])]
|
||||
files.append(path / "target.bin")
|
||||
for entry in files:
|
||||
entry.write_bytes(b"P15-021\n")
|
||||
entry.chmod(0o644)
|
||||
for index in range(spec["fixture"]["attribute_count"]):
|
||||
name = f"user.attr{index:02d}".encode()
|
||||
prefix = f"value-{index:02d}-".encode()
|
||||
length = spec["fixture"]["attribute_value_bytes"]
|
||||
value = (prefix * (length // len(prefix) + 1))[:length]
|
||||
os.setxattr(entry, name, value)
|
||||
os.utime(entry, (0, 0), follow_symlinks=False)
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def build_valid(output: Path, source: Path, spec: dict) -> list[str]:
|
||||
mkfs = shutil.which("mkfs.erofs")
|
||||
if mkfs is None:
|
||||
raise SystemExit("mkfs.erofs is required")
|
||||
command = [
|
||||
mkfs,
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"--sort=path",
|
||||
f"-U{spec['fixture']['uuid']}",
|
||||
"-x2",
|
||||
"-Exattr-name-filter,force-inode-extended",
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
completed = subprocess.run(command, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"mkfs.erofs failed: {completed.stdout.decode('utf-8', 'replace')}")
|
||||
return command
|
||||
|
||||
|
||||
def mutate(valid: Path, output: Path, mutation: str, spec: dict) -> None:
|
||||
image = Image.load(valid)
|
||||
target = image.resolve(spec["fixture"]["target"])
|
||||
body = target["offset"] + target["inode_size"]
|
||||
if mutation == "unknown-filter":
|
||||
image.data[SUPER + 104] = 1
|
||||
elif mutation == "feature-off":
|
||||
image.put_u32(SUPER + 8, image.u32(SUPER + 8) & ~spec["format"]["feature_compat"])
|
||||
elif mutation == "corrupt-shared-count":
|
||||
image.data[body + 4] = 255
|
||||
elif mutation == "corrupt-shared-id":
|
||||
if image.data[body + 4] == 0:
|
||||
raise SystemExit("valid fixture has no shared IDs")
|
||||
image.put_u32(body + 12, 0xFFFFFFFF)
|
||||
else:
|
||||
raise SystemExit(f"unknown mutation: {mutation}")
|
||||
image.update_checksum()
|
||||
image.write(output)
|
||||
|
||||
|
||||
def generate(output: Path, work: Path, spec: dict) -> None:
|
||||
if output.exists():
|
||||
raise SystemExit(f"refusing existing output: {output}")
|
||||
if work.exists():
|
||||
raise SystemExit(f"refusing existing work directory: {work}")
|
||||
output.mkdir(parents=True)
|
||||
work.mkdir(parents=True)
|
||||
source = work / "source"
|
||||
create_source(source, spec)
|
||||
valid = output / "valid.erofs"
|
||||
command = build_valid(valid, source, spec)
|
||||
for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"):
|
||||
mutate(valid, output / f"{mutation}.erofs", mutation, spec)
|
||||
hashes = {path.name: sha256(path) for path in sorted(output.glob("*.erofs"))}
|
||||
if hashes != spec["fixture_sha256"]:
|
||||
raise SystemExit(f"B19b generated fixture hashes differ: {hashes}")
|
||||
fsck = shutil.which("fsck.erofs")
|
||||
if fsck is None:
|
||||
raise SystemExit("fsck.erofs is required")
|
||||
for name in ("valid.erofs", "unknown-filter.erofs", "feature-off.erofs"):
|
||||
completed = subprocess.run([fsck, str(output / name)], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"fsck.erofs rejected legal fixture {name}")
|
||||
sums = "".join(f"{digest} {name}\n" for name, digest in sorted(hashes.items()))
|
||||
(output / "SHA256SUMS").write_text(sums, encoding="ascii")
|
||||
write_json(output / "manifest.json", {
|
||||
"command": [*command[:-2], "OUTPUT/valid.erofs", "WORK/source"],
|
||||
"fixture_sha256": hashes,
|
||||
"schema": 1,
|
||||
"status": "PASS",
|
||||
})
|
||||
print(json.dumps({"fixture_sha256": hashes, "status": "PASS"}, sort_keys=True))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("generate", choices=("generate",))
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--work", type=Path, required=True)
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
generate(args.output, args.work, load_spec(args.spec))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+426
@@ -0,0 +1,426 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently verify B19b fixtures, Bloom semantics, and source scope."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
|
||||
|
||||
class Reject(RuntimeError):
|
||||
def __init__(self, errno_name: str, point: str):
|
||||
super().__init__(f"{errno_name} at {point}")
|
||||
self.errno_name = errno_name
|
||||
self.point = point
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
|
||||
|
||||
|
||||
def rotl32(value: int, count: int) -> int:
|
||||
return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF
|
||||
|
||||
|
||||
def xxh32(data: bytes, seed: int) -> int:
|
||||
prime1 = 2654435761
|
||||
prime2 = 2246822519
|
||||
prime3 = 3266489917
|
||||
prime4 = 668265263
|
||||
prime5 = 374761393
|
||||
cursor = 0
|
||||
|
||||
def round32(accumulator: int, lane: int) -> int:
|
||||
accumulator = (accumulator + lane * prime2) & 0xFFFFFFFF
|
||||
return (rotl32(accumulator, 13) * prime1) & 0xFFFFFFFF
|
||||
|
||||
if len(data) >= 16:
|
||||
accumulator1 = (seed + prime1 + prime2) & 0xFFFFFFFF
|
||||
accumulator2 = (seed + prime2) & 0xFFFFFFFF
|
||||
accumulator3 = seed & 0xFFFFFFFF
|
||||
accumulator4 = (seed - prime1) & 0xFFFFFFFF
|
||||
limit = len(data) - 16
|
||||
while cursor <= limit:
|
||||
accumulator1 = round32(accumulator1, int.from_bytes(data[cursor:cursor + 4], "little"))
|
||||
accumulator2 = round32(accumulator2, int.from_bytes(data[cursor + 4:cursor + 8], "little"))
|
||||
accumulator3 = round32(accumulator3, int.from_bytes(data[cursor + 8:cursor + 12], "little"))
|
||||
accumulator4 = round32(accumulator4, int.from_bytes(data[cursor + 12:cursor + 16], "little"))
|
||||
cursor += 16
|
||||
value = (
|
||||
rotl32(accumulator1, 1)
|
||||
+ rotl32(accumulator2, 7)
|
||||
+ rotl32(accumulator3, 12)
|
||||
+ rotl32(accumulator4, 18)
|
||||
) & 0xFFFFFFFF
|
||||
else:
|
||||
value = (seed + prime5) & 0xFFFFFFFF
|
||||
value = (value + len(data)) & 0xFFFFFFFF
|
||||
while cursor + 4 <= len(data):
|
||||
value = (value + int.from_bytes(data[cursor:cursor + 4], "little") * prime3) & 0xFFFFFFFF
|
||||
value = (rotl32(value, 17) * prime4) & 0xFFFFFFFF
|
||||
cursor += 4
|
||||
while cursor < len(data):
|
||||
value = (value + data[cursor] * prime5) & 0xFFFFFFFF
|
||||
value = (rotl32(value, 11) * prime1) & 0xFFFFFFFF
|
||||
cursor += 1
|
||||
value ^= value >> 15
|
||||
value = (value * prime2) & 0xFFFFFFFF
|
||||
value ^= value >> 13
|
||||
value = (value * prime3) & 0xFFFFFFFF
|
||||
value ^= value >> 16
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, path: Path, spec: dict):
|
||||
self.data = path.read_bytes()
|
||||
self.spec = spec
|
||||
self.calls = 0
|
||||
self.bytes = 0
|
||||
self.blocks_read: set[int] = set()
|
||||
header = self.read(SUPER, 144, "super")
|
||||
if struct.unpack_from("<I", header)[0] != MAGIC:
|
||||
raise Reject("EINTEGRITY", "super.magic")
|
||||
self.feature_compat = struct.unpack_from("<I", header, 8)[0]
|
||||
self.block_bits = header[12]
|
||||
self.block_size = 1 << self.block_bits
|
||||
self.root_nid = struct.unpack_from("<H", header, 14)[0]
|
||||
self.blocks = struct.unpack_from("<I", header, 36)[0]
|
||||
self.limit = self.blocks << self.block_bits
|
||||
self.meta_blkaddr = struct.unpack_from("<I", header, 40)[0]
|
||||
self.xattr_blkaddr = struct.unpack_from("<I", header, 44)[0]
|
||||
self.filter_reserved = header[104]
|
||||
if self.limit > len(self.data):
|
||||
raise Reject("EINTEGRITY", "super.bounds")
|
||||
|
||||
def read(self, offset: int, length: int, point: str) -> bytes:
|
||||
limit = self.limit if hasattr(self, "limit") else len(self.data)
|
||||
if offset < 0 or length < 0 or offset > limit or length > limit - offset:
|
||||
raise Reject("EINTEGRITY", f"{point}.bounds")
|
||||
self.calls += 1
|
||||
self.bytes += length
|
||||
if length:
|
||||
self.blocks_read.update(range(offset // 4096, (offset + length - 1) // 4096 + 1))
|
||||
return self.data[offset:offset + length]
|
||||
|
||||
def reset_reads(self) -> None:
|
||||
self.calls = 0
|
||||
self.bytes = 0
|
||||
self.blocks_read.clear()
|
||||
|
||||
def inode(self, nid: int) -> dict[str, int]:
|
||||
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
|
||||
raw = self.read(offset, 64, "inode")
|
||||
inode_format, xattr_count = struct.unpack_from("<HH", raw)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
|
||||
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", raw, 8)[0]
|
||||
return {
|
||||
"offset": offset,
|
||||
"inode_size": inode_size,
|
||||
"xattr_size": xattr_size,
|
||||
"layout": (inode_format >> 1) & 7,
|
||||
"size": size,
|
||||
"start_block": struct.unpack_from("<I", raw, 16)[0],
|
||||
}
|
||||
|
||||
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
|
||||
if inode["layout"] == 2:
|
||||
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
|
||||
elif inode["layout"] == 0:
|
||||
physical = (inode["start_block"] << self.block_bits) + logical
|
||||
else:
|
||||
raise Reject("EINTEGRITY", "directory.layout")
|
||||
return self.read(physical, length, "directory.data")
|
||||
|
||||
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
|
||||
entries = []
|
||||
logical = 0
|
||||
while logical < inode["size"]:
|
||||
length = min(self.block_size, inode["size"] - logical)
|
||||
data = self.inode_data(inode, logical, length)
|
||||
if len(data) < 12:
|
||||
raise Reject("EINTEGRITY", "directory.header")
|
||||
first_name = struct.unpack_from("<H", data, 8)[0]
|
||||
if first_name == 0 or first_name % 12 or first_name > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-offset")
|
||||
count = first_name // 12
|
||||
for index in range(count):
|
||||
offset = index * 12
|
||||
nid = struct.unpack_from("<Q", data, offset)[0]
|
||||
start = struct.unpack_from("<H", data, offset + 8)[0]
|
||||
end = struct.unpack_from("<H", data, offset + 20)[0] if index + 1 < count else len(data)
|
||||
if start > end or end > len(data):
|
||||
raise Reject("EINTEGRITY", "directory.name-bounds")
|
||||
entries.append((data[start:end].split(b"\0", 1)[0], nid))
|
||||
logical += length
|
||||
return entries
|
||||
|
||||
def resolve(self, path: str) -> dict[str, int]:
|
||||
inode = self.inode(self.root_nid)
|
||||
for component in path.strip("/").encode().split(b"/"):
|
||||
matches = [nid for name, nid in self.directory_entries(inode) if name == component]
|
||||
if len(matches) != 1:
|
||||
raise Reject("ENOATTR", "path.lookup")
|
||||
inode = self.inode(matches[0])
|
||||
return inode
|
||||
|
||||
def stats(self, scanned: bool) -> dict[str, int | bool]:
|
||||
return {
|
||||
"bytes": self.bytes,
|
||||
"calls": self.calls,
|
||||
"provider_blocks": len(self.blocks_read),
|
||||
"scanned": scanned,
|
||||
}
|
||||
|
||||
def lookup(self, inode: dict[str, int], name: bytes, candidate: bool) -> tuple[str, bytes | None, dict]:
|
||||
self.reset_reads()
|
||||
body_offset = inode["offset"] + inode["inode_size"]
|
||||
usable = self.feature_compat & self.spec["format"]["feature_compat"] and self.filter_reserved == 0
|
||||
if candidate and usable:
|
||||
if inode["xattr_size"] < 12:
|
||||
raise Reject("EINTEGRITY", "ibody.header")
|
||||
if inode["xattr_size"] == 12:
|
||||
raise Reject("EOPNOTSUPP", "ibody.header-only")
|
||||
header = self.read(body_offset, 12, "ibody.header")
|
||||
shared_count = header[4]
|
||||
if 12 + shared_count * 4 > inode["xattr_size"]:
|
||||
raise Reject("EINTEGRITY", "ibody.shared-count")
|
||||
name_filter = struct.unpack_from("<I", header)[0]
|
||||
bit = xxh32(name, self.spec["format"]["seed"] + 1) & 31
|
||||
if name_filter & (1 << bit):
|
||||
return "ENOATTR", None, self.stats(False)
|
||||
body = self.read(body_offset, inode["xattr_size"], "ibody")
|
||||
if len(body) < 12:
|
||||
raise Reject("EINTEGRITY", "ibody.header")
|
||||
if len(body) == 12:
|
||||
raise Reject("EOPNOTSUPP", "ibody.header-only")
|
||||
shared_count = body[4]
|
||||
header_size = 12 + shared_count * 4
|
||||
if header_size > len(body):
|
||||
raise Reject("EINTEGRITY", "ibody.shared-count")
|
||||
cursor = header_size
|
||||
while cursor < len(body):
|
||||
if cursor + 4 > len(body):
|
||||
raise Reject("EINTEGRITY", "inline.header")
|
||||
name_length, name_index, value_length = struct.unpack_from("<BBH", body, cursor)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
if total > len(body) - cursor:
|
||||
raise Reject("EINTEGRITY", "inline.bounds")
|
||||
actual = body[cursor + 4:cursor + 4 + name_length]
|
||||
if b"\0" in actual:
|
||||
raise Reject("EINTEGRITY", "inline.name-nul")
|
||||
if name_index == 1 and actual == name:
|
||||
start = cursor + 4 + name_length
|
||||
return "PASS", body[start:start + value_length], self.stats(True)
|
||||
cursor += total
|
||||
for index in range(shared_count):
|
||||
shared_id = struct.unpack_from("<I", body, 12 + index * 4)[0]
|
||||
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
|
||||
header = self.read(offset, 4, "shared.header")
|
||||
name_length, name_index, value_length = struct.unpack("<BBH", header)
|
||||
total = (4 + name_length + value_length + 3) & ~3
|
||||
raw = self.read(offset, total, "shared.entry")
|
||||
actual = raw[4:4 + name_length]
|
||||
if b"\0" in actual:
|
||||
raise Reject("EINTEGRITY", "shared.name-nul")
|
||||
if name_index == 1 and actual == name:
|
||||
return "PASS", raw[4 + name_length:4 + name_length + value_length], self.stats(True)
|
||||
return "ENOATTR", None, self.stats(True)
|
||||
|
||||
|
||||
def lookup(path: Path, name: bytes, candidate: bool, spec: dict) -> dict:
|
||||
try:
|
||||
reader = Reader(path, spec)
|
||||
inode = reader.resolve(spec["fixture"]["target"])
|
||||
status, value, reads = reader.lookup(inode, name, candidate)
|
||||
return {"errno": 0 if status == "PASS" else spec["errno"][status], "reads": reads, "status": status, "value_hex": None if value is None else value.hex()}
|
||||
except Reject as error:
|
||||
return {"errno": spec["errno"].get(error.errno_name, -1), "point": error.point, "status": error.errno_name, "value_hex": None}
|
||||
|
||||
|
||||
def function(source: str, name: str) -> str:
|
||||
match = re.search(rf"\n{name}\([^;]*?\n\{{", source, re.DOTALL)
|
||||
if match is None:
|
||||
raise SystemExit(f"missing source function: {name}")
|
||||
start = match.start() + 1
|
||||
brace = source.index("{", match.start())
|
||||
depth = 0
|
||||
for index in range(brace, len(source)):
|
||||
if source[index] == "{":
|
||||
depth += 1
|
||||
elif source[index] == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start:index + 1]
|
||||
raise SystemExit(f"unterminated source function: {name}")
|
||||
|
||||
|
||||
def audit_source(root: Path, dut: Path, spec: dict) -> dict:
|
||||
header = (dut / "src/erofs_fs.h").read_text(encoding="utf-8")
|
||||
internal = (dut / "src/internal.h").read_text(encoding="utf-8")
|
||||
xattr = (dut / "src/xattr.c").read_text(encoding="utf-8")
|
||||
linux_header = (root / "src-linux/erofs_fs.h").read_text(encoding="utf-8")
|
||||
linux_xattr = (root / "src-linux/xattr.c").read_text(encoding="utf-8")
|
||||
for marker in (
|
||||
"#define EROFS_XATTR_FILTER_BITS",
|
||||
"#define EROFS_XATTR_FILTER_DEFAULT",
|
||||
"#define EROFS_XATTR_FILTER_SEED",
|
||||
):
|
||||
if marker not in header or marker not in linux_header:
|
||||
raise SystemExit(f"Bloom format marker missing: {marker}")
|
||||
if "erofs_sb_has_xattr_filter_v1" not in internal or "xattr_filter_reserved == 0" not in internal:
|
||||
raise SystemExit("FreeBSD unknown-filter fallback helper is incomplete")
|
||||
hash_body = function(xattr, "erofs_xxh32")
|
||||
filter_name = function(xattr, "erofs_xattr_filter_name")
|
||||
filter_negative = function(xattr, "erofs_xattr_filter_negative")
|
||||
getxattr = function(xattr, "erofs_getxattr")
|
||||
if "static uint32_t\nerofs_xxh32" not in xattr:
|
||||
raise SystemExit("xxh32 is not file-local and namespaced")
|
||||
for marker in ("le32dec(cursor)", "UINT32_C(2654435761)", "hash ^= hash >> 16"):
|
||||
if marker not in hash_body:
|
||||
raise SystemExit(f"xxh32 source marker missing: {marker}")
|
||||
for marker in (
|
||||
"EXTATTR_NAMESPACE_USER",
|
||||
"EXTATTR_NAMESPACE_SYSTEM",
|
||||
"EROFS_XATTR_INDEX_USER",
|
||||
"EROFS_XATTR_INDEX_POSIX_ACL_ACCESS",
|
||||
"EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT",
|
||||
"EROFS_XATTR_INDEX_TRUSTED",
|
||||
"EROFS_XATTR_INDEX_SECURITY",
|
||||
):
|
||||
if marker not in filter_name:
|
||||
raise SystemExit(f"FreeBSD namespace mapping marker missing: {marker}")
|
||||
for marker in (
|
||||
"erofs_sb_has_xattr_filter_v1(sbi)",
|
||||
"header_size > vi->xattr_isize",
|
||||
"erofs_xxh32(filter_name, filter_name_len",
|
||||
"erofs_put_metabuf(&buf)",
|
||||
):
|
||||
if marker not in filter_negative:
|
||||
raise SystemExit(f"fast-negative integrity marker missing: {marker}")
|
||||
if any(marker in filter_negative.lower() for marker in ("malloc", "mtx_", "cv_", "cache")):
|
||||
raise SystemExit("B19b introduced cache/allocation/locking into the fast-negative")
|
||||
if getxattr.index("erofs_xattr_filter_negative") > getxattr.index("erofs_xattr_load_body"):
|
||||
raise SystemExit("fast-negative runs after the full xattr body read")
|
||||
if "if (filter_negative)\n\t\treturn (ENOATTR);" not in getxattr:
|
||||
raise SystemExit("getxattr does not restrict the shortcut to proven negatives")
|
||||
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", header + internal + xattr):
|
||||
raise SystemExit("Linux negative errno entered the FreeBSD implementation")
|
||||
for marker in (
|
||||
"xxh32(name, strlen(name)",
|
||||
"EROFS_XATTR_FILTER_SEED + index",
|
||||
"vi->xattr_name_filter & (1U << hashbit)",
|
||||
"!sbi->xattr_filter_reserved",
|
||||
):
|
||||
if marker not in linux_xattr:
|
||||
raise SystemExit(f"Linux comparison anchor changed: {marker}")
|
||||
changed = subprocess.run(
|
||||
["git", "-C", str(root), "diff", "--name-only", spec["gate"]["commit"], "--", "repo-pre-15/src"],
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout.splitlines()
|
||||
expected = ["repo-pre-15/src/erofs_fs.h", "repo-pre-15/src/internal.h", "repo-pre-15/src/xattr.c"]
|
||||
if sorted(changed) != expected:
|
||||
raise SystemExit(f"B19b source write set differs: {changed}")
|
||||
return {
|
||||
"freebsd_errno": "positive",
|
||||
"freebsd_namespace": "extattr user/system mapping retained",
|
||||
"hash_symbol": "file-local erofs_xxh32",
|
||||
"linux_semantics": "xxh32(name suffix, seed + index), inverse 32-bit filter",
|
||||
"source_write_set": expected,
|
||||
"status": "PASS",
|
||||
}
|
||||
|
||||
|
||||
def verify(fixtures: Path, spec: dict, root: Path, dut: Path) -> dict:
|
||||
hashes = {path.name: sha256(path) for path in sorted(fixtures.glob("*.erofs"))}
|
||||
if hashes != spec["fixture_sha256"]:
|
||||
raise SystemExit(f"fixture hashes differ: {hashes}")
|
||||
cases = [
|
||||
("hit", "valid.erofs", b"attr00", "PASS", True),
|
||||
("miss", "valid.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", False),
|
||||
("false-positive", "valid.erofs", spec["fixture"]["collision"].encode(), "ENOATTR", True),
|
||||
("unknown-filter", "unknown-filter.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True),
|
||||
("feature-off", "feature-off.erofs", spec["fixture"]["miss"].encode(), "ENOATTR", True),
|
||||
("corrupt-shared-count", "corrupt-shared-count.erofs", spec["fixture"]["miss"].encode(), "EINTEGRITY", None),
|
||||
("corrupt-shared-id", "corrupt-shared-id.erofs", b"attr00", "EINTEGRITY", None),
|
||||
]
|
||||
results = []
|
||||
for identifier, image, name, expected_status, expected_scan in cases:
|
||||
baseline = lookup(fixtures / image, name, False, spec)
|
||||
candidate = lookup(fixtures / image, name, True, spec)
|
||||
if baseline["status"] != expected_status or candidate["status"] != expected_status:
|
||||
raise SystemExit(f"{identifier} status differs: {baseline}, {candidate}")
|
||||
if baseline["errno"] < 0 or candidate["errno"] < 0:
|
||||
raise SystemExit(f"{identifier} returned negative errno")
|
||||
if expected_scan is not None and candidate["reads"]["scanned"] != expected_scan:
|
||||
raise SystemExit(f"{identifier} scan decision differs")
|
||||
results.append({"baseline": baseline, "candidate": candidate, "id": identifier, "name": name.decode()})
|
||||
baseline_calls = []
|
||||
candidate_calls = []
|
||||
baseline_blocks = []
|
||||
candidate_blocks = []
|
||||
for _ in range(spec["thresholds"]["samples"]):
|
||||
baseline = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), False, spec)["reads"]
|
||||
candidate = lookup(fixtures / "valid.erofs", spec["fixture"]["miss"].encode(), True, spec)["reads"]
|
||||
baseline_calls.append(baseline["calls"])
|
||||
candidate_calls.append(candidate["calls"])
|
||||
baseline_blocks.append(baseline["provider_blocks"])
|
||||
candidate_blocks.append(candidate["provider_blocks"])
|
||||
call_reduction = 100 * (statistics.median(baseline_calls) - statistics.median(candidate_calls)) / statistics.median(baseline_calls)
|
||||
block_reduction = 100 * (statistics.median(baseline_blocks) - statistics.median(candidate_blocks)) / statistics.median(baseline_blocks)
|
||||
threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"]
|
||||
if call_reduction < threshold or block_reduction < threshold:
|
||||
raise SystemExit("B19b source model no longer meets the measured benefit gate")
|
||||
return {
|
||||
"benchmark": {
|
||||
"baseline_calls": baseline_calls,
|
||||
"baseline_provider_blocks": baseline_blocks,
|
||||
"call_reduction_percent": call_reduction,
|
||||
"candidate_calls": candidate_calls,
|
||||
"candidate_provider_blocks": candidate_blocks,
|
||||
"provider_block_reduction_percent": block_reduction,
|
||||
"threshold_percent": threshold,
|
||||
},
|
||||
"cases": results,
|
||||
"fixture_sha256": hashes,
|
||||
"source": audit_source(root, dut, spec),
|
||||
"status": "PASS",
|
||||
"test": spec["test"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dut", type=Path, required=True)
|
||||
parser.add_argument("--fixtures", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B19b":
|
||||
raise SystemExit("invalid B19b oracle spec")
|
||||
report = verify(args.fixtures, spec, args.root, args.dut)
|
||||
write_json(args.report, report)
|
||||
print(json.dumps({"case_count": len(report["cases"]), "status": "PASS", "test": report["test"]}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,207 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/extattr.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define VALUE_SIZE 96
|
||||
|
||||
static int
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "%s\n", message);
|
||||
return (1);
|
||||
}
|
||||
|
||||
static int
|
||||
parse_long(const char *text, long minimum, long maximum, long *valuep)
|
||||
{
|
||||
char *end;
|
||||
long value;
|
||||
|
||||
errno = 0;
|
||||
value = strtol(text, &end, 10);
|
||||
if (errno != 0 || text[0] == '\0' || end[0] != '\0' ||
|
||||
value < minimum || value > maximum)
|
||||
return (-1);
|
||||
*valuep = value;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
namespace_id(const char *name)
|
||||
{
|
||||
if (strcmp(name, "user") == 0)
|
||||
return (EXTATTR_NAMESPACE_USER);
|
||||
if (strcmp(name, "system") == 0)
|
||||
return (EXTATTR_NAMESPACE_SYSTEM);
|
||||
return (-1);
|
||||
}
|
||||
|
||||
static void
|
||||
expected_value(unsigned int index, uint8_t *value)
|
||||
{
|
||||
char prefix[16];
|
||||
size_t length;
|
||||
|
||||
(void)snprintf(prefix, sizeof(prefix), "value-%02u-", index);
|
||||
length = strlen(prefix);
|
||||
for (size_t offset = 0; offset < VALUE_SIZE; offset++)
|
||||
value[offset] = prefix[offset % length];
|
||||
}
|
||||
|
||||
static int
|
||||
get_hit(const char *path, const char *name, unsigned int index)
|
||||
{
|
||||
uint8_t actual[VALUE_SIZE + 16], expected[VALUE_SIZE];
|
||||
ssize_t length;
|
||||
|
||||
memset(actual, 0xa5, sizeof(actual));
|
||||
expected_value(index, expected);
|
||||
errno = 0;
|
||||
length = extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, actual,
|
||||
sizeof(actual));
|
||||
if (length != VALUE_SIZE || errno != 0)
|
||||
return (fail("extattr hit returned the wrong length or errno"));
|
||||
if (memcmp(actual, expected, sizeof(expected)) != 0)
|
||||
return (fail("extattr hit returned the wrong value"));
|
||||
for (size_t offset = VALUE_SIZE; offset < sizeof(actual); offset++) {
|
||||
if (actual[offset] != 0xa5)
|
||||
return (fail("extattr hit wrote beyond the value"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
get_errno(const char *path, int attrnamespace, const char *name,
|
||||
int expected_errno)
|
||||
{
|
||||
uint8_t actual[VALUE_SIZE + 16];
|
||||
ssize_t length;
|
||||
|
||||
memset(actual, 0xa5, sizeof(actual));
|
||||
errno = 0;
|
||||
length = extattr_get_file(path, attrnamespace, name, actual,
|
||||
sizeof(actual));
|
||||
if (length != -1 || errno != expected_errno)
|
||||
return (fail("extattr failure returned the wrong positive errno"));
|
||||
for (size_t offset = 0; offset < sizeof(actual); offset++) {
|
||||
if (actual[offset] != 0xa5)
|
||||
return (fail("failed extattr lookup modified the output"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
list_user(const char *path)
|
||||
{
|
||||
uint8_t *buffer;
|
||||
ssize_t length;
|
||||
size_t offset;
|
||||
unsigned int seen;
|
||||
|
||||
errno = 0;
|
||||
length = extattr_list_file(path, EXTATTR_NAMESPACE_USER, NULL, 0);
|
||||
if (length <= 0 || errno != 0)
|
||||
return (fail("extattr list sizing failed"));
|
||||
buffer = malloc((size_t)length);
|
||||
if (buffer == NULL)
|
||||
return (fail("extattr list allocation failed"));
|
||||
if (extattr_list_file(path, EXTATTR_NAMESPACE_USER, buffer,
|
||||
(size_t)length) != length) {
|
||||
free(buffer);
|
||||
return (fail("extattr list transfer failed"));
|
||||
}
|
||||
seen = 0;
|
||||
for (offset = 0; offset < (size_t)length;) {
|
||||
char expected[16];
|
||||
uint8_t name_length;
|
||||
unsigned int index;
|
||||
|
||||
name_length = buffer[offset++];
|
||||
if (name_length == 0 || name_length > (size_t)length - offset) {
|
||||
free(buffer);
|
||||
return (fail("extattr list record is malformed"));
|
||||
}
|
||||
for (index = 0; index < 8; index++) {
|
||||
(void)snprintf(expected, sizeof(expected), "attr%02u", index);
|
||||
if (strlen(expected) == name_length &&
|
||||
memcmp(buffer + offset, expected, name_length) == 0)
|
||||
break;
|
||||
}
|
||||
if (index == 8 || (seen & (1U << index)) != 0) {
|
||||
free(buffer);
|
||||
return (fail("extattr list returned an unknown or duplicate name"));
|
||||
}
|
||||
seen |= 1U << index;
|
||||
offset += name_length;
|
||||
}
|
||||
free(buffer);
|
||||
if (seen != 0xff)
|
||||
return (fail("extattr list omitted a user attribute"));
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
concurrent(const char *path, const char *hit, const char *miss,
|
||||
const char *collision, long workers, long loops)
|
||||
{
|
||||
long worker;
|
||||
int status;
|
||||
pid_t child;
|
||||
|
||||
for (worker = 0; worker < workers; worker++) {
|
||||
child = fork();
|
||||
if (child < 0)
|
||||
return (fail("fork failed"));
|
||||
if (child == 0) {
|
||||
for (long iteration = 0; iteration < loops; iteration++) {
|
||||
if (get_hit(path, hit, 0) != 0 ||
|
||||
get_errno(path, EXTATTR_NAMESPACE_USER, miss, ENOATTR) != 0 ||
|
||||
get_errno(path, EXTATTR_NAMESPACE_USER, collision, ENOATTR) != 0)
|
||||
_exit(1);
|
||||
}
|
||||
_exit(0);
|
||||
}
|
||||
}
|
||||
for (worker = 0; worker < workers; worker++) {
|
||||
if (wait(&status) < 0 || !WIFEXITED(status) ||
|
||||
WEXITSTATUS(status) != 0)
|
||||
return (fail("concurrent extattr worker failed"));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
long first, second;
|
||||
int attrnamespace;
|
||||
|
||||
if (argc == 5 && strcmp(argv[1], "hit") == 0) {
|
||||
if (parse_long(argv[4], 0, 7, &first) != 0)
|
||||
return (fail("invalid hit index"));
|
||||
return (get_hit(argv[2], argv[3], (unsigned int)first));
|
||||
}
|
||||
if (argc == 6 && strcmp(argv[1], "errno") == 0) {
|
||||
attrnamespace = namespace_id(argv[3]);
|
||||
if (attrnamespace < 0 || parse_long(argv[5], 1, 255, &first) != 0)
|
||||
return (fail("invalid errno arguments"));
|
||||
return (get_errno(argv[2], attrnamespace, argv[4], (int)first));
|
||||
}
|
||||
if (argc == 3 && strcmp(argv[1], "list") == 0)
|
||||
return (list_user(argv[2]));
|
||||
if (argc == 8 && strcmp(argv[1], "concurrent") == 0) {
|
||||
if (parse_long(argv[6], 1, 64, &first) != 0 ||
|
||||
parse_long(argv[7], 1, 1000, &second) != 0)
|
||||
return (fail("invalid concurrency arguments"));
|
||||
return (concurrent(argv[2], argv[3], argv[4], argv[5], first,
|
||||
second));
|
||||
}
|
||||
return (fail("usage: B19b-xattr-probe MODE PATH ARG..."));
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"batch": "B19b",
|
||||
"candidate": "P15-021",
|
||||
"concurrency": {
|
||||
"loops_per_worker": 10,
|
||||
"workers": 64
|
||||
},
|
||||
"errno": {
|
||||
"EINTEGRITY": 97,
|
||||
"ENOATTR": 87
|
||||
},
|
||||
"fixture": {
|
||||
"attribute_count": 8,
|
||||
"attribute_value_bytes": 96,
|
||||
"collision": "absent-00000",
|
||||
"miss": "absent-00001",
|
||||
"peer_count": 64,
|
||||
"shared_count": 8,
|
||||
"target": "/target.bin",
|
||||
"uuid": "00000000-0000-0000-0000-000000000021"
|
||||
},
|
||||
"fixture_sha256": {
|
||||
"corrupt-shared-count.erofs": "c1f061f1a7f60c4e347b70cb17f7dac01b9c4e98904aa9da038cc12dad26f15d",
|
||||
"corrupt-shared-id.erofs": "1ea7b408d8b725352970d14c7b73146055db26bc15ac5fb9ca8a9be05bb22314",
|
||||
"feature-off.erofs": "2c802394381dbab9d30e50486975e5c1f40688f8b3a21ffeac26edc8ab09ce85",
|
||||
"unknown-filter.erofs": "4333632363689cb16a715c4204d8fd9d2a5cf5cc2ace77828d03edcb391e4fe5",
|
||||
"valid.erofs": "b06b7c4c30adb3684d11505edf815dc0a395c03f6a95120b3150361008a182cb"
|
||||
},
|
||||
"format": {
|
||||
"bits": 32,
|
||||
"feature_compat": 4,
|
||||
"seed": 633069711
|
||||
},
|
||||
"gate": {
|
||||
"commit": "675ed9b650b3bc157b38bcc165a0cb215a2aa989",
|
||||
"decision_base": "666e52f710363df07f7c93919eb835d41092d011",
|
||||
"input_sha256": "2c39d55e5d9db278beb638a40188b60d343617be6c1e7c30a658ae92fe12d9bd",
|
||||
"script_sha256": "376200a950ec09ad8df29f1e76b0595533911e6e253201aa31b351bdef009947"
|
||||
},
|
||||
"schema": 1,
|
||||
"test": "TC169-xattr-filter",
|
||||
"thresholds": {
|
||||
"minimum_provider_metadata_read_reduction_percent": 25.0,
|
||||
"samples": 5
|
||||
}
|
||||
}
|
||||
Executable
+258
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
evidence=${1:?evidence directory is required}
|
||||
root=/work/erofs-freebsd-pre
|
||||
dut=$root/repo-pre-15
|
||||
base=/work/build/vm-freebsd-build.qcow2.bp
|
||||
askpass=/work/build/.repo22-ssh-askpass
|
||||
module=${B21_MODULE:-$root/planning/pre15/evidence/20260816T-B21-abi15-build/B21-erofs-zstdio0.ko}
|
||||
source_tree=${B21_FREEBSD_SRC:-/work/dev-freebsd-releng}
|
||||
owned_pid=
|
||||
port=
|
||||
loaded=0
|
||||
mounted=0
|
||||
md_units=
|
||||
|
||||
test ! -e "$evidence"
|
||||
mkdir -p "$evidence"
|
||||
: >"$evidence/cleanup.log"
|
||||
|
||||
ssh_args=(env DISPLAY=:0 SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force ssh
|
||||
-n
|
||||
-o BatchMode=no -o PubkeyAuthentication=no
|
||||
-o PreferredAuthentications=keyboard-interactive,password
|
||||
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||||
-o LogLevel=ERROR -o ConnectTimeout=5)
|
||||
scp_args=(env DISPLAY=:0 SSH_ASKPASS="$askpass" SSH_ASKPASS_REQUIRE=force scp
|
||||
-O -q -o BatchMode=no -o PubkeyAuthentication=no
|
||||
-o PreferredAuthentications=keyboard-interactive,password
|
||||
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||||
-o LogLevel=ERROR -o ConnectTimeout=5)
|
||||
|
||||
guest()
|
||||
{
|
||||
timeout -k 5 60 "${ssh_args[@]}" -p "$port" [email protected] "$@"
|
||||
}
|
||||
|
||||
capture()
|
||||
{
|
||||
local name=$1
|
||||
shift
|
||||
set +e
|
||||
guest "$@" >"$evidence/$name.stdout" 2>"$evidence/$name.stderr"
|
||||
local rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$rc" >"$evidence/$name.rc"
|
||||
}
|
||||
|
||||
scp_to()
|
||||
{
|
||||
local source_path=$1
|
||||
local destination_path=$2
|
||||
timeout -k 5 60 "${scp_args[@]}" -P "$port" "$source_path" \
|
||||
"[email protected]:$destination_path"
|
||||
}
|
||||
|
||||
cleanup()
|
||||
{
|
||||
set +e
|
||||
local cleanup_rc=0
|
||||
if test "$mounted" = 1; then guest umount /mnt/pre15-b21 >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; fi
|
||||
for md in $md_units; do guest mdconfig -d -u "$md" >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; done
|
||||
if test "$loaded" = 1; then guest kldunload /root/B21-erofs-zstdio0.ko >>"$evidence/cleanup.log" 2>&1 || cleanup_rc=1; fi
|
||||
if test -n "$owned_pid" && kill -0 "$owned_pid" 2>/dev/null; then
|
||||
test "$owned_pid" != 26318 || exit 99
|
||||
kill "$owned_pid" 2>/dev/null
|
||||
wait "$owned_pid" 2>/dev/null
|
||||
fi
|
||||
test ! -e "$evidence/owned-overlay.qcow2" || unlink "$evidence/owned-overlay.qcow2"
|
||||
python3 - "$port" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
if not sys.argv[1]:
|
||||
raise SystemExit(0)
|
||||
with socket.socket() as sock:
|
||||
sock.settimeout(0.2)
|
||||
raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) else 1)
|
||||
PY
|
||||
test $? = 0 || cleanup_rc=1
|
||||
stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \
|
||||
>"$evidence/base-metadata-after.txt"
|
||||
cmp -s "$evidence/base-metadata-before.txt" "$evidence/base-metadata-after.txt" || cleanup_rc=1
|
||||
if test "$cleanup_rc" = 0 && test ! -e "$evidence/owned-overlay.qcow2"; then
|
||||
printf '%s\n' PASS >"$evidence/cleanup.status"
|
||||
else
|
||||
printf '%s\n' FAIL >"$evidence/cleanup.status"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
test -x "$askpass"
|
||||
test -f "$module"
|
||||
test -f "$base"
|
||||
stat -c 'path=%n size=%s inode=%i mode=%f mtime=%Y ctime=%Z' "$base" \
|
||||
>"$evidence/base-metadata-before.txt"
|
||||
git -C "$dut" rev-parse HEAD >"$evidence/dut-head.txt"
|
||||
{
|
||||
git -C "$source_tree" log -1 --format='%H %cs %s'
|
||||
sed -n '76,77p' "$source_tree/sys/sys/param.h"
|
||||
sed -n '53,54p' "$source_tree/sys/conf/newvers.sh"
|
||||
} >"$evidence/source-identity.txt"
|
||||
printf '%s\n' 'SSH options precede [email protected] and the remote command' \
|
||||
>"$evidence/transport-contract.txt"
|
||||
|
||||
python3 -B "$dut/tests/pre15/fixtures/B21-super-generate.py" \
|
||||
--spec "$dut/tests/pre15/fixtures/B21-super-spec.json" \
|
||||
--output "$evidence/first" --work "$evidence/first-work" \
|
||||
>"$evidence/generate-first.stdout" 2>"$evidence/generate-first.stderr"
|
||||
python3 -B "$dut/tests/pre15/fixtures/B21-super-generate.py" \
|
||||
--spec "$dut/tests/pre15/fixtures/B21-super-spec.json" \
|
||||
--output "$evidence/second" --work "$evidence/second-work" \
|
||||
>"$evidence/generate-second.stdout" 2>"$evidence/generate-second.stderr"
|
||||
cmp "$evidence/first/fixture-index.json" "$evidence/second/fixture-index.json"
|
||||
python3 -B "$dut/tests/pre15/fixtures/B21-super-oracle.py" \
|
||||
--fixtures "$evidence/first" --report "$evidence/oracle.json" \
|
||||
>"$evidence/oracle.stdout" 2>"$evidence/oracle.stderr"
|
||||
|
||||
python3 - "$evidence/first/fixture-index.json" "$evidence/oracle.json" \
|
||||
"$evidence/matrix.tsv" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
index = json.loads(Path(sys.argv[1]).read_text())
|
||||
oracle = json.loads(Path(sys.argv[2]).read_text())
|
||||
assert index["case_count"] == 13 and oracle["passed_count"] == 13
|
||||
rows = ["id\texpected_errno\treject"]
|
||||
for fixture, result in zip(index["cases"], oracle["results"], strict=True):
|
||||
rows.append(
|
||||
f"{fixture['id']}\t{result['actual_errno']}\t"
|
||||
f"{fixture['expected_reject']}"
|
||||
)
|
||||
Path(sys.argv[3]).write_text("\n".join(rows) + "\n")
|
||||
PY
|
||||
tar -C "$evidence/first" -czf "$evidence/fixtures.tar.gz" .
|
||||
|
||||
qemu-img create -f qcow2 -F qcow2 -b "$base" \
|
||||
"$evidence/owned-overlay.qcow2" >"$evidence/qemu-img.stdout" \
|
||||
2>"$evidence/qemu-img.stderr"
|
||||
port=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
if port == 9222:
|
||||
raise SystemExit("protected port selected")
|
||||
print(port)
|
||||
PY
|
||||
)
|
||||
printf '%s\n' "$port" >"$evidence/owned-port.txt"
|
||||
qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 -m 6144 -smp 4 \
|
||||
-drive "file=$evidence/owned-overlay.qcow2,if=virtio,format=qcow2" \
|
||||
-netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \
|
||||
-device virtio-net-pci,netdev=net0 -display none \
|
||||
-serial "file:$evidence/serial.log" -monitor none \
|
||||
>"$evidence/qemu.stdout" 2>"$evidence/qemu.stderr" &
|
||||
owned_pid=$!
|
||||
printf '%s\n' "$owned_pid" >"$evidence/owned-pid.txt"
|
||||
test "$owned_pid" != 26318
|
||||
|
||||
ready=0
|
||||
for attempt in $(seq 1 180); do
|
||||
if ! kill -0 "$owned_pid" 2>/dev/null; then exit 21; fi
|
||||
set +e
|
||||
guest true >"$evidence/readiness.stdout" 2>"$evidence/readiness.stderr"
|
||||
readiness_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$readiness_rc" >"$evidence/readiness.rc"
|
||||
if test "$readiness_rc" = 0; then ready=1; break; fi
|
||||
sleep 1
|
||||
done
|
||||
printf 'attempts=%s ready=%s\n' "$attempt" "$ready" >"$evidence/readiness.txt"
|
||||
test "$ready" = 1
|
||||
|
||||
capture guest-uname uname -a
|
||||
capture guest-freebsd-version freebsd-version -kru
|
||||
capture guest-osreldate sysctl -n kern.osreldate
|
||||
capture guest-kldstat-before kldstat
|
||||
capture guest-kldstat-module-before kldstat -q -m erofs
|
||||
capture guest-dmesg-before dmesg
|
||||
test "$(cat "$evidence/guest-kldstat-module-before.rc")" = 1
|
||||
|
||||
scp_to "$module" /root/B21-erofs-zstdio0.ko \
|
||||
>"$evidence/upload-module.stdout" 2>"$evidence/upload-module.stderr"
|
||||
scp_to "$dut/tests/pre15/fixtures/B21-qemu-probe.c" /root/B21-qemu-probe.c \
|
||||
>"$evidence/upload-probe.stdout" 2>"$evidence/upload-probe.stderr"
|
||||
scp_to "$evidence/fixtures.tar.gz" /root/B21-fixtures.tar.gz \
|
||||
>"$evidence/upload-fixtures.stdout" 2>"$evidence/upload-fixtures.stderr"
|
||||
capture guest-module-file file /root/B21-erofs-zstdio0.ko
|
||||
capture guest-module-sha256 sha256 -q /root/B21-erofs-zstdio0.ko
|
||||
capture guest-prepare sh -c \
|
||||
'rm -rf /root/B21-fixtures /mnt/pre15-b21; mkdir /root/B21-fixtures /mnt/pre15-b21; tar -xzf /root/B21-fixtures.tar.gz -C /root/B21-fixtures; cc -O2 -Wall -Wextra -Werror -std=c17 -o /root/B21-qemu-probe /root/B21-qemu-probe.c'
|
||||
test "$(cat "$evidence/guest-prepare.rc")" = 0
|
||||
|
||||
capture B21-kldload kldload /root/B21-erofs-zstdio0.ko
|
||||
test "$(cat "$evidence/B21-kldload.rc")" = 0
|
||||
loaded=1
|
||||
capture guest-kldstat-after-load kldstat
|
||||
capture guest-kldstat-module-after-load kldstat -q -m erofs
|
||||
capture guest-dmesg-after-load dmesg
|
||||
test "$(cat "$evidence/guest-kldstat-module-after-load.rc")" = 0
|
||||
|
||||
printf 'id\texpected_errno\tactual_errno\treject\n' \
|
||||
>"$evidence/qemu-matrix.tsv"
|
||||
while IFS="$(printf '\t')" read -r id expected_errno reject; do
|
||||
test "$id" != id || continue
|
||||
set +e
|
||||
md=$(guest mdconfig -a -t vnode -f "/root/B21-fixtures/$id.erofs" \
|
||||
2>"$evidence/$id-md.stderr")
|
||||
md_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$md_rc" >"$evidence/$id-md.rc"
|
||||
test "$md_rc" = 0
|
||||
md_units="$md_units $md"
|
||||
set +e
|
||||
probe_result=$(guest /root/B21-qemu-probe "/dev/$md" /mnt/pre15-b21 \
|
||||
2>"$evidence/$id-probe.stderr")
|
||||
probe_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$probe_rc" >"$evidence/$id-probe.rc"
|
||||
printf '%s\n' "$probe_result" >"$evidence/$id-probe.stdout"
|
||||
actual_errno=$(printf '%s\n' "$probe_result" | \
|
||||
sed -n 's/.*errno=\([0-9][0-9]*\).*/\1/p')
|
||||
test -n "$actual_errno"
|
||||
printf '%s\t%s\t%s\t%s\n' "$id" "$expected_errno" "$actual_errno" \
|
||||
"$reject" >>"$evidence/qemu-matrix.tsv"
|
||||
test "$actual_errno" = "$expected_errno"
|
||||
if test "$actual_errno" = 0; then
|
||||
mounted=1
|
||||
guest find /mnt/pre15-b21 -mindepth 1 -maxdepth 1 -print \
|
||||
>"$evidence/$id-readdir.txt"
|
||||
guest umount /mnt/pre15-b21
|
||||
mounted=0
|
||||
fi
|
||||
guest mdconfig -d -u "$md"
|
||||
md_units=$(printf '%s\n' "$md_units" | sed "s/ $md//")
|
||||
done <"$evidence/matrix.tsv"
|
||||
|
||||
capture guest-dmesg-after dmesg
|
||||
capture guest-kldstat-after kldstat
|
||||
capture guest-kldstat-module-after kldstat -q -m erofs
|
||||
diff -u "$evidence/guest-dmesg-before.stdout" \
|
||||
"$evidence/guest-dmesg-after.stdout" >"$evidence/dmesg.diff" || true
|
||||
if grep -Eqi 'panic:|lock order reversal|witness.*warning|use-after-free' \
|
||||
"$evidence/dmesg.diff"; then
|
||||
exit 10
|
||||
fi
|
||||
guest kldunload /root/B21-erofs-zstdio0.ko >"$evidence/guest-kldunload.stdout" \
|
||||
2>"$evidence/guest-kldunload.stderr"
|
||||
loaded=0
|
||||
capture guest-kldstat-module-clean kldstat -q -m erofs
|
||||
capture guest-mount-clean mount -p
|
||||
capture guest-md-clean mdconfig -l
|
||||
test "$(cat "$evidence/guest-kldstat-module-clean.rc")" = 1
|
||||
test ! -s "$evidence/guest-md-clean.stdout"
|
||||
test -z "$(awk '$3 == "erofs" { print }' "$evidence/guest-mount-clean.stdout")"
|
||||
printf '%s\n' reached >"$evidence/target.marker"
|
||||
printf '%s\n' 'B21_PASS TC172-super-trust 13/13' >"$evidence/RESULT.txt"
|
||||
@@ -0,0 +1,35 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/mount.h>
|
||||
#include <sys/uio.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
struct iovec options[6];
|
||||
int result;
|
||||
|
||||
if (argc != 3) {
|
||||
fprintf(stderr, "usage: B21-qemu-probe DEVICE MOUNTPOINT\n");
|
||||
return (2);
|
||||
}
|
||||
options[0].iov_base = "fstype";
|
||||
options[0].iov_len = sizeof("fstype");
|
||||
options[1].iov_base = "erofs";
|
||||
options[1].iov_len = sizeof("erofs");
|
||||
options[2].iov_base = "fspath";
|
||||
options[2].iov_len = sizeof("fspath");
|
||||
options[3].iov_base = argv[2];
|
||||
options[3].iov_len = strlen(argv[2]) + 1;
|
||||
options[4].iov_base = "from";
|
||||
options[4].iov_len = sizeof("from");
|
||||
options[5].iov_base = argv[1];
|
||||
options[5].iov_len = strlen(argv[1]) + 1;
|
||||
errno = 0;
|
||||
result = nmount(options, sizeof(options) / sizeof(options[0]), MNT_RDONLY);
|
||||
printf("rc=%d errno=%d\n", result, result == -1 ? errno : 0);
|
||||
return (0);
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic B21 superblock trust-order fixtures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
||||
FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040
|
||||
UNKNOWN_INCOMPAT = 0x80000000
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray):
|
||||
self.data = bytearray(data)
|
||||
if len(self.data) < SUPER + 144:
|
||||
raise ValueError("image is shorter than the extended superblock")
|
||||
if self.u32(SUPER) != 0xE0F5E1E2:
|
||||
raise ValueError("image has the wrong EROFS magic")
|
||||
if not self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
|
||||
raise ValueError("image does not declare a superblock checksum")
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def clone(self) -> "Image":
|
||||
return Image(self.data)
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def put_u16(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<H", self.data, offset, value)
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
@property
|
||||
def block_bits(self) -> int:
|
||||
return self.data[SUPER + 12]
|
||||
|
||||
@property
|
||||
def block_size(self) -> int:
|
||||
return 1 << self.block_bits
|
||||
|
||||
@property
|
||||
def checksum_end(self) -> int:
|
||||
span = self.block_size
|
||||
if span > SUPER:
|
||||
span -= SUPER
|
||||
return SUPER + span
|
||||
|
||||
def checksum_valid(self) -> bool:
|
||||
expected = self.u32(SUPER + 4)
|
||||
canonical = bytearray(self.data[SUPER : self.checksum_end])
|
||||
struct.pack_into("<I", canonical, 4, 0)
|
||||
kernel = crc32c(self.data[SUPER + 8 : self.checksum_end], 0x5045B54A)
|
||||
return expected == crc32c(canonical) == kernel
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
self.put_u32(SUPER + 4, 0)
|
||||
checksum = crc32c(self.data[SUPER : self.checksum_end])
|
||||
self.put_u32(SUPER + 4, checksum)
|
||||
if not self.checksum_valid():
|
||||
raise AssertionError("updated B21 checksum does not verify")
|
||||
|
||||
def inode_offset(self, nid: int) -> int:
|
||||
return (self.u32(SUPER + 40) << self.block_bits) + (nid << 5)
|
||||
|
||||
def fix_packed_inode_mode(self) -> None:
|
||||
packed_nid = self.u64(SUPER + 96)
|
||||
if packed_nid == 0:
|
||||
raise ValueError("B21 seed did not materialize a packed prefix carrier")
|
||||
self.put_u16(self.inode_offset(packed_nid) + 4, 0o100644)
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B21":
|
||||
raise ValueError("invalid B21 fixture spec identity")
|
||||
if not isinstance(spec.get("cases"), list) or not spec["cases"]:
|
||||
raise ValueError("B21 fixture spec has no cases")
|
||||
return spec
|
||||
|
||||
|
||||
def create_source(root: Path) -> None:
|
||||
root.mkdir(parents=True)
|
||||
control = root / "control.txt"
|
||||
prefix = root / "prefix.bin"
|
||||
control.write_bytes(b"B21 superblock control\n")
|
||||
prefix.write_bytes(b"B21 prefix control\n")
|
||||
control.chmod(0o644)
|
||||
prefix.chmod(0o644)
|
||||
os.setxattr(prefix, b"user.company.branch.leaf", b"B21-prefix-value")
|
||||
for path in (control, prefix, root):
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def build_seed(spec: dict, output: Path, work: Path) -> dict[str, object]:
|
||||
mkfs = shutil.which("mkfs.erofs")
|
||||
if mkfs is None:
|
||||
raise SystemExit("mkfs.erofs is required for B21 fixtures")
|
||||
version_run = subprocess.run(
|
||||
[mkfs, "-V"], check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
version = version_run.stdout.splitlines()[0] if version_run.stdout else ""
|
||||
if version != spec["seed"]["mkfs_version"]:
|
||||
raise SystemExit(
|
||||
f"mkfs.erofs version differs: expected {spec['seed']['mkfs_version']!r}, "
|
||||
f"got {version!r}"
|
||||
)
|
||||
source = work / "source"
|
||||
create_source(source)
|
||||
command = [
|
||||
mkfs,
|
||||
"-d0",
|
||||
"-T0",
|
||||
"--all-time",
|
||||
"--all-root",
|
||||
"--workers=1",
|
||||
"--sort=path",
|
||||
f"-U{spec['seed']['uuid']}",
|
||||
"-x2",
|
||||
"-Eforce-inode-extended",
|
||||
"--xattr-prefix=user.company.",
|
||||
str(output),
|
||||
str(source),
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command, check=False, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
f"mkfs.erofs failed with {completed.returncode}:\n{completed.stdout}"
|
||||
)
|
||||
image = Image.load(output)
|
||||
image.fix_packed_inode_mode()
|
||||
image.update_checksum()
|
||||
output.write_bytes(image.data)
|
||||
if not image.u32(SUPER + 80) & FEATURE_INCOMPAT_XATTR_PREFIXES:
|
||||
raise ValueError("B21 seed does not declare xattr prefixes")
|
||||
return {"version": version}
|
||||
|
||||
|
||||
def mutate(image: Image, mutation: str) -> list[dict[str, int | str]]:
|
||||
changes: list[dict[str, int | str]] = []
|
||||
|
||||
def record(field: str, offset: int, size: int) -> None:
|
||||
changes.append({"field": field, "offset": offset, "size": size})
|
||||
|
||||
if mutation == "none":
|
||||
return changes
|
||||
if mutation in {"unknown-feature", "feature-prefix"}:
|
||||
offset = SUPER + 80
|
||||
image.put_u32(offset, image.u32(offset) | UNKNOWN_INCOMPAT)
|
||||
record("feature_incompat", offset, 4)
|
||||
if mutation == "blocks":
|
||||
offset = SUPER + 36
|
||||
image.put_u32(offset, 0xFFFFFFFF)
|
||||
record("blocks_lo", offset, 4)
|
||||
elif mutation == "extslots":
|
||||
offset = SUPER + 13
|
||||
image.data[offset] = 0xFF
|
||||
record("sb_extslots", offset, 1)
|
||||
elif mutation == "dirblk":
|
||||
offset = SUPER + 90
|
||||
image.data[offset] = 1
|
||||
record("dirblkbits", offset, 1)
|
||||
elif mutation in {"prefix-offset", "feature-prefix"}:
|
||||
offset = SUPER + 92
|
||||
image.put_u32(offset, 0xFFFFFFFF)
|
||||
record("xattr_prefix_start", offset, 4)
|
||||
elif mutation != "unknown-feature":
|
||||
raise ValueError(f"unknown B21 mutation: {mutation}")
|
||||
return changes
|
||||
|
||||
|
||||
def generate(spec_path: Path, output: Path, work: Path) -> int:
|
||||
spec = load_spec(spec_path)
|
||||
if output.exists() or work.exists():
|
||||
raise SystemExit("B21 output and work paths must not already exist")
|
||||
output.mkdir(parents=True)
|
||||
work.mkdir(parents=True)
|
||||
seed_path = work / "seed.erofs"
|
||||
build = build_seed(spec, seed_path, work / "seed-work")
|
||||
seed = Image.load(seed_path)
|
||||
seed_hash = sha256(seed_path)
|
||||
cases = []
|
||||
for item in spec["cases"]:
|
||||
image = seed.clone()
|
||||
changes = mutate(image, item["mutation"])
|
||||
if item["authenticated"] and item["mutation"] != "none":
|
||||
image.update_checksum()
|
||||
path = output / f"{item['id']}.erofs"
|
||||
path.write_bytes(image.data)
|
||||
checksum_valid = image.checksum_valid()
|
||||
if checksum_valid != item["authenticated"]:
|
||||
raise AssertionError(f"checksum state mismatch for {item['id']}")
|
||||
cases.append(
|
||||
{
|
||||
**item,
|
||||
"changes": changes,
|
||||
"checksum_valid": checksum_valid,
|
||||
"filename": path.name,
|
||||
"sha256": sha256(path),
|
||||
"size": path.stat().st_size,
|
||||
}
|
||||
)
|
||||
index = {
|
||||
"batch": "B21",
|
||||
"build": build,
|
||||
"case_count": len(cases),
|
||||
"cases": cases,
|
||||
"damaged_count": sum(item["class"] == "damaged" for item in cases),
|
||||
"legal_count": sum(item["class"] == "legal" for item in cases),
|
||||
"schema": 1,
|
||||
"seed_sha256": seed_hash,
|
||||
}
|
||||
write_json(output / "fixture-index.json", index)
|
||||
print(json.dumps({"status": "PASS", **index}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--work", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
return generate(args.spec, args.output, args.work)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent B21 superblock ordering oracle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
PAGE_SHIFT = 12
|
||||
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
||||
FEATURE_INCOMPAT_ALL = 0x000001FF
|
||||
FEATURE_INCOMPAT_48BIT = 0x00000080
|
||||
FEATURE_INCOMPAT_XATTR_PREFIXES = 0x00000040
|
||||
CRC32C_POLY = 0x82F63B78
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
|
||||
value = seed
|
||||
for byte in data:
|
||||
value ^= byte
|
||||
for _ in range(8):
|
||||
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
|
||||
return value & 0xFFFFFFFF
|
||||
|
||||
|
||||
def u16(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<H", data, offset)[0]
|
||||
|
||||
|
||||
def u32(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<I", data, offset)[0]
|
||||
|
||||
|
||||
def u64(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", data, offset)[0]
|
||||
|
||||
|
||||
def checksum_valid(data: bytes) -> bool:
|
||||
block_bits = data[SUPER + 12]
|
||||
if not 9 <= block_bits <= PAGE_SHIFT:
|
||||
return False
|
||||
span = 1 << block_bits
|
||||
if span > SUPER:
|
||||
span -= SUPER
|
||||
end = SUPER + span
|
||||
expected = u32(data, SUPER + 4)
|
||||
return expected == crc32c(data[SUPER + 8 : end], 0x5045B54A)
|
||||
|
||||
|
||||
def classify(data: bytes) -> tuple[int, str]:
|
||||
if u32(data, SUPER) != 0xE0F5E1E2:
|
||||
return 22, "magic"
|
||||
block_bits = data[SUPER + 12]
|
||||
if not 9 <= block_bits <= PAGE_SHIFT:
|
||||
return 22, "block-size"
|
||||
feature_compat = u32(data, SUPER + 8)
|
||||
if feature_compat & FEATURE_COMPAT_SB_CHKSUM and not checksum_valid(data):
|
||||
return 97, "checksum"
|
||||
if data[SUPER + 90] != 0:
|
||||
return 45, "dirblkbits"
|
||||
feature_incompat = u32(data, SUPER + 80)
|
||||
if feature_incompat & ~FEATURE_INCOMPAT_ALL:
|
||||
if u32(data, SUPER + 92) == 0xFFFFFFFF:
|
||||
return 45, "feature-before-prefix"
|
||||
return 45, "feature-incompat"
|
||||
if 128 + data[SUPER + 13] * 16 > (1 << PAGE_SHIFT) - SUPER:
|
||||
return 22, "super-extension"
|
||||
blocks = u32(data, SUPER + 36)
|
||||
root_nid_8b = u64(data, SUPER + 112)
|
||||
if feature_incompat & FEATURE_INCOMPAT_48BIT and root_nid_8b != 0:
|
||||
blocks |= u16(data, SUPER + 14) << 32
|
||||
if blocks == 0:
|
||||
return 97, "blocks-zero"
|
||||
if blocks << block_bits > len(data):
|
||||
return 6, "provider-size"
|
||||
if (
|
||||
feature_incompat & FEATURE_INCOMPAT_XATTR_PREFIXES
|
||||
and data[SUPER + 91] != 0
|
||||
and u32(data, SUPER + 92) == 0xFFFFFFFF
|
||||
):
|
||||
return 97, "prefix-offset"
|
||||
return 0, "accepted"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--fixtures", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
index = json.loads(
|
||||
(args.fixtures / "fixture-index.json").read_text(encoding="ascii")
|
||||
)
|
||||
results = []
|
||||
for item in index["cases"]:
|
||||
path = args.fixtures / item["filename"]
|
||||
actual_errno, actual_reject = classify(path.read_bytes())
|
||||
passed = (
|
||||
actual_errno == item["expected_errno"]
|
||||
and actual_reject == item["expected_reject"]
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"actual_errno": actual_errno,
|
||||
"actual_reject": actual_reject,
|
||||
"expected_errno": item["expected_errno"],
|
||||
"expected_reject": item["expected_reject"],
|
||||
"id": item["id"],
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
report = {
|
||||
"batch": "B21",
|
||||
"case_count": len(results),
|
||||
"passed_count": sum(item["passed"] for item in results),
|
||||
"results": results,
|
||||
"schema": 1,
|
||||
"status": "PASS" if all(item["passed"] for item in results) else "FAIL",
|
||||
}
|
||||
args.report.write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
for item in results:
|
||||
print(
|
||||
f"{'PASS' if item['passed'] else 'FAIL'} {item['id']} "
|
||||
f"errno={item['actual_errno']} reject={item['actual_reject']}"
|
||||
)
|
||||
return 0 if report["status"] == "PASS" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,125 @@
|
||||
{
|
||||
"batch": "B21",
|
||||
"candidates": [
|
||||
"P15-048",
|
||||
"P15-049",
|
||||
"P15-072"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "legal",
|
||||
"expected_errno": 0,
|
||||
"expected_reject": "accepted",
|
||||
"id": "legal-control",
|
||||
"mutation": "none"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "unknown-feature-unauthenticated",
|
||||
"mutation": "unknown-feature"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 45,
|
||||
"expected_reject": "feature-incompat",
|
||||
"id": "unknown-feature-authenticated",
|
||||
"mutation": "unknown-feature"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "blocks-unauthenticated",
|
||||
"mutation": "blocks"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 6,
|
||||
"expected_reject": "provider-size",
|
||||
"id": "blocks-authenticated",
|
||||
"mutation": "blocks"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "extslots-unauthenticated",
|
||||
"mutation": "extslots"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 22,
|
||||
"expected_reject": "super-extension",
|
||||
"id": "extslots-authenticated",
|
||||
"mutation": "extslots"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "dirblk-unauthenticated",
|
||||
"mutation": "dirblk"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 45,
|
||||
"expected_reject": "dirblkbits",
|
||||
"id": "dirblk-authenticated",
|
||||
"mutation": "dirblk"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "prefix-offset-unauthenticated",
|
||||
"mutation": "prefix-offset"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "prefix-offset",
|
||||
"id": "prefix-offset-authenticated",
|
||||
"mutation": "prefix-offset"
|
||||
},
|
||||
{
|
||||
"authenticated": false,
|
||||
"class": "damaged",
|
||||
"expected_errno": 97,
|
||||
"expected_reject": "checksum",
|
||||
"id": "feature-prefix-unauthenticated",
|
||||
"mutation": "feature-prefix"
|
||||
},
|
||||
{
|
||||
"authenticated": true,
|
||||
"class": "damaged",
|
||||
"expected_errno": 45,
|
||||
"expected_reject": "feature-before-prefix",
|
||||
"id": "feature-prefix-authenticated",
|
||||
"mutation": "feature-prefix"
|
||||
}
|
||||
],
|
||||
"freebsd_errno": {
|
||||
"EINVAL": 22,
|
||||
"EINTEGRITY": 97,
|
||||
"ENXIO": 6,
|
||||
"EOPNOTSUPP": 45
|
||||
},
|
||||
"schema": 1,
|
||||
"seed": {
|
||||
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
|
||||
"uuid": "00000000-0000-0000-0000-000000000321"
|
||||
}
|
||||
}
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B22 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B22 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
fi
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' 'B22 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -q ' ZSTD_'; then
|
||||
printf '%s\n' 'B22 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,320 @@
|
||||
{
|
||||
"batch": "B22",
|
||||
"candidates": [
|
||||
"P15-047",
|
||||
"P15-059",
|
||||
"P15-071",
|
||||
"P15-079",
|
||||
"P15-084"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"base": 4294967295,
|
||||
"expected": 4294967296,
|
||||
"mutated_fields": [],
|
||||
"name": "compact-pblk-crosses-u32",
|
||||
"nblk": 1,
|
||||
"operation": "compact_pblk",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:compact-pblk-32bit-crossing"
|
||||
},
|
||||
{
|
||||
"base": 4294967295,
|
||||
"expected": 4294967310,
|
||||
"mutated_fields": [],
|
||||
"name": "compact-pblk-widened-run",
|
||||
"nblk": 15,
|
||||
"operation": "compact_pblk",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:compact-pblk-32bit-crossing"
|
||||
},
|
||||
{
|
||||
"expected": 4168,
|
||||
"inode_isize": 64,
|
||||
"inode_off": 4096,
|
||||
"mutated_fields": [],
|
||||
"name": "index-base-aligned",
|
||||
"operation": "index_base",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:index-position"
|
||||
},
|
||||
{
|
||||
"expected": 4144,
|
||||
"inode_isize": 32,
|
||||
"inode_off": 4097,
|
||||
"mutated_fields": [],
|
||||
"name": "index-base-roundup",
|
||||
"operation": "index_base",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:index-position",
|
||||
"xattr_isize": 4
|
||||
},
|
||||
{
|
||||
"inode_isize": 32,
|
||||
"inode_off": 18446744073709551584,
|
||||
"mutated_fields": [
|
||||
"inode_off"
|
||||
],
|
||||
"name": "index-base-add-overflow",
|
||||
"operation": "index_base",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:index-position"
|
||||
},
|
||||
{
|
||||
"inode_isize": 0,
|
||||
"inode_off": 18446744073709551609,
|
||||
"mutated_fields": [
|
||||
"inode_off"
|
||||
],
|
||||
"name": "index-base-roundup-overflow",
|
||||
"operation": "index_base",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:index-position"
|
||||
},
|
||||
{
|
||||
"count": 5,
|
||||
"expected": 4136,
|
||||
"mutated_fields": [],
|
||||
"name": "index-advance-normal",
|
||||
"operation": "index_advance",
|
||||
"position": 4096,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:index-position",
|
||||
"unit": 8
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"expected": 18446744073709551615,
|
||||
"mutated_fields": [],
|
||||
"name": "index-advance-last-byte",
|
||||
"operation": "index_advance",
|
||||
"position": 18446744073709551608,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:index-position",
|
||||
"unit": 7
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"mutated_fields": [
|
||||
"unit"
|
||||
],
|
||||
"name": "index-advance-add-overflow",
|
||||
"operation": "index_advance",
|
||||
"position": 18446744073709551608,
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:index-position",
|
||||
"unit": 8
|
||||
},
|
||||
{
|
||||
"count": 2305843009213693952,
|
||||
"mutated_fields": [
|
||||
"count"
|
||||
],
|
||||
"name": "index-advance-multiply-overflow",
|
||||
"operation": "index_advance",
|
||||
"position": 0,
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:index-position",
|
||||
"unit": 8
|
||||
},
|
||||
{
|
||||
"expected": 3,
|
||||
"lclusterbits": 12,
|
||||
"mutated_fields": [],
|
||||
"name": "lcluster-count-roundup",
|
||||
"operation": "lcluster_count",
|
||||
"size": 8193,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"expected": 4503599627370496,
|
||||
"lclusterbits": 12,
|
||||
"mutated_fields": [],
|
||||
"name": "lcluster-count-u64-max",
|
||||
"operation": "lcluster_count",
|
||||
"size": 18446744073709551615,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"lclusterbits": 64,
|
||||
"mutated_fields": [
|
||||
"lclusterbits"
|
||||
],
|
||||
"name": "lcluster-count-invalid-shift",
|
||||
"operation": "lcluster_count",
|
||||
"size": 4096,
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"clusterofs": 4095,
|
||||
"expected": 18446744073709551615,
|
||||
"lclusterbits": 12,
|
||||
"lcn": 4503599627370495,
|
||||
"mutated_fields": [],
|
||||
"name": "lcluster-last-byte",
|
||||
"operation": "lcluster_pos",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"clusterofs": 0,
|
||||
"lclusterbits": 12,
|
||||
"lcn": 4503599627370496,
|
||||
"mutated_fields": [
|
||||
"lcn"
|
||||
],
|
||||
"name": "lcluster-shift-overflow",
|
||||
"operation": "lcluster_pos",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"clusterofs": 4096,
|
||||
"lclusterbits": 12,
|
||||
"lcn": 4503599627370495,
|
||||
"mutated_fields": [
|
||||
"clusterofs"
|
||||
],
|
||||
"name": "lcluster-add-overflow",
|
||||
"operation": "lcluster_pos",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"delta": 7,
|
||||
"expected": 18446744073709551615,
|
||||
"lcn": 18446744073709551608,
|
||||
"mutated_fields": [],
|
||||
"name": "lcn-delta-last-value",
|
||||
"operation": "lcn_advance",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"delta": 1,
|
||||
"lcn": 18446744073709551615,
|
||||
"mutated_fields": [
|
||||
"delta"
|
||||
],
|
||||
"name": "lcn-delta-overflow",
|
||||
"operation": "lcn_advance",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:delta-lcn"
|
||||
},
|
||||
{
|
||||
"block_size": 4096,
|
||||
"mutated_fields": [],
|
||||
"name": "physical-last-48bit-block",
|
||||
"operation": "physical_end",
|
||||
"pa": 1152921504606842880,
|
||||
"plen": 4096,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:physical-end-48bit"
|
||||
},
|
||||
{
|
||||
"block_size": 4096,
|
||||
"mutated_fields": [
|
||||
"pa"
|
||||
],
|
||||
"name": "physical-one-byte-beyond-48bit",
|
||||
"operation": "physical_end",
|
||||
"pa": 1152921504606842881,
|
||||
"plen": 4096,
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:physical-end-48bit"
|
||||
},
|
||||
{
|
||||
"block_size": 4096,
|
||||
"mutated_fields": [
|
||||
"plen"
|
||||
],
|
||||
"name": "physical-u64-add-overflow",
|
||||
"operation": "physical_end",
|
||||
"pa": 18446744073709550592,
|
||||
"plen": 1024,
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:physical-end-48bit"
|
||||
},
|
||||
{
|
||||
"block_size": 65536,
|
||||
"mutated_fields": [],
|
||||
"name": "physical-64k-representable-end",
|
||||
"operation": "physical_end",
|
||||
"pa": 18446744073709547520,
|
||||
"plen": 4095,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:physical-end-48bit"
|
||||
},
|
||||
{
|
||||
"expected": 1,
|
||||
"la": 4096,
|
||||
"mutated_fields": [],
|
||||
"name": "post-eof-at-eof",
|
||||
"operation": "post_eof",
|
||||
"size": 4096,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:post-eof"
|
||||
},
|
||||
{
|
||||
"expected": 4097,
|
||||
"la": 8192,
|
||||
"mutated_fields": [],
|
||||
"name": "post-eof-bounded",
|
||||
"operation": "post_eof",
|
||||
"size": 4096,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:post-eof"
|
||||
},
|
||||
{
|
||||
"expected": 18446744073709551615,
|
||||
"la": 18446744073709551615,
|
||||
"mutated_fields": [],
|
||||
"name": "post-eof-last-offset",
|
||||
"operation": "post_eof",
|
||||
"size": 1,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:post-eof"
|
||||
},
|
||||
{
|
||||
"expected": 18446744073709551615,
|
||||
"la": 18446744073709551615,
|
||||
"mutated_fields": [],
|
||||
"name": "post-eof-saturates-empty-inode",
|
||||
"operation": "post_eof",
|
||||
"size": 0,
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:post-eof"
|
||||
},
|
||||
{
|
||||
"expected": 18446744073709551615,
|
||||
"high": 4294967295,
|
||||
"low": 4294967295,
|
||||
"mutated_fields": [],
|
||||
"name": "fragment-offset-max-representable",
|
||||
"operation": "fragment_offset",
|
||||
"status": "PASS",
|
||||
"target_marker": "TC170:fragment-high-bits"
|
||||
},
|
||||
{
|
||||
"high": 4294967296,
|
||||
"low": 4294967295,
|
||||
"mutated_fields": [
|
||||
"high"
|
||||
],
|
||||
"name": "fragment-offset-high-unrepresentable",
|
||||
"operation": "fragment_offset",
|
||||
"status": "EINTEGRITY",
|
||||
"target_marker": "TC170:fragment-high-bits"
|
||||
}
|
||||
],
|
||||
"errno": {
|
||||
"corruption": 97,
|
||||
"provider_io": "unchanged",
|
||||
"sign": "positive"
|
||||
},
|
||||
"schema": 1,
|
||||
"test": "TC170-zmap-arithmetic"
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B23 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B23 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
fi
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
if nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' 'B23 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if nm -u "$output" | grep -q ' ZSTD_'; then
|
||||
printf '%s\n' 'B23 zstdio0 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,538 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic EROFS explicit-extent fixtures for Pre15 B23."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
||||
FEATURE_INCOMPAT_METABOX = 0x00000100
|
||||
METABOX_NID_BIT = 1 << 63
|
||||
LAYOUT_FLAT_PLAIN = 0
|
||||
LAYOUT_COMPRESSED_FULL = 1
|
||||
Z_EROFS_ADVISE_EXTENTS = 0x0001
|
||||
BLOCK_SIZE = 4096
|
||||
CARRIER_BYTES = 512 * 1024
|
||||
CRC32C_POLYNOMIAL = 0x82F63B78
|
||||
FIXED_UUID = "23232323-1515-4234-8123-000000000023"
|
||||
|
||||
|
||||
class FixtureError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Inode:
|
||||
nid: int
|
||||
offset: int
|
||||
inode_size: int
|
||||
layout: int
|
||||
size: int
|
||||
start_block: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryEntry:
|
||||
name: bytes
|
||||
nid: int
|
||||
offset: int
|
||||
|
||||
|
||||
def align(value: int, alignment: int) -> int:
|
||||
return (value + alignment - 1) & -alignment
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes | bytearray) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray) -> int:
|
||||
checksum = 0xFFFFFFFF
|
||||
for byte in data:
|
||||
checksum ^= byte
|
||||
for _ in range(8):
|
||||
checksum = (checksum >> 1) ^ (
|
||||
CRC32C_POLYNOMIAL if checksum & 1 else 0
|
||||
)
|
||||
return checksum & 0xFFFFFFFF
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, data: bytes | bytearray) -> None:
|
||||
self.data = bytearray(data)
|
||||
self.validate_super()
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "Image":
|
||||
return cls(path.read_bytes())
|
||||
|
||||
def clone(self) -> "Image":
|
||||
return Image(self.data)
|
||||
|
||||
def u16(self, offset: int) -> int:
|
||||
return struct.unpack_from("<H", self.data, offset)[0]
|
||||
|
||||
def u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self.data, offset)[0]
|
||||
|
||||
def u64(self, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", self.data, offset)[0]
|
||||
|
||||
def put_u8(self, offset: int, value: int) -> None:
|
||||
self.data[offset] = value
|
||||
|
||||
def put_u16(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<H", self.data, offset, value)
|
||||
|
||||
def put_u32(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<I", self.data, offset, value)
|
||||
|
||||
def put_u64(self, offset: int, value: int) -> None:
|
||||
struct.pack_into("<Q", self.data, offset, value)
|
||||
|
||||
@property
|
||||
def block_bits(self) -> int:
|
||||
return self.data[SUPER + 12]
|
||||
|
||||
@property
|
||||
def block_size(self) -> int:
|
||||
return 1 << self.block_bits
|
||||
|
||||
@property
|
||||
def meta_offset(self) -> int:
|
||||
return self.u32(SUPER + 40) << self.block_bits
|
||||
|
||||
@property
|
||||
def root_nid(self) -> int:
|
||||
return self.u16(SUPER + 14)
|
||||
|
||||
@property
|
||||
def blocks(self) -> int:
|
||||
return self.u32(SUPER + 36)
|
||||
|
||||
@property
|
||||
def declared_size(self) -> int:
|
||||
return self.blocks << self.block_bits
|
||||
|
||||
def validate_super(self) -> None:
|
||||
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
|
||||
raise FixtureError("invalid EROFS superblock")
|
||||
if self.block_size != BLOCK_SIZE:
|
||||
raise FixtureError("B23 requires a 4096-byte EROFS block")
|
||||
if self.declared_size > len(self.data):
|
||||
raise FixtureError("declared EROFS image exceeds provider bytes")
|
||||
if self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
|
||||
window = bytearray(self.data[SUPER : self.block_size])
|
||||
expected = struct.unpack_from("<I", window, 4)[0]
|
||||
struct.pack_into("<I", window, 4, 0)
|
||||
if crc32c(window) != expected:
|
||||
raise FixtureError("EROFS superblock checksum mismatch")
|
||||
|
||||
def update_checksum(self) -> None:
|
||||
if not self.u32(SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
|
||||
raise FixtureError("B23 seed lacks the checksum feature")
|
||||
self.put_u32(SUPER + 4, 0)
|
||||
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.block_size]))
|
||||
|
||||
def inode(self, nid: int) -> Inode:
|
||||
offset = self.meta_offset + (nid << 5)
|
||||
if offset + 32 > self.declared_size:
|
||||
raise FixtureError(f"inode {nid} exceeds the primary metadata")
|
||||
inode_format = self.u16(offset)
|
||||
inode_size = 64 if inode_format & 1 else 32
|
||||
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
|
||||
return Inode(
|
||||
nid=nid,
|
||||
offset=offset,
|
||||
inode_size=inode_size,
|
||||
layout=(inode_format >> 1) & 7,
|
||||
size=size,
|
||||
start_block=self.u32(offset + 16),
|
||||
)
|
||||
|
||||
def inode_data(self, inode: Inode) -> tuple[bytes, int]:
|
||||
if inode.layout == LAYOUT_FLAT_PLAIN:
|
||||
start = inode.start_block << self.block_bits
|
||||
end = start + inode.size
|
||||
if end > self.declared_size:
|
||||
raise FixtureError("plain inode data exceeds the image")
|
||||
return bytes(self.data[start:end]), start
|
||||
if inode.layout != 2:
|
||||
raise FixtureError(f"unsupported host inode layout {inode.layout}")
|
||||
start = inode.offset + inode.inode_size
|
||||
end = start + inode.size
|
||||
if end > self.declared_size:
|
||||
raise FixtureError("inline inode data exceeds the image")
|
||||
return bytes(self.data[start:end]), start
|
||||
|
||||
def directory_entries(self, inode: Inode) -> list[DirectoryEntry]:
|
||||
data, data_base = self.inode_data(inode)
|
||||
entries: list[DirectoryEntry] = []
|
||||
for block_start in range(0, len(data), self.block_size):
|
||||
block = data[block_start : block_start + self.block_size]
|
||||
if len(block) < 12:
|
||||
raise FixtureError("short directory block")
|
||||
first_name = struct.unpack_from("<H", block, 8)[0]
|
||||
if first_name < 12 or first_name % 12 or first_name > len(block):
|
||||
raise FixtureError("invalid directory entry table")
|
||||
count = first_name // 12
|
||||
for index in range(count):
|
||||
item = index * 12
|
||||
name_start = struct.unpack_from("<H", block, item + 8)[0]
|
||||
name_end = (
|
||||
struct.unpack_from("<H", block, item + 20)[0]
|
||||
if index + 1 < count
|
||||
else len(block)
|
||||
)
|
||||
name = block[name_start:name_end].split(b"\0", 1)[0]
|
||||
entries.append(
|
||||
DirectoryEntry(
|
||||
name=name,
|
||||
nid=struct.unpack_from("<Q", block, item)[0],
|
||||
offset=data_base + block_start + item,
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
def resolve_root(self, name: str) -> tuple[DirectoryEntry, Inode]:
|
||||
encoded = name.encode("ascii")
|
||||
for entry in self.directory_entries(self.inode(self.root_nid)):
|
||||
if entry.name == encoded:
|
||||
return entry, self.inode(entry.nid)
|
||||
raise FixtureError(f"root entry not found: {name}")
|
||||
|
||||
def ensure_size(self, end: int) -> None:
|
||||
rounded = align(end, self.block_size)
|
||||
if len(self.data) < rounded:
|
||||
self.data.extend(bytes(rounded - len(self.data)))
|
||||
self.put_u32(SUPER + 36, rounded >> self.block_bits)
|
||||
|
||||
|
||||
def normalize_times(root: Path) -> None:
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
os.utime(path, (0, 0), follow_symlinks=False)
|
||||
os.utime(root, (0, 0), follow_symlinks=False)
|
||||
|
||||
|
||||
def payload_block() -> bytes:
|
||||
return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE))
|
||||
|
||||
|
||||
def make_seed(output: Path, expected_version: str) -> Image:
|
||||
source = output / ".source"
|
||||
source.mkdir()
|
||||
(source / "target.bin").write_bytes(b"placeholder\n")
|
||||
(source / "payload.bin").write_bytes(payload_block())
|
||||
(source / "carrier.bin").write_bytes(bytes(CARRIER_BYTES))
|
||||
normalize_times(source)
|
||||
version_output = subprocess.run(
|
||||
["mkfs.erofs", "-V"], check=True, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
).stdout.strip()
|
||||
version = version_output.splitlines()[0] if version_output else ""
|
||||
if version != expected_version:
|
||||
raise FixtureError(f"mkfs version changed: {version!r}")
|
||||
seed = output / ".seed.erofs"
|
||||
command = [
|
||||
"mkfs.erofs", "-d0", "-x-1", "-T0", "--all-time", "--all-root",
|
||||
"--workers=1", "--sort=path", "-U", FIXED_UUID,
|
||||
"-E", "force-inode-extended", str(seed), str(source),
|
||||
]
|
||||
subprocess.run(command, check=True)
|
||||
shutil.rmtree(source)
|
||||
image = Image.load(seed)
|
||||
_, payload = image.resolve_root("payload.bin")
|
||||
_, carrier = image.resolve_root("carrier.bin")
|
||||
if payload.layout != LAYOUT_FLAT_PLAIN or payload.size != BLOCK_SIZE:
|
||||
raise FixtureError("payload seed is not one plain block")
|
||||
if carrier.layout != LAYOUT_FLAT_PLAIN or carrier.size != CARRIER_BYTES:
|
||||
raise FixtureError("carrier seed is not the expected plain file")
|
||||
return image
|
||||
|
||||
|
||||
def write_inode(image: Image, offset: int, size: int) -> None:
|
||||
image.data[offset : offset + 64] = bytes(64)
|
||||
image.put_u16(offset, 1 | (LAYOUT_COMPRESSED_FULL << 1))
|
||||
image.put_u16(offset + 4, stat.S_IFREG | 0o444)
|
||||
image.put_u64(offset + 8, size)
|
||||
image.put_u32(offset + 16, 1)
|
||||
image.put_u32(offset + 44, 1)
|
||||
|
||||
|
||||
def logical_starts(case: dict[str, object]) -> tuple[list[int], int, list[int]]:
|
||||
count = int(case["count"])
|
||||
mutation = case.get("mutation")
|
||||
if case.get("logical_mode") == "high32":
|
||||
starts = [0, 1 << 32, (1 << 32) + BLOCK_SIZE]
|
||||
return starts, (1 << 32) + 2 * BLOCK_SIZE, starts[1:]
|
||||
if mutation == "descending-lstart":
|
||||
starts = [0, 2 * BLOCK_SIZE, 3 * BLOCK_SIZE]
|
||||
return starts, 4 * BLOCK_SIZE, []
|
||||
starts = [index * BLOCK_SIZE for index in range(count)]
|
||||
size = count * BLOCK_SIZE
|
||||
probes = sorted({0, ((count // 2) * BLOCK_SIZE), size - BLOCK_SIZE})
|
||||
return starts, size, probes
|
||||
|
||||
|
||||
def extent_span(record_size: int, count: int) -> int:
|
||||
return 72 + (8 if record_size == 4 else 0) + record_size * count
|
||||
|
||||
|
||||
def write_extent_table(
|
||||
image: Image,
|
||||
storage_base: int,
|
||||
inode_offset: int,
|
||||
record_size: int,
|
||||
starts: list[int],
|
||||
file_size: int,
|
||||
payload_offset: int,
|
||||
) -> dict[str, int]:
|
||||
write_inode(image, storage_base + inode_offset, file_size)
|
||||
header = align(inode_offset + 64, 8)
|
||||
table = align(header + 8, record_size)
|
||||
advise = Z_EROFS_ADVISE_EXTENTS | ({4: 0, 8: 1, 16: 2, 32: 3}[record_size] << 1)
|
||||
struct.pack_into(
|
||||
"<IHH", image.data, storage_base + header,
|
||||
len(starts) & 0xFFFFFFFF, advise, len(starts) >> 32,
|
||||
)
|
||||
records = table
|
||||
if record_size == 4:
|
||||
image.put_u64(storage_base + table, payload_offset)
|
||||
records += 8
|
||||
for index, lstart in enumerate(starts):
|
||||
offset = storage_base + records + index * record_size
|
||||
image.data[offset : offset + record_size] = bytes(record_size)
|
||||
image.put_u32(offset, BLOCK_SIZE)
|
||||
if record_size >= 8:
|
||||
image.put_u32(offset + 4, payload_offset & 0xFFFFFFFF)
|
||||
if record_size >= 16:
|
||||
image.put_u32(offset + 8, payload_offset >> 32)
|
||||
image.put_u32(offset + 12, lstart & 0xFFFFFFFF)
|
||||
if record_size == 32:
|
||||
image.put_u32(offset + 16, lstart >> 32)
|
||||
return {
|
||||
"header": header,
|
||||
"table": table,
|
||||
"records": records,
|
||||
"end": records + record_size * len(starts),
|
||||
}
|
||||
|
||||
|
||||
def enable_metabox(image: Image) -> None:
|
||||
if image.meta_offset != 0:
|
||||
raise FixtureError("B23 metabox seed expects metadata block zero")
|
||||
metadata = bytes(image.data[: image.block_size])
|
||||
relocated = align(len(image.data), image.block_size)
|
||||
image.ensure_size(relocated + image.block_size)
|
||||
image.data[relocated : relocated + image.block_size] = metadata
|
||||
image.put_u32(SUPER + 40, relocated >> image.block_bits)
|
||||
image.put_u8(SUPER + 13, 1)
|
||||
image.put_u32(
|
||||
SUPER + 80, image.u32(SUPER + 80) | FEATURE_INCOMPAT_METABOX
|
||||
)
|
||||
|
||||
|
||||
def save_case(
|
||||
output: Path,
|
||||
seed: Image,
|
||||
case: dict[str, object],
|
||||
payload_offset: int,
|
||||
) -> dict[str, object]:
|
||||
image = seed.clone()
|
||||
backing = str(case["backing"])
|
||||
record_size = int(case["record_size"])
|
||||
starts, file_size, probes = logical_starts(case)
|
||||
stored_count = len(starts)
|
||||
mutation = case.get("mutation")
|
||||
if backing == "primary":
|
||||
inode_absolute = align(len(image.data), image.block_size) + 4000
|
||||
inode_offset = inode_absolute
|
||||
storage_base = 0
|
||||
stored_end = inode_absolute + extent_span(record_size, stored_count)
|
||||
image.ensure_size(stored_end)
|
||||
target_entry, _ = image.resolve_root("target.bin")
|
||||
nid_delta = inode_absolute - image.meta_offset
|
||||
if nid_delta < 0 or nid_delta % 32:
|
||||
raise FixtureError("primary synthetic inode is not NID aligned")
|
||||
target_nid = nid_delta >> 5
|
||||
image.put_u64(target_entry.offset, target_nid)
|
||||
carrier_inode_offset = None
|
||||
carrier_size = None
|
||||
elif backing == "metabox":
|
||||
enable_metabox(image)
|
||||
target_entry, _ = image.resolve_root("target.bin")
|
||||
_, carrier = image.resolve_root("carrier.bin")
|
||||
storage_base = carrier.start_block << image.block_bits
|
||||
span = extent_span(record_size, stored_count)
|
||||
inode_offset = ((CARRIER_BYTES - span) // 32) * 32
|
||||
carrier_size = inode_offset + span
|
||||
if inode_offset < 4096 or carrier_size > CARRIER_BYTES:
|
||||
raise FixtureError("metabox explicit table does not fit the carrier")
|
||||
target_nid = METABOX_NID_BIT | (inode_offset >> 5)
|
||||
image.put_u64(target_entry.offset, target_nid)
|
||||
carrier_inode_offset = carrier.offset
|
||||
image.put_u64(carrier.offset + 8, carrier_size)
|
||||
image.put_u64(SUPER + 128, carrier.nid)
|
||||
else:
|
||||
raise FixtureError(f"unknown backing: {backing}")
|
||||
|
||||
layout = write_extent_table(
|
||||
image, storage_base, inode_offset, record_size, starts, file_size,
|
||||
payload_offset,
|
||||
)
|
||||
image.update_checksum()
|
||||
baseline = image.clone()
|
||||
mutation_record = None
|
||||
if mutation == "missing-tail":
|
||||
image.put_u32(storage_base + layout["header"], 2)
|
||||
mutation_record = {
|
||||
"field": "map_header.h_extents_lo",
|
||||
"offset": storage_base + layout["header"],
|
||||
"size": 4,
|
||||
}
|
||||
elif mutation == "huge-count":
|
||||
image.put_u16(storage_base + layout["header"] + 6, 256)
|
||||
mutation_record = {
|
||||
"field": "map_header.h_extents_hi",
|
||||
"offset": storage_base + layout["header"] + 6,
|
||||
"size": 2,
|
||||
}
|
||||
elif mutation == "descending-lstart":
|
||||
changed = storage_base + layout["records"] + 2 * record_size + 12
|
||||
image.put_u32(changed, BLOCK_SIZE)
|
||||
mutation_record = {
|
||||
"field": "extent[2].lstart_lo",
|
||||
"offset": changed,
|
||||
"size": 4,
|
||||
}
|
||||
starts[2] = BLOCK_SIZE
|
||||
elif mutation == "tail-minus-one":
|
||||
if carrier_inode_offset is None or carrier_size is None:
|
||||
raise FixtureError("tail-minus-one requires metabox backing")
|
||||
image.put_u64(carrier_inode_offset + 8, carrier_size - 1)
|
||||
mutation_record = {
|
||||
"field": "metabox_carrier.i_size",
|
||||
"offset": carrier_inode_offset + 8,
|
||||
"size": 8,
|
||||
}
|
||||
elif mutation is not None:
|
||||
raise FixtureError(f"unknown mutation: {mutation}")
|
||||
|
||||
name = str(case["name"])
|
||||
image_path = output / f"{name}.erofs"
|
||||
image_path.write_bytes(image.data)
|
||||
baseline_name = None
|
||||
if mutation_record is not None:
|
||||
baseline_name = f"{name}-baseline.erofs"
|
||||
(output / baseline_name).write_bytes(baseline.data)
|
||||
|
||||
declared_count = stored_count
|
||||
if mutation == "missing-tail":
|
||||
declared_count = 2
|
||||
elif mutation == "huge-count":
|
||||
declared_count = 1 + (256 << 32)
|
||||
table_bytes = record_size * declared_count
|
||||
scan_calls = 1 if record_size <= 8 else (
|
||||
(table_bytes + 65535) // 65536
|
||||
if case["outcome"] == "PASS" else 1
|
||||
)
|
||||
return {
|
||||
"backing": backing,
|
||||
"baseline": baseline_name,
|
||||
"carrier_inode_offset": carrier_inode_offset,
|
||||
"carrier_size": carrier_size,
|
||||
"declared_count": declared_count,
|
||||
"expected_errno": 0 if case["outcome"] == "PASS" else 97,
|
||||
"expected_scan_calls": scan_calls,
|
||||
"file_size": file_size,
|
||||
"header_image_offset": storage_base + layout["header"],
|
||||
"header_logical_offset": layout["header"],
|
||||
"image": image_path.name,
|
||||
"image_sha256": sha256_bytes(image.data),
|
||||
"inode_image_offset": storage_base + inode_offset,
|
||||
"inode_logical_offset": inode_offset,
|
||||
"lstarts": starts,
|
||||
"mutation": mutation_record,
|
||||
"payload_offset": payload_offset,
|
||||
"probe_offsets": probes,
|
||||
"record_image_offset": storage_base + layout["records"],
|
||||
"record_logical_offset": layout["records"],
|
||||
"record_size": record_size,
|
||||
"stored_count": stored_count,
|
||||
"table_end_logical": layout["end"],
|
||||
"target_dirent_offset": target_entry.offset,
|
||||
"target_marker": case["target_marker"],
|
||||
"target_nid": target_nid,
|
||||
}
|
||||
|
||||
|
||||
def generate(spec_path: Path, output: Path) -> None:
|
||||
spec = json.loads(spec_path.read_text(encoding="ascii"))
|
||||
if spec.get("schema") != 1 or spec.get("batch") != "B23":
|
||||
raise FixtureError("invalid B23 spec identity")
|
||||
if output.exists() and any(output.iterdir()):
|
||||
raise FixtureError(f"output is not empty: {output}")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
seed = make_seed(output, str(spec["mkfs_version"]))
|
||||
_, payload = seed.resolve_root("payload.bin")
|
||||
payload_offset = payload.start_block << seed.block_bits
|
||||
if seed.data[payload_offset : payload_offset + BLOCK_SIZE] != payload_block():
|
||||
raise FixtureError("plain payload bytes changed")
|
||||
|
||||
cases: dict[str, object] = {}
|
||||
for case in spec["cases"]:
|
||||
cases[str(case["name"])] = save_case(
|
||||
output, seed, case, payload_offset
|
||||
)
|
||||
(output / ".seed.erofs").unlink()
|
||||
manifest = {
|
||||
"batch": "B23",
|
||||
"cases": cases,
|
||||
"chunk_size": spec["chunk_size"],
|
||||
"mkfs_version": spec["mkfs_version"],
|
||||
"payload_sha256": sha256_bytes(payload_block()),
|
||||
"schema": 1,
|
||||
"test": spec["test"],
|
||||
}
|
||||
(output / "fixture-manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
with (output / "SHA256SUMS").open("w", encoding="ascii") as sums:
|
||||
for path in sorted(output.glob("*.erofs")):
|
||||
sums.write(f"{sha256_bytes(path.read_bytes())} {path.name}\n")
|
||||
print(
|
||||
f"B23 fixtures generated cases={len(cases)} "
|
||||
f"images={len(list(output.glob('*.erofs')))}"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--spec", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
generate(args.spec, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (FixtureError, OSError, subprocess.CalledProcessError) as error:
|
||||
raise SystemExit(f"B23 fixture generation failed: {error}") from error
|
||||
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently verify B23 explicit-extent images and expectations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
|
||||
SUPER = 1024
|
||||
MAGIC = 0xE0F5E1E2
|
||||
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
|
||||
METABOX_NID_BIT = 1 << 63
|
||||
BLOCK_SIZE = 4096
|
||||
CRC32C_POLYNOMIAL = 0x82F63B78
|
||||
|
||||
|
||||
class OracleError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def sha256(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def crc32c(data: bytes | bytearray) -> int:
|
||||
checksum = 0xFFFFFFFF
|
||||
for byte in data:
|
||||
checksum ^= byte
|
||||
for _ in range(8):
|
||||
checksum = (checksum >> 1) ^ (
|
||||
CRC32C_POLYNOMIAL if checksum & 1 else 0
|
||||
)
|
||||
return checksum & 0xFFFFFFFF
|
||||
|
||||
|
||||
def u16(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<H", data, offset)[0]
|
||||
|
||||
|
||||
def u32(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<I", data, offset)[0]
|
||||
|
||||
|
||||
def u64(data: bytes, offset: int) -> int:
|
||||
return struct.unpack_from("<Q", data, offset)[0]
|
||||
|
||||
|
||||
def verify_super(data: bytes) -> tuple[int, int]:
|
||||
if len(data) < SUPER + 144 or u32(data, SUPER) != MAGIC:
|
||||
raise OracleError("invalid EROFS superblock")
|
||||
block_bits = data[SUPER + 12]
|
||||
if block_bits != 12:
|
||||
raise OracleError("B23 image block size changed")
|
||||
declared_size = u32(data, SUPER + 36) << block_bits
|
||||
if declared_size > len(data):
|
||||
raise OracleError("declared image exceeds provider")
|
||||
if u32(data, SUPER + 8) & FEATURE_COMPAT_SB_CHKSUM:
|
||||
window = bytearray(data[SUPER : 1 << block_bits])
|
||||
expected = u32(window, 4)
|
||||
struct.pack_into("<I", window, 4, 0)
|
||||
if crc32c(window) != expected:
|
||||
raise OracleError("superblock checksum mismatch")
|
||||
return 1 << block_bits, declared_size
|
||||
|
||||
|
||||
def expected_payload() -> bytes:
|
||||
return bytes((index * 37 + 11) & 0xFF for index in range(BLOCK_SIZE))
|
||||
|
||||
|
||||
def changed_offsets(left: bytes, right: bytes) -> list[int]:
|
||||
if len(left) != len(right):
|
||||
raise OracleError("negative and baseline provider sizes differ")
|
||||
return [index for index, pair in enumerate(zip(left, right)) if pair[0] != pair[1]]
|
||||
|
||||
|
||||
def verify_mutation(directory: Path, image: bytes, case: dict[str, object]) -> None:
|
||||
mutation = case.get("mutation")
|
||||
baseline_name = case.get("baseline")
|
||||
if mutation is None:
|
||||
if baseline_name is not None:
|
||||
raise OracleError("positive case unexpectedly names a baseline")
|
||||
return
|
||||
if not isinstance(mutation, dict) or not isinstance(baseline_name, str):
|
||||
raise OracleError("negative case lacks mutation evidence")
|
||||
baseline = (directory / baseline_name).read_bytes()
|
||||
differences = changed_offsets(image, baseline)
|
||||
start = int(mutation["offset"])
|
||||
end = start + int(mutation["size"])
|
||||
if not differences or any(offset < start or offset >= end for offset in differences):
|
||||
raise OracleError(f"mutation is not confined to {mutation['field']}")
|
||||
|
||||
|
||||
def record_values(
|
||||
image: bytes, case: dict[str, object]
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
record_size = int(case["record_size"])
|
||||
record_offset = int(case["record_image_offset"])
|
||||
stored_count = int(case["stored_count"])
|
||||
payload_offset = int(case["payload_offset"])
|
||||
plens: list[int] = []
|
||||
pstarts: list[int] = []
|
||||
lstarts: list[int] = []
|
||||
physical = u64(image, int(case["header_image_offset"]) + 8) if record_size == 4 else 0
|
||||
for index in range(stored_count):
|
||||
offset = record_offset + index * record_size
|
||||
if offset + record_size > len(image):
|
||||
raise OracleError("stored record exceeds provider bytes")
|
||||
plen = u32(image, offset)
|
||||
if record_size == 4:
|
||||
pstart = physical
|
||||
physical += plen & ((2 * 1024 * 1024) - 1)
|
||||
lstart = index * BLOCK_SIZE
|
||||
else:
|
||||
pstart = u32(image, offset + 4)
|
||||
if record_size >= 16:
|
||||
pstart |= u32(image, offset + 8) << 32
|
||||
lstart = u32(image, offset + 12)
|
||||
if record_size == 32:
|
||||
lstart |= u32(image, offset + 16) << 32
|
||||
else:
|
||||
lstart = index * BLOCK_SIZE
|
||||
plens.append(plen)
|
||||
pstarts.append(pstart)
|
||||
lstarts.append(lstart)
|
||||
if any(plen != BLOCK_SIZE for plen in plens):
|
||||
raise OracleError("mapped shifted extent length changed")
|
||||
if any(pstart != payload_offset for pstart in pstarts):
|
||||
raise OracleError("mapped shifted extent physical offset changed")
|
||||
return plens, pstarts, lstarts
|
||||
|
||||
|
||||
def verify_case(
|
||||
directory: Path,
|
||||
spec_case: dict[str, object],
|
||||
case: dict[str, object],
|
||||
chunk_size: int,
|
||||
) -> dict[str, object]:
|
||||
image_path = directory / str(case["image"])
|
||||
image = image_path.read_bytes()
|
||||
block_size, declared_size = verify_super(image)
|
||||
if sha256(image) != case["image_sha256"]:
|
||||
raise OracleError(f"image hash mismatch: {image_path.name}")
|
||||
if image[int(case["payload_offset"]) : int(case["payload_offset"]) + block_size] != expected_payload():
|
||||
raise OracleError(f"payload block mismatch: {image_path.name}")
|
||||
verify_mutation(directory, image, case)
|
||||
|
||||
if u64(image, int(case["target_dirent_offset"])) != int(case["target_nid"]):
|
||||
raise OracleError("target dirent NID changed")
|
||||
if case["backing"] == "metabox":
|
||||
if not int(case["target_nid"]) & METABOX_NID_BIT:
|
||||
raise OracleError("metabox target lacks bit-63 NID")
|
||||
elif int(case["target_nid"]) & METABOX_NID_BIT:
|
||||
raise OracleError("primary target unexpectedly has bit-63 NID")
|
||||
|
||||
inode = int(case["inode_image_offset"])
|
||||
inode_format = u16(image, inode)
|
||||
if inode_format != 3 or u16(image, inode + 4) != 0o100444:
|
||||
raise OracleError("synthetic inode format or mode changed")
|
||||
if u64(image, inode + 8) != int(case["file_size"]):
|
||||
raise OracleError("synthetic inode size changed")
|
||||
header = int(case["header_image_offset"])
|
||||
advise = u16(image, header + 4)
|
||||
observed_record_size = 4 << ((advise >> 1) & 3)
|
||||
if not advise & 1 or observed_record_size != int(case["record_size"]):
|
||||
raise OracleError("explicit extent advise changed")
|
||||
declared_count = u32(image, header) | (u16(image, header + 6) << 32)
|
||||
if int(case["record_size"]) >= 16 and declared_count != int(case["declared_count"]):
|
||||
raise OracleError("explicit extent count changed")
|
||||
|
||||
_, _, observed_lstarts = record_values(image, case)
|
||||
if observed_lstarts != case["lstarts"]:
|
||||
raise OracleError("little-endian lstart decode differs from the manifest")
|
||||
outcome = str(spec_case["outcome"])
|
||||
violations = [
|
||||
index for index in range(1, len(observed_lstarts))
|
||||
if observed_lstarts[index] <= observed_lstarts[index - 1]
|
||||
]
|
||||
if spec_case.get("mutation") == "descending-lstart":
|
||||
if violations != [2]:
|
||||
raise OracleError("ordering negative lacks its single violation")
|
||||
elif violations:
|
||||
raise OracleError("non-ordering case has unordered logical starts")
|
||||
|
||||
if outcome == "PASS":
|
||||
if int(case["expected_errno"]) != 0:
|
||||
raise OracleError("positive case has an errno")
|
||||
if int(case["record_size"]) <= 8:
|
||||
expected_calls = 1
|
||||
else:
|
||||
expected_calls = math.ceil(
|
||||
int(case["stored_count"]) * int(case["record_size"]) / chunk_size
|
||||
)
|
||||
if int(case["expected_scan_calls"]) != expected_calls:
|
||||
raise OracleError("bounded scan call count changed")
|
||||
if int(case["stored_count"]) >= 4096 and expected_calls >= int(case["stored_count"]):
|
||||
raise OracleError("large table regressed to per-record I/O")
|
||||
elif int(case["expected_errno"]) != 97:
|
||||
raise OracleError("corruption case does not use positive EINTEGRITY")
|
||||
|
||||
mutation = spec_case.get("mutation")
|
||||
if mutation == "missing-tail":
|
||||
requested_end = int(case["record_logical_offset"]) + 2 * int(case["record_size"])
|
||||
if requested_end <= declared_size:
|
||||
raise OracleError("primary missing-tail request remains provider-backed")
|
||||
elif mutation == "tail-minus-one":
|
||||
if int(case["carrier_size"]) != int(case["table_end_logical"]):
|
||||
raise OracleError("metabox baseline does not end at the final record")
|
||||
if u64(image, int(case["carrier_inode_offset"]) + 8) + 1 != int(case["table_end_logical"]):
|
||||
raise OracleError("metabox tail is not exactly one byte short")
|
||||
elif mutation == "huge-count":
|
||||
if int(case["declared_count"]) <= (1 << 32):
|
||||
raise OracleError("huge-count fixture does not exercise 48-bit count decode")
|
||||
|
||||
return {
|
||||
"backing": case["backing"],
|
||||
"expected_scan_calls": case["expected_scan_calls"],
|
||||
"name": spec_case["name"],
|
||||
"outcome": outcome,
|
||||
"record_size": case["record_size"],
|
||||
"stored_count": case["stored_count"],
|
||||
"target_marker": case["target_marker"],
|
||||
}
|
||||
|
||||
|
||||
def verify(spec_path: Path, directory: Path, report_path: Path) -> None:
|
||||
spec = json.loads(spec_path.read_text(encoding="ascii"))
|
||||
manifest = json.loads((directory / "fixture-manifest.json").read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B23"
|
||||
or spec.get("test") != "TC171-explicit-extents"
|
||||
or spec.get("candidates") != ["P15-060", "P15-085"]
|
||||
):
|
||||
raise OracleError("B23 spec identity changed")
|
||||
if (
|
||||
manifest.get("schema") != 1
|
||||
or manifest.get("batch") != "B23"
|
||||
or manifest.get("test") != spec["test"]
|
||||
or manifest.get("chunk_size") != spec["chunk_size"]
|
||||
):
|
||||
raise OracleError("B23 manifest identity changed")
|
||||
if spec.get("errno") != {
|
||||
"corruption": 97,
|
||||
"provider_io": "unchanged",
|
||||
"sign": "positive",
|
||||
}:
|
||||
raise OracleError("B23 errno contract changed")
|
||||
|
||||
cases = manifest.get("cases")
|
||||
if not isinstance(cases, dict) or set(cases) != {
|
||||
str(case["name"]) for case in spec["cases"]
|
||||
}:
|
||||
raise OracleError("B23 case set changed")
|
||||
decoded = []
|
||||
markers = set()
|
||||
for spec_case in spec["cases"]:
|
||||
name = str(spec_case["name"])
|
||||
case = cases[name]
|
||||
if case["target_marker"] != spec_case["target_marker"]:
|
||||
raise OracleError("target marker changed")
|
||||
markers.add(str(case["target_marker"]))
|
||||
decoded.append(
|
||||
verify_case(directory, spec_case, case, int(spec["chunk_size"]))
|
||||
)
|
||||
if len(markers) != len(spec["cases"]):
|
||||
raise OracleError("target markers are not unique")
|
||||
if {int(case["record_size"]) for case in decoded} != {4, 8, 16, 32}:
|
||||
raise OracleError("record-size coverage is incomplete")
|
||||
if {str(case["backing"]) for case in decoded} != {"primary", "metabox"}:
|
||||
raise OracleError("backing coverage is incomplete")
|
||||
|
||||
report = {
|
||||
"batch": "B23",
|
||||
"case_count": len(decoded),
|
||||
"cases": decoded,
|
||||
"chunk_size": spec["chunk_size"],
|
||||
"large_scan_calls": {
|
||||
str(case["name"]): case["expected_scan_calls"]
|
||||
for case in decoded if int(case["stored_count"]) >= 4096
|
||||
},
|
||||
"status": "PASS",
|
||||
"test": spec["test"],
|
||||
}
|
||||
report_path.write_text(
|
||||
json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
print(
|
||||
f"B23 independent oracle PASS cases={len(decoded)} "
|
||||
f"markers={len(markers)}"
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--spec", required=True, type=Path)
|
||||
parser.add_argument("--fixtures", required=True, type=Path)
|
||||
parser.add_argument("--report", required=True, type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
verify(args.spec, args.fixtures, args.report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OracleError, OSError, ValueError, KeyError, struct.error) as error:
|
||||
raise SystemExit(f"B23 oracle failed: {error}") from error
|
||||
@@ -0,0 +1,122 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define B23_BLOCK_SIZE 4096
|
||||
|
||||
static int
|
||||
parse_u64(const char *text, uint64_t *value)
|
||||
{
|
||||
char *end;
|
||||
uintmax_t parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtoumax(text, &end, 10);
|
||||
if (errno != 0 || end == text || *end != '\0' || parsed > INT64_MAX)
|
||||
return (-1);
|
||||
*value = (uint64_t)parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
check_payload(const unsigned char *data)
|
||||
{
|
||||
size_t index;
|
||||
|
||||
for (index = 0; index < B23_BLOCK_SIZE; index++) {
|
||||
if (data[index] != (unsigned char)((index * 37 + 11) & 0xff))
|
||||
return (-1);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
check_pass(int argc, char **argv)
|
||||
{
|
||||
unsigned char data[B23_BLOCK_SIZE];
|
||||
struct stat status;
|
||||
uint64_t expected_size, offset;
|
||||
ssize_t count;
|
||||
int descriptor, index;
|
||||
|
||||
if (argc < 5 || parse_u64(argv[3], &expected_size) != 0)
|
||||
return (2);
|
||||
if (stat(argv[2], &status) != 0) {
|
||||
perror("stat positive explicit target");
|
||||
return (1);
|
||||
}
|
||||
if ((uint64_t)status.st_size != expected_size) {
|
||||
fprintf(stderr, "size mismatch: got=%jd expected=%" PRIu64 "\n",
|
||||
(intmax_t)status.st_size, expected_size);
|
||||
return (1);
|
||||
}
|
||||
descriptor = open(argv[2], O_RDONLY);
|
||||
if (descriptor < 0) {
|
||||
perror("open positive explicit target");
|
||||
return (1);
|
||||
}
|
||||
for (index = 4; index < argc; index++) {
|
||||
if (parse_u64(argv[index], &offset) != 0 ||
|
||||
offset > expected_size || B23_BLOCK_SIZE > expected_size - offset) {
|
||||
fprintf(stderr, "invalid probe offset: %s\n", argv[index]);
|
||||
close(descriptor);
|
||||
return (2);
|
||||
}
|
||||
count = pread(descriptor, data, sizeof(data), (off_t)offset);
|
||||
if (count != (ssize_t)sizeof(data) || check_payload(data) != 0) {
|
||||
fprintf(stderr, "mapped payload mismatch at offset=%" PRIu64
|
||||
" count=%zd errno=%d\n", offset, count, errno);
|
||||
close(descriptor);
|
||||
return (1);
|
||||
}
|
||||
}
|
||||
if (close(descriptor) != 0) {
|
||||
perror("close positive explicit target");
|
||||
return (1);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
check_errno(int argc, char **argv)
|
||||
{
|
||||
struct stat status;
|
||||
uint64_t expected;
|
||||
|
||||
if (argc != 4 || parse_u64(argv[3], &expected) != 0 || expected > INT_MAX)
|
||||
return (2);
|
||||
errno = 0;
|
||||
if (stat(argv[2], &status) == 0) {
|
||||
fprintf(stderr, "corrupt explicit target unexpectedly published\n");
|
||||
return (1);
|
||||
}
|
||||
if (errno != (int)expected) {
|
||||
fprintf(stderr, "errno mismatch: got=%d expected=%" PRIu64 "\n",
|
||||
errno, expected);
|
||||
return (1);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
if (argc >= 2 && strcmp(argv[1], "pass") == 0)
|
||||
return (check_pass(argc, argv));
|
||||
if (argc >= 2 && strcmp(argv[1], "errno") == 0)
|
||||
return (check_errno(argc, argv));
|
||||
fprintf(stderr, "usage: %s pass PATH SIZE OFFSET... | errno PATH ERRNO\n",
|
||||
argv[0]);
|
||||
return (2);
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"batch": "B23",
|
||||
"candidates": [
|
||||
"P15-060",
|
||||
"P15-085"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 1,
|
||||
"name": "primary-r4-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 4,
|
||||
"target_marker": "TC171:primary-r4-tail-backed"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 1,
|
||||
"name": "primary-r8-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 8,
|
||||
"target_marker": "TC171:primary-r8-tail-backed"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 3,
|
||||
"name": "primary-r16-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:primary-r16-mapped"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 3,
|
||||
"logical_mode": "high32",
|
||||
"name": "primary-r32-lstart-hi",
|
||||
"outcome": "PASS",
|
||||
"record_size": 32,
|
||||
"target_marker": "TC171:primary-r32-lstart-hi"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 8192,
|
||||
"name": "primary-r16-large",
|
||||
"outcome": "PASS",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:primary-r16-bounded-scan"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 4097,
|
||||
"name": "primary-r32-large",
|
||||
"outcome": "PASS",
|
||||
"record_size": 32,
|
||||
"target_marker": "TC171:primary-r32-bounded-scan"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 1,
|
||||
"mutation": "missing-tail",
|
||||
"name": "primary-r16-missing-tail",
|
||||
"outcome": "EINTEGRITY",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:primary-missing-tail"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 1,
|
||||
"mutation": "huge-count",
|
||||
"name": "primary-r32-huge-count",
|
||||
"outcome": "EINTEGRITY",
|
||||
"record_size": 32,
|
||||
"target_marker": "TC171:primary-huge-count"
|
||||
},
|
||||
{
|
||||
"backing": "primary",
|
||||
"count": 3,
|
||||
"mutation": "descending-lstart",
|
||||
"name": "primary-r16-order",
|
||||
"outcome": "EINTEGRITY",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:primary-order"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 1,
|
||||
"name": "metabox-r4-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 4,
|
||||
"target_marker": "TC171:metabox-r4-tail-backed"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 1,
|
||||
"name": "metabox-r8-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 8,
|
||||
"target_marker": "TC171:metabox-r8-tail-backed"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 3,
|
||||
"name": "metabox-r16-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:metabox-r16-mapped"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 3,
|
||||
"name": "metabox-r32-mapped",
|
||||
"outcome": "PASS",
|
||||
"record_size": 32,
|
||||
"target_marker": "TC171:metabox-r32-mapped"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 1,
|
||||
"mutation": "tail-minus-one",
|
||||
"name": "metabox-r4-tail-minus-one",
|
||||
"outcome": "EINTEGRITY",
|
||||
"record_size": 4,
|
||||
"target_marker": "TC171:metabox-r4-tail-minus-one"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 1,
|
||||
"mutation": "tail-minus-one",
|
||||
"name": "metabox-r8-tail-minus-one",
|
||||
"outcome": "EINTEGRITY",
|
||||
"record_size": 8,
|
||||
"target_marker": "TC171:metabox-r8-tail-minus-one"
|
||||
},
|
||||
{
|
||||
"backing": "metabox",
|
||||
"count": 4097,
|
||||
"name": "metabox-r16-large",
|
||||
"outcome": "PASS",
|
||||
"record_size": 16,
|
||||
"target_marker": "TC171:metabox-bounded-scan"
|
||||
}
|
||||
],
|
||||
"chunk_size": 65536,
|
||||
"errno": {
|
||||
"corruption": 97,
|
||||
"provider_io": "unchanged",
|
||||
"sign": "positive"
|
||||
},
|
||||
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
|
||||
"schema": 1,
|
||||
"test": "TC171-explicit-extents"
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
{
|
||||
"candidate": "P15-023",
|
||||
"config": [
|
||||
{
|
||||
"class": "malformed",
|
||||
"errno": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"class": "unsupported",
|
||||
"errno": "EOPNOTSUPP"
|
||||
}
|
||||
],
|
||||
"dispatch": [
|
||||
{
|
||||
"backend_errno": "EINTEGRITY",
|
||||
"expected": "EINTEGRITY"
|
||||
},
|
||||
{
|
||||
"backend_errno": "ENOMEM",
|
||||
"expected": "ENOMEM"
|
||||
},
|
||||
{
|
||||
"backend_errno": "EIO",
|
||||
"expected": "EIO"
|
||||
}
|
||||
],
|
||||
"schema": 1,
|
||||
"xz": [
|
||||
{
|
||||
"expected": "ENOMEM",
|
||||
"status": "XZ_MEM_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EOPNOTSUPP",
|
||||
"status": "XZ_MEMLIMIT_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EOPNOTSUPP",
|
||||
"status": "XZ_OPTIONS_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EOPNOTSUPP",
|
||||
"status": "XZ_UNSUPPORTED_CHECK"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "XZ_FORMAT_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "XZ_DATA_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "XZ_BUF_ERROR"
|
||||
}
|
||||
],
|
||||
"zlib": [
|
||||
{
|
||||
"expected": "ENOMEM",
|
||||
"status": "Z_MEM_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EOPNOTSUPP",
|
||||
"status": "Z_VERSION_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "Z_NEED_DICT"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "Z_DATA_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"status": "Z_BUF_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EIO",
|
||||
"status": "Z_STREAM_ERROR"
|
||||
},
|
||||
{
|
||||
"expected": "EIO",
|
||||
"status": "Z_ERRNO"
|
||||
}
|
||||
],
|
||||
"zstd": [
|
||||
{
|
||||
"expected": "ENOMEM",
|
||||
"operation": "create_context"
|
||||
},
|
||||
{
|
||||
"expected": "EOPNOTSUPP",
|
||||
"operation": "set_window_limit"
|
||||
},
|
||||
{
|
||||
"expected": "EINTEGRITY",
|
||||
"operation": "decompress_stream"
|
||||
},
|
||||
{
|
||||
"expected": "EIO",
|
||||
"operation": "free_context"
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+434
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
SOURCE_NAMES = (
|
||||
"compress.h",
|
||||
"decompressor.c",
|
||||
"decompressor_lz4.c",
|
||||
"decompressor_lzma.c",
|
||||
"decompressor_deflate.c",
|
||||
"decompressor_zstd.c",
|
||||
"zdata.c",
|
||||
)
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
raise SystemExit(message)
|
||||
|
||||
|
||||
def replace_once(source: str, old: str, new: str, label: str) -> str:
|
||||
count = source.count(old)
|
||||
if count != 1:
|
||||
fail(f"{label}: expected one source occurrence, found {count}")
|
||||
return source.replace(old, new, 1)
|
||||
|
||||
|
||||
def extract_function(source: str, name: str) -> str:
|
||||
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
|
||||
if not match:
|
||||
fail(f"missing function: {name}")
|
||||
name_line = source.rfind("\n", 0, match.start()) + 1
|
||||
start = source.rfind("\n", 0, name_line - 1) + 1
|
||||
brace = source.find("{", match.end())
|
||||
if brace < 0:
|
||||
fail(f"missing function body: {name}")
|
||||
depth = 0
|
||||
state = "code"
|
||||
index = brace
|
||||
while index < len(source):
|
||||
char = source[index]
|
||||
following = source[index + 1] if index + 1 < len(source) else ""
|
||||
if state == "code":
|
||||
if char == "/" and following == "*":
|
||||
state = "block"
|
||||
index += 2
|
||||
continue
|
||||
if char == "/" and following == "/":
|
||||
state = "line"
|
||||
index += 2
|
||||
continue
|
||||
if char == '"':
|
||||
state = "string"
|
||||
elif char == "'":
|
||||
state = "character"
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start : index + 1]
|
||||
elif state == "block" and char == "*" and following == "/":
|
||||
state = "code"
|
||||
index += 2
|
||||
continue
|
||||
elif state == "line" and char == "\n":
|
||||
state = "code"
|
||||
elif state in {"string", "character"}:
|
||||
if char == "\\":
|
||||
index += 2
|
||||
continue
|
||||
if (state == "string" and char == '"') or (
|
||||
state == "character" and char == "'"
|
||||
):
|
||||
state = "code"
|
||||
index += 1
|
||||
fail(f"unterminated function: {name}")
|
||||
|
||||
|
||||
def committed(root: Path, baseline: str, name: str) -> str:
|
||||
completed = subprocess.run(
|
||||
["git", "-C", str(root), "show", f"{baseline}:repo-pre-15/src/{name}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
fail(f"cannot read B25 baseline {name}: {completed.stderr.strip()}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def require_once(source: str, needle: str, label: str) -> None:
|
||||
count = source.count(needle)
|
||||
if count != 1:
|
||||
fail(f"{label}: expected one occurrence, found {count}")
|
||||
|
||||
|
||||
def verify_exact_small_transforms(current: dict[str, str], base: dict[str, str]) -> None:
|
||||
expected = replace_once(
|
||||
base["compress.h"],
|
||||
"\tconst char *name;\n",
|
||||
"\tconst char *name;\n"
|
||||
"\t/* Callbacks return zero or a positive FreeBSD errno. */\n",
|
||||
"callback errno contract",
|
||||
)
|
||||
if current["compress.h"] != expected:
|
||||
fail("compress.h differs from the declared callback contract annotation")
|
||||
|
||||
expected = base["decompressor.c"]
|
||||
expected = replace_once(
|
||||
expected,
|
||||
"\t\tif (size < sizeof(*lz4))\n\t\t\treturn (EINVAL);",
|
||||
"\t\tif (size < sizeof(*lz4))\n\t\t\treturn (EINTEGRITY);",
|
||||
"short LZ4 config",
|
||||
)
|
||||
expected = replace_once(
|
||||
expected,
|
||||
"\t\t (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits))\n"
|
||||
"\t\t\treturn (EINVAL);",
|
||||
"\t\t (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits))\n"
|
||||
"\t\t\treturn (EOPNOTSUPP);",
|
||||
"unsupported LZ4 pcluster size",
|
||||
)
|
||||
if expected.count("\t\treturn (EOVERFLOW);\n") != 2:
|
||||
fail("baseline config arithmetic errno count changed")
|
||||
expected = expected.replace(
|
||||
"\t\treturn (EOVERFLOW);\n", "\t\treturn (EINTEGRITY);\n"
|
||||
)
|
||||
expected = replace_once(expected, "\tint ret;\n", "", "folding temporary")
|
||||
expected = replace_once(
|
||||
expected,
|
||||
"\tret = decompressor->decompress(&rq);\n"
|
||||
"\treturn (ret == 0 ? 0 : EIO);",
|
||||
"\treturn (decompressor->decompress(&rq));",
|
||||
"dispatch preservation",
|
||||
)
|
||||
if current["decompressor.c"] != expected:
|
||||
fail("decompressor.c differs from the declared B25 dispatch/config transforms")
|
||||
|
||||
expected = base["decompressor_lz4.c"]
|
||||
if expected.count("return (-1);") != 12 or expected.count(": -1);") != 1:
|
||||
fail("baseline LZ4 private-status count changed")
|
||||
expected = expected.replace("return (-1);", "return (EINTEGRITY);")
|
||||
expected = expected.replace(": -1);", ": EINTEGRITY);")
|
||||
if current["decompressor_lz4.c"] != expected:
|
||||
fail("LZ4 differs from the exact private-status conversion")
|
||||
|
||||
if current["zdata.c"] != base["zdata.c"]:
|
||||
fail("B25 changed the already-correct zdata I/O or cleanup path")
|
||||
|
||||
|
||||
def verify_source_contracts(
|
||||
source: dict[str, str], fixture: dict[str, object]
|
||||
) -> dict[str, object]:
|
||||
joined = "\n".join(source[name] for name in SOURCE_NAMES)
|
||||
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", joined):
|
||||
fail("negative Linux errno entered the FreeBSD codec path")
|
||||
for name in SOURCE_NAMES:
|
||||
if "return (-1)" in source[name] or re.search(r":\s*-1\s*\)", source[name]):
|
||||
fail(f"private -1 codec status remains in {name}")
|
||||
|
||||
dispatch = extract_function(source["decompressor.c"], "z_erofs_decompress")
|
||||
if dispatch.count("return (decompressor->decompress(&rq));") != 2:
|
||||
fail("typed backend dispatch does not have two direct callback exits")
|
||||
if "ret == 0 ? 0 : EIO" in dispatch:
|
||||
fail("dispatcher still folds typed backend errors")
|
||||
require_once(dispatch, "return (EOPNOTSUPP);", "unsupported algorithm")
|
||||
if dispatch.count("return (EINTEGRITY);") != 2:
|
||||
fail("dispatch-local corruption checks changed")
|
||||
for case in fixture["dispatch"]:
|
||||
if case.get("backend_errno") != case.get("expected"):
|
||||
fail("dispatch fixture does not require exact errno preservation")
|
||||
|
||||
config_expectations = {
|
||||
"z_erofs_load_lz4_config": ("decompressor.c", "EINTEGRITY", "EOPNOTSUPP"),
|
||||
"z_erofs_load_lzma_config": (
|
||||
"decompressor_lzma.c",
|
||||
"EINTEGRITY",
|
||||
"EOPNOTSUPP",
|
||||
),
|
||||
"z_erofs_load_deflate_config": (
|
||||
"decompressor_deflate.c",
|
||||
"EINTEGRITY",
|
||||
"EOPNOTSUPP",
|
||||
),
|
||||
"z_erofs_load_zstd_config": (
|
||||
"decompressor_zstd.c",
|
||||
"EINTEGRITY",
|
||||
"EOPNOTSUPP",
|
||||
),
|
||||
}
|
||||
for function, (name, malformed, unsupported) in config_expectations.items():
|
||||
body = extract_function(source[name], function)
|
||||
if f"return ({malformed});" not in body:
|
||||
fail(f"{function} lacks malformed-config errno")
|
||||
if f"return ({unsupported});" not in body:
|
||||
fail(f"{function} lacks unsupported-config errno")
|
||||
if fixture["config"] != [
|
||||
{"class": "malformed", "errno": "EINTEGRITY"},
|
||||
{"class": "unsupported", "errno": "EOPNOTSUPP"},
|
||||
]:
|
||||
fail("config fixture does not encode the B25 typed classes")
|
||||
|
||||
lzma = extract_function(source["decompressor_lzma.c"], "z_erofs_lzma_decompress")
|
||||
require_once(lzma, "return (EOVERFLOW);", "LZMA ABI overflow")
|
||||
require_once(lzma, "return (ENOMEM);", "LZMA allocation failure")
|
||||
if not (
|
||||
lzma.index("xz_dec_microlzma_run")
|
||||
< lzma.index("xz_dec_microlzma_end")
|
||||
< lzma.index("return (error);")
|
||||
):
|
||||
fail("LZMA state is not released before status publication")
|
||||
|
||||
deflate = extract_function(
|
||||
source["decompressor_deflate.c"], "z_erofs_deflate_decompress"
|
||||
)
|
||||
require_once(deflate, "inflateEnd(&strm)", "Deflate cleanup")
|
||||
require_once(
|
||||
deflate,
|
||||
"if (error == 0 && endret != Z_OK)",
|
||||
"Deflate primary-error preservation",
|
||||
)
|
||||
if deflate.index("inflateEnd(&strm)") > deflate.index("return (error);"):
|
||||
fail("Deflate returns before releasing initialized state")
|
||||
|
||||
zstd_source = source["decompressor_zstd.c"]
|
||||
enabled = extract_function(zstd_source, "z_erofs_zstd_decompress")
|
||||
require_once(enabled, "return (ENOMEM);", "Zstd allocation failure")
|
||||
require_once(
|
||||
enabled,
|
||||
"if (error == 0 && ZSTD_isError(ret))",
|
||||
"Zstd primary-error preservation",
|
||||
)
|
||||
require_once(enabled, "error = EOPNOTSUPP;", "Zstd parameter mapping")
|
||||
if enabled.count("error = EINTEGRITY;") != 4:
|
||||
fail("Zstd stream corruption/completion mappings changed")
|
||||
require_once(enabled, "error = EIO;", "Zstd release mapping")
|
||||
if "ZSTD_getErrorCode" in zstd_source:
|
||||
fail("B25 added a Zstd provider symbol outside the existing ABI")
|
||||
if zstd_source.count("return (EOPNOTSUPP);") < 3:
|
||||
fail("Zstd config/runtime disabled paths are not typed unsupported")
|
||||
expected_zstd = {
|
||||
"create_context": "ENOMEM",
|
||||
"set_window_limit": "EOPNOTSUPP",
|
||||
"decompress_stream": "EINTEGRITY",
|
||||
"free_context": "EIO",
|
||||
}
|
||||
actual_zstd = {
|
||||
case.get("operation"): case.get("expected") for case in fixture["zstd"]
|
||||
}
|
||||
if actual_zstd != expected_zstd or len(fixture["zstd"]) != len(expected_zstd):
|
||||
fail("Zstd phase fixture does not match the source contract")
|
||||
|
||||
read_extent = extract_function(source["zdata.c"], "z_erofs_read_extent")
|
||||
decode_at = read_extent.index("error = z_erofs_decompress")
|
||||
meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at)
|
||||
physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at)
|
||||
error_at = read_extent.index("if (error != 0)", decode_at)
|
||||
free_at = read_extent.index("free(decoded, M_EROFS)", error_at)
|
||||
if not (
|
||||
decode_at < meta_release_at < error_at < free_at
|
||||
and decode_at < physical_release_at < error_at < free_at
|
||||
):
|
||||
fail("zdata input/output cleanup ordering changed")
|
||||
|
||||
return {
|
||||
"callbacks": 2,
|
||||
"config_loaders": len(config_expectations),
|
||||
"dispatch_direct": True,
|
||||
"dispatch_fixture_cases": len(fixture["dispatch"]),
|
||||
"negative_errno": False,
|
||||
"zdata_unchanged": True,
|
||||
}
|
||||
|
||||
|
||||
def compile_status_harness(
|
||||
source: dict[str, str], fixture: dict[str, object], artifacts: Path
|
||||
) -> dict[str, int]:
|
||||
xz_helper = extract_function(source["decompressor_lzma.c"], "z_erofs_lzma_error")
|
||||
zlib_helper = extract_function(
|
||||
source["decompressor_deflate.c"], "z_erofs_deflate_error"
|
||||
)
|
||||
checks: list[str] = []
|
||||
for group, function in (
|
||||
("xz", "z_erofs_lzma_error"),
|
||||
("zlib", "z_erofs_deflate_error"),
|
||||
):
|
||||
for case in fixture[group]:
|
||||
checks.append(
|
||||
f'\tcheck("{group}:{case["status"]}", '
|
||||
f'{function}({case["status"]}), {case["expected"]});'
|
||||
)
|
||||
program = f'''#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#define EIO 5
|
||||
#define ENOMEM 12
|
||||
#define EOVERFLOW 75
|
||||
#define EOPNOTSUPP 95
|
||||
#define EINTEGRITY 97
|
||||
|
||||
enum xz_ret {{
|
||||
\tXZ_OK,
|
||||
\tXZ_STREAM_END,
|
||||
\tXZ_UNSUPPORTED_CHECK,
|
||||
\tXZ_MEM_ERROR,
|
||||
\tXZ_MEMLIMIT_ERROR,
|
||||
\tXZ_FORMAT_ERROR,
|
||||
\tXZ_OPTIONS_ERROR,
|
||||
\tXZ_DATA_ERROR,
|
||||
\tXZ_BUF_ERROR
|
||||
}};
|
||||
|
||||
#define Z_OK 0
|
||||
#define Z_STREAM_END 1
|
||||
#define Z_NEED_DICT 2
|
||||
#define Z_ERRNO (-1)
|
||||
#define Z_STREAM_ERROR (-2)
|
||||
#define Z_DATA_ERROR (-3)
|
||||
#define Z_MEM_ERROR (-4)
|
||||
#define Z_BUF_ERROR (-5)
|
||||
#define Z_VERSION_ERROR (-6)
|
||||
|
||||
{xz_helper}
|
||||
|
||||
{zlib_helper}
|
||||
|
||||
static int failures;
|
||||
|
||||
static void
|
||||
check(const char *name, int actual, int expected)
|
||||
{{
|
||||
\tif (actual != expected) {{
|
||||
\t\tfprintf(stderr, "%s: %d != %d\\n", name, actual, expected);
|
||||
\t\t++failures;
|
||||
\t}}
|
||||
}}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{{
|
||||
{chr(10).join(checks)}
|
||||
\tif (failures != 0)
|
||||
\t\treturn (1);
|
||||
\tprintf("status-map cases={len(checks)}\\n");
|
||||
\treturn (0);
|
||||
}}
|
||||
'''
|
||||
harness = artifacts / "B25-status-map.c"
|
||||
binary = artifacts / "B25-status-map"
|
||||
harness.write_text(program, encoding="ascii")
|
||||
compile_run = subprocess.run(
|
||||
["cc", "-std=c11", "-Wall", "-Wextra", "-Werror", str(harness), "-o", str(binary)],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
(artifacts / "B25-status-map-build.stdout").write_text(
|
||||
compile_run.stdout, encoding="utf-8"
|
||||
)
|
||||
(artifacts / "B25-status-map-build.stderr").write_text(
|
||||
compile_run.stderr, encoding="utf-8"
|
||||
)
|
||||
if compile_run.returncode != 0:
|
||||
fail("B25 status-map harness did not compile")
|
||||
execute = subprocess.run(
|
||||
[str(binary)],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
(artifacts / "B25-status-map.stdout").write_text(execute.stdout, encoding="utf-8")
|
||||
(artifacts / "B25-status-map.stderr").write_text(execute.stderr, encoding="utf-8")
|
||||
if execute.returncode != 0:
|
||||
fail("B25 status-map harness failed")
|
||||
binary.unlink()
|
||||
return {
|
||||
"xz": len(fixture["xz"]),
|
||||
"zlib": len(fixture["zlib"]),
|
||||
"zstd": len(fixture["zstd"]),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--artifacts", required=True, type=Path)
|
||||
parser.add_argument("--baseline", required=True)
|
||||
parser.add_argument("--dut", required=True, type=Path)
|
||||
parser.add_argument("--fixture", required=True, type=Path)
|
||||
parser.add_argument("--root", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
fixture = json.loads(args.fixture.read_text(encoding="ascii"))
|
||||
if fixture.get("schema") != 1 or fixture.get("candidate") != "P15-023":
|
||||
fail("invalid B25 fixture identity")
|
||||
for key in ("config", "dispatch", "xz", "zlib", "zstd"):
|
||||
if not isinstance(fixture.get(key), list) or not fixture[key]:
|
||||
fail(f"empty B25 fixture group: {key}")
|
||||
|
||||
source_dir = args.dut / "src"
|
||||
current = {
|
||||
name: (source_dir / name).read_text(encoding="utf-8") for name in SOURCE_NAMES
|
||||
}
|
||||
base = {name: committed(args.root, args.baseline, name) for name in SOURCE_NAMES}
|
||||
verify_exact_small_transforms(current, base)
|
||||
contracts = verify_source_contracts(current, fixture)
|
||||
mappings = compile_status_harness(current, fixture, args.artifacts)
|
||||
|
||||
report = {
|
||||
"candidate": "P15-023",
|
||||
"contracts": contracts,
|
||||
"mapping_cases": mappings,
|
||||
"status": "PASS",
|
||||
}
|
||||
(args.artifacts / "B25-codec-errors.json").write_text(
|
||||
json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
print(
|
||||
"B25 codec errno contract: "
|
||||
f"xz={mappings['xz']} zlib={mappings['zlib']} zstd={mappings['zstd']}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
zstdio=${5:?WITH_ZSTDIO value is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
case "$zstdio" in
|
||||
0|1) ;;
|
||||
*) printf '%s\n' 'WITH_ZSTDIO must be 0 or 1' >&2; exit 20 ;;
|
||||
esac
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang cmp grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B27 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B27 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
test "$zstdio" = 0 || extra="$extra -DZSTDIO"
|
||||
fi
|
||||
printf 'CC WITH_ZSTDIO=%s %s\n' "$zstdio" "$file"
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
nm -u "$output" | LC_ALL=C sort > B27-nm-u.txt
|
||||
if ! nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
printf '%s\n' 'B27 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if test "$zstdio" = 0; then
|
||||
if grep -q ' ZSTD_' B27-nm-u.txt; then
|
||||
printf '%s\n' 'B27 zstdio0 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
awk '$NF ~ /^ZSTD_/ { print $NF }' B27-nm-u.txt > B27-zstd-symbols.txt
|
||||
printf '%s\n' ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \
|
||||
ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError > B27-zstd-expected.txt
|
||||
if ! cmp B27-zstd-expected.txt B27-zstd-symbols.txt; then
|
||||
printf '%s\n' 'B27 zstdio1 KLD Zstd ABI changed' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,50 @@
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#define Z_EROFS_LI_D0_CBLKCNT 0x800U
|
||||
#define Z_EROFS_LCLUSTER_TYPE_NONHEAD 2U
|
||||
|
||||
static unsigned int
|
||||
compact_head_nblocks(const uint16_t *values, const uint8_t *types, int index)
|
||||
{
|
||||
unsigned int blocks, value;
|
||||
|
||||
blocks = 0;
|
||||
while (index > 0) {
|
||||
--index;
|
||||
value = values[index];
|
||||
if (types[index] == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
|
||||
if ((value & Z_EROFS_LI_D0_CBLKCNT) != 0) {
|
||||
--index;
|
||||
blocks += value & ~Z_EROFS_LI_D0_CBLKCNT;
|
||||
continue;
|
||||
}
|
||||
if (value <= 1)
|
||||
return (UINT32_MAX);
|
||||
index -= value - 2;
|
||||
continue;
|
||||
}
|
||||
++blocks;
|
||||
}
|
||||
return (blocks);
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
uint16_t values[16] = { 0 };
|
||||
uint8_t types[16] = { 0 };
|
||||
unsigned int blocks;
|
||||
|
||||
values[0] = Z_EROFS_LI_D0_CBLKCNT | 1U;
|
||||
types[0] = Z_EROFS_LCLUSTER_TYPE_NONHEAD;
|
||||
values[4] = 5U;
|
||||
types[4] = Z_EROFS_LCLUSTER_TYPE_NONHEAD;
|
||||
blocks = compact_head_nblocks(values, types, 5);
|
||||
if (blocks != 1U) {
|
||||
fprintf(stderr, "compact HEAD finalization blocks=%u expected=1\n",
|
||||
blocks);
|
||||
return (1);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
Executable
+176
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict[str, object]:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B27"
|
||||
or spec.get("candidate") != "P15-083"
|
||||
or spec.get("test") != "TC176-stream-runtime"
|
||||
):
|
||||
raise SystemExit("invalid B27 stream fixture spec")
|
||||
if set(spec.get("codecs", {})) != {"lzma", "deflate", "zstd"}:
|
||||
raise SystemExit("B27 codec set changed")
|
||||
return spec
|
||||
|
||||
|
||||
def make_source(root: Path, spec: dict[str, object]) -> bytes:
|
||||
root.mkdir(parents=True)
|
||||
source_spec = spec["source"]
|
||||
content = b"".join(
|
||||
f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii")
|
||||
for index in range(source_spec["line_count"])
|
||||
)
|
||||
if len(content) != source_spec["size"] or sha256_bytes(content) != source_spec["sha256"]:
|
||||
raise SystemExit("B27 deterministic payload identity changed")
|
||||
payload = root / "payload.bin"
|
||||
payload.write_bytes(content)
|
||||
os.utime(payload, (0, 0))
|
||||
os.utime(root, (0, 0))
|
||||
return content
|
||||
|
||||
|
||||
def run(argv: list[str], cwd: Path) -> None:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}"
|
||||
)
|
||||
|
||||
|
||||
def build_codec(
|
||||
codec: str,
|
||||
codec_spec: dict[str, object],
|
||||
spec: dict[str, object],
|
||||
source_dir: Path,
|
||||
images: Path,
|
||||
) -> list[dict[str, object]]:
|
||||
valid = images / f"{codec}-valid.erofs"
|
||||
run(["mkfs.erofs", *codec_spec["mkfs_args"], str(valid), str(source_dir)], images)
|
||||
valid_data = valid.read_bytes()
|
||||
if (
|
||||
len(valid_data) != codec_spec["expected_image_size"]
|
||||
or sha256_bytes(valid_data) != codec_spec["expected_image_sha256"]
|
||||
):
|
||||
raise SystemExit(f"{codec} valid image is not the G04 reproducible fixture")
|
||||
|
||||
extent = codec_spec["extent"]
|
||||
start = extent["physical_offset"]
|
||||
end = start + extent["physical_length"]
|
||||
block = valid_data[start:end]
|
||||
leading = next((index for index, value in enumerate(block) if value), len(block))
|
||||
stream = block[leading:]
|
||||
if leading != extent["leading_zero_bytes"] or len(stream) != extent["stream_bytes"]:
|
||||
raise SystemExit(f"{codec} pcluster padding/stream boundary changed")
|
||||
|
||||
tail = bytes.fromhex(spec["tail_bytes_hex"])
|
||||
if not tail or any(value == 0 for value in tail) or leading <= len(tail):
|
||||
raise SystemExit("B27 trailing mutation is not nonzero or lacks pcluster room")
|
||||
variants: dict[str, bytearray] = {}
|
||||
tail_data = bytearray(valid_data)
|
||||
tail_data[start + leading - len(tail) : end] = stream + tail
|
||||
variants["tail"] = tail_data
|
||||
|
||||
truncated_data = bytearray(valid_data)
|
||||
truncated_data[start + leading : end] = b"\0" + stream[:-1]
|
||||
variants["truncated"] = truncated_data
|
||||
|
||||
corruption_start = codec_spec["corruption_start"]
|
||||
if not (codec_spec["gate"]["partial_consumed"] < corruption_start < len(stream)):
|
||||
raise SystemExit(f"{codec} corruption is not after partial oracle consumption")
|
||||
corrupt_data = bytearray(valid_data)
|
||||
corrupt_data[start + leading : end] = (
|
||||
stream[:corruption_start] + b"\0" * (len(stream) - corruption_start)
|
||||
)
|
||||
if corrupt_data == valid_data:
|
||||
raise SystemExit(f"{codec} corruption mutation changed no bytes")
|
||||
variants["corrupt"] = corrupt_data
|
||||
|
||||
records = [
|
||||
{
|
||||
"class": "valid",
|
||||
"codec": codec,
|
||||
"expected_errno": 0,
|
||||
"path": valid.name,
|
||||
"sha256": sha256_path(valid),
|
||||
}
|
||||
]
|
||||
for kind, data in variants.items():
|
||||
path = images / f"{codec}-{kind}.erofs"
|
||||
path.write_bytes(data)
|
||||
records.append(
|
||||
{
|
||||
"class": kind,
|
||||
"codec": codec,
|
||||
"expected_errno": 0 if kind == "corrupt-partial" else 97,
|
||||
"path": path.name,
|
||||
"sha256": sha256_path(path),
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = load_spec(args.spec)
|
||||
if args.output.exists():
|
||||
raise SystemExit(f"refusing existing B27 output: {args.output}")
|
||||
if shutil.which("mkfs.erofs") is None:
|
||||
raise SystemExit("mkfs.erofs is required")
|
||||
args.output.mkdir(parents=True)
|
||||
source_dir = args.output / "source"
|
||||
images = args.output / "images"
|
||||
images.mkdir()
|
||||
make_source(source_dir, spec)
|
||||
records = []
|
||||
for codec in sorted(spec["codecs"]):
|
||||
records.extend(
|
||||
build_codec(codec, spec["codecs"][codec], spec, source_dir, images)
|
||||
)
|
||||
index = {
|
||||
"batch": "B27",
|
||||
"candidate": "P15-083",
|
||||
"fixture_count": len(records),
|
||||
"fixtures": records,
|
||||
"schema": 1,
|
||||
"source_sha256": spec["source"]["sha256"],
|
||||
"status": "READY",
|
||||
"test": "TC176-stream-runtime",
|
||||
}
|
||||
(args.output / "fixture-index.json").write_text(
|
||||
json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(json.dumps(index, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int
|
||||
parse_offset(const char *text, off_t *value)
|
||||
{
|
||||
char *end;
|
||||
long long parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtoll(text, &end, 10);
|
||||
if (errno != 0 || end == text || *end != '\0' || parsed < 0)
|
||||
return (-1);
|
||||
*value = (off_t)parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
parse_size(const char *text, size_t *value)
|
||||
{
|
||||
char *end;
|
||||
unsigned long parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtoul(text, &end, 10);
|
||||
if (errno != 0 || end == text || *end != '\0' || parsed == 0 ||
|
||||
parsed > 1024 * 1024)
|
||||
return (-1);
|
||||
*value = (size_t)parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
check_range(const char *path, const char *reference, off_t offset, size_t size)
|
||||
{
|
||||
unsigned char *actual, *expected;
|
||||
ssize_t actual_count, expected_count;
|
||||
int actual_fd, expected_fd, result;
|
||||
|
||||
actual = malloc(size);
|
||||
expected = malloc(size);
|
||||
if (actual == NULL || expected == NULL)
|
||||
return (1);
|
||||
actual_fd = open(path, O_RDONLY);
|
||||
expected_fd = open(reference, O_RDONLY);
|
||||
if (actual_fd < 0 || expected_fd < 0) {
|
||||
perror("open B27 range input");
|
||||
return (1);
|
||||
}
|
||||
actual_count = pread(actual_fd, actual, size, offset);
|
||||
expected_count = pread(expected_fd, expected, size, offset);
|
||||
result = actual_count == (ssize_t)size && expected_count == (ssize_t)size &&
|
||||
memcmp(actual, expected, size) == 0 ? 0 : 1;
|
||||
if (result != 0)
|
||||
fprintf(stderr, "range mismatch offset=%jd size=%zu actual=%zd expected=%zd errno=%d\n",
|
||||
(intmax_t)offset, size, actual_count, expected_count, errno);
|
||||
close(actual_fd);
|
||||
close(expected_fd);
|
||||
free(actual);
|
||||
free(expected);
|
||||
return (result);
|
||||
}
|
||||
|
||||
static int
|
||||
check_errno(const char *path, int expected_errno)
|
||||
{
|
||||
unsigned char buffer[4096];
|
||||
ssize_t count;
|
||||
int descriptor;
|
||||
|
||||
descriptor = open(path, O_RDONLY);
|
||||
if (descriptor < 0) {
|
||||
if (errno == expected_errno)
|
||||
return (0);
|
||||
perror("open B27 damaged payload");
|
||||
return (1);
|
||||
}
|
||||
for (;;) {
|
||||
errno = 0;
|
||||
count = read(descriptor, buffer, sizeof(buffer));
|
||||
if (count < 0) {
|
||||
int actual_errno = errno;
|
||||
|
||||
close(descriptor);
|
||||
if (actual_errno == expected_errno)
|
||||
return (0);
|
||||
fprintf(stderr, "read errno=%d expected=%d\n", actual_errno,
|
||||
expected_errno);
|
||||
return (1);
|
||||
}
|
||||
if (count == 0) {
|
||||
close(descriptor);
|
||||
fprintf(stderr, "damaged stream reached clean EOF\n");
|
||||
return (1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
off_t offset;
|
||||
size_t size;
|
||||
|
||||
if (argc == 6 && strcmp(argv[1], "range") == 0 &&
|
||||
parse_offset(argv[4], &offset) == 0 &&
|
||||
parse_size(argv[5], &size) == 0)
|
||||
return (check_range(argv[2], argv[3], offset, size));
|
||||
if (argc == 4 && strcmp(argv[1], "errno") == 0)
|
||||
return (check_errno(argv[2], atoi(argv[3])));
|
||||
fprintf(stderr, "usage: %s range PATH REFERENCE OFFSET SIZE | errno PATH ERRNO\n",
|
||||
argv[0]);
|
||||
return (2);
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"baseline": "029f3bf821d7bb5fabcf43d942ea6e9e40159ba6",
|
||||
"batch": "B27",
|
||||
"candidate": "P15-083",
|
||||
"codecs": {
|
||||
"deflate": {
|
||||
"corruption_start": 2553,
|
||||
"expected_image_sha256": "39455150c3e999bb7ae6c36402c15a408a5679726b6d099e7d121e009ef43af6",
|
||||
"expected_image_size": 32768,
|
||||
"extent": {
|
||||
"leading_zero_bytes": 1479,
|
||||
"logical_length": 14348,
|
||||
"logical_offset": 137204,
|
||||
"partial_size": 3587,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 28672,
|
||||
"stream_bytes": 2617
|
||||
},
|
||||
"gate": {
|
||||
"full_consumed": 2617,
|
||||
"partial_consumed": 715,
|
||||
"tail_consumed": 2617,
|
||||
"tail_fsck_exit": 0,
|
||||
"truncated_consumed": 2616
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U20000000-0000-4000-8000-000000000083",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zdeflate,level=1,dictsize=32768",
|
||||
"-C4096"
|
||||
]
|
||||
},
|
||||
"lzma": {
|
||||
"corruption_start": 476,
|
||||
"expected_image_sha256": "c26cf15844fe45a21a746bacbad2ef551c683bb9b2538de7a7eced37d8eadff9",
|
||||
"expected_image_size": 8192,
|
||||
"extent": {
|
||||
"leading_zero_bytes": 3556,
|
||||
"logical_length": 151552,
|
||||
"logical_offset": 0,
|
||||
"partial_size": 4096,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 4096,
|
||||
"stream_bytes": 540
|
||||
},
|
||||
"gate": {
|
||||
"full_consumed": 540,
|
||||
"partial_consumed": 352,
|
||||
"tail_consumed": 540,
|
||||
"tail_fsck_exit": 1,
|
||||
"truncated_consumed": 539
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U10000000-0000-4000-8000-000000000083",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zlzma,level=6,dictsize=65536",
|
||||
"-C4096"
|
||||
]
|
||||
},
|
||||
"zstd": {
|
||||
"corruption_start": 2940,
|
||||
"expected_image_sha256": "b905803f0e08500cc3c6cfe07a95fe027859165acae19ca8865856af256f32ca",
|
||||
"expected_image_size": 8192,
|
||||
"extent": {
|
||||
"leading_zero_bytes": 1092,
|
||||
"logical_length": 151552,
|
||||
"logical_offset": 0,
|
||||
"partial_size": 4096,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 4096,
|
||||
"stream_bytes": 3004
|
||||
},
|
||||
"gate": {
|
||||
"full_consumed": 3004,
|
||||
"partial_consumed": 1457,
|
||||
"tail_consumed": 3004,
|
||||
"tail_fsck_exit": 1,
|
||||
"truncated_consumed": 3003
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U30000000-0000-4000-8000-000000000083",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zzstd,level=3,dictsize=65536",
|
||||
"-C4096"
|
||||
]
|
||||
}
|
||||
},
|
||||
"errno": {
|
||||
"integrity": 97,
|
||||
"success": 0
|
||||
},
|
||||
"gate": {
|
||||
"commit": "338ba8daf81c1461b5d28ca9c5345686e4e0f2eb",
|
||||
"input_sha256": "b3c71b4ae03c6f9511246813952aab3178a1451433ece96f04562ec7cc6c1d1e",
|
||||
"required_base": "68bbe94c44e35d53cec8ab55d007f40b01cf0502",
|
||||
"script_sha256": "6847541266b0668ad12442ffb54b67a772a29ed8ebe6fba0e1e25214db16b2c2"
|
||||
},
|
||||
"schema": 1,
|
||||
"source": {
|
||||
"line_count": 4096,
|
||||
"sha256": "f9ebc3ccae455cd9a01afae784c9bbbc28c45936880f444fdf2c8b64dff44bfc",
|
||||
"size": 151552
|
||||
},
|
||||
"tail_bytes_hex": "a55ac33c96696996",
|
||||
"test": "TC176-stream-runtime"
|
||||
}
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
dut=${1:?DUT path is required}
|
||||
freebsd_src=${2:?FreeBSD source path is required}
|
||||
output=${3:?output module path is required}
|
||||
work=${4:?work directory is required}
|
||||
zstdio=${5:?WITH_ZSTDIO value is required}
|
||||
src=$dut/src
|
||||
sys=$freebsd_src/sys
|
||||
target=x86_64-unknown-freebsd15.0
|
||||
|
||||
case "$zstdio" in
|
||||
0|1) ;;
|
||||
*) printf '%s\n' 'WITH_ZSTDIO must be 0 or 1' >&2; exit 20 ;;
|
||||
esac
|
||||
test -d "$sys" || { printf '%s\n' "FreeBSD sys tree is absent: $sys" >&2; exit 21; }
|
||||
for tool in awk clang cmp grep nm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf '%s\n' "missing B28 KLD tool: $tool" >&2
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test ! -e "$work" || { printf '%s\n' "B28 KLD work path exists: $work" >&2; exit 20; }
|
||||
mkdir "$work"
|
||||
cd "$work"
|
||||
ln -s "$sys/amd64/include" machine
|
||||
ln -s "$sys/x86/include" x86
|
||||
ln -s "$sys/i386/include" i386
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -p
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -q
|
||||
awk -f "$sys/tools/vnode_if.awk" "$sys/kern/vnode_if.src" -h
|
||||
: > opt_global.h
|
||||
|
||||
cflags="-O2 -pipe -fno-common -fno-strict-aliasing \
|
||||
-D_KERNEL -DKLD_MODULE -nostdinc -include $work/opt_global.h \
|
||||
-I$work -I$sys -I$sys/contrib/ck/include -mcmodel=kernel \
|
||||
-mno-red-zone -mno-mmx -mno-sse -msoft-float \
|
||||
-fno-asynchronous-unwind-tables -ffreestanding -fwrapv \
|
||||
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fstack-protector \
|
||||
-Wall -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith \
|
||||
-Wcast-qual -Wundef -Wno-pointer-sign -Wmissing-include-dirs \
|
||||
-Wno-unknown-pragmas -Wno-address-of-packed-member \
|
||||
-Wno-format-zero-length -mno-aes -mno-avx -std=gnu17 \
|
||||
-D__printf__=__freebsd_kprintf__ --target=$target"
|
||||
|
||||
for file in super.c inode.c data.c namei.c dir.c xattr.c erofs_vnops.c \
|
||||
decompressor.c zmap.c zdata.c decompressor_lz4.c \
|
||||
decompressor_lzma.c decompressor_deflate.c decompressor_zstd.c; do
|
||||
extra=
|
||||
if test "$file" = decompressor_zstd.c; then
|
||||
extra="-I$sys/contrib/zstd/lib/freebsd"
|
||||
test "$zstdio" = 0 || extra="$extra -DZSTDIO"
|
||||
fi
|
||||
printf 'CC WITH_ZSTDIO=%s %s\n' "$zstdio" "$file"
|
||||
clang $cflags $extra -c "$src/$file" -o "${file%.c}.o"
|
||||
done
|
||||
|
||||
clang --target="$target" -r -nostdlib ./*.o -o "$output"
|
||||
nm -u "$output" | LC_ALL=C sort > B28-nm-u.txt
|
||||
nm -g "$output" | LC_ALL=C sort > B28-nm-global.txt
|
||||
if ! awk '$NF == "z_erofs_decompress_supports_subextent" { found = 1 } END { exit !found }' \
|
||||
B28-nm-global.txt; then
|
||||
printf '%s\n' 'B28 partial capability symbol is absent' >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! nm -u "$output" | awk '$NF == "bcmp" { found = 1 } END { exit found }'; then
|
||||
printf '%s\n' 'B28 KLD contains an unresolved bcmp reference' >&2
|
||||
exit 1
|
||||
fi
|
||||
if test "$zstdio" = 0; then
|
||||
if grep -q ' ZSTD_' B28-nm-u.txt; then
|
||||
printf '%s\n' 'B28 zstdio0 KLD contains an unexpected Zstd symbol' >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
awk '$NF ~ /^ZSTD_/ { print $NF }' B28-nm-u.txt > B28-zstd-symbols.txt
|
||||
printf '%s\n' ZSTD_DCtx_setParameter ZSTD_createDCtx_advanced \
|
||||
ZSTD_decompressStream ZSTD_freeDCtx ZSTD_isError > B28-zstd-expected.txt
|
||||
if ! cmp B28-zstd-expected.txt B28-zstd-symbols.txt; then
|
||||
printf '%s\n' 'B28 zstdio1 KLD Zstd ABI changed' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def load_spec(path: Path) -> dict[str, object]:
|
||||
spec = json.loads(path.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B28"
|
||||
or spec.get("candidate") != "P15-086"
|
||||
or spec.get("test") != "TC176-stream-runtime"
|
||||
):
|
||||
raise SystemExit("invalid B28 partial fixture spec")
|
||||
if set(spec.get("codecs", {})) != {"lz4", "lzma", "deflate", "zstd"}:
|
||||
raise SystemExit("B28 codec set changed")
|
||||
return spec
|
||||
|
||||
|
||||
def make_sources(root: Path, spec: dict[str, object]) -> dict[str, bytes]:
|
||||
sources: dict[str, bytes] = {}
|
||||
lz4_spec = spec["sources"]["lz4"]
|
||||
sources["lz4"] = b"".join(
|
||||
bytes([segment["byte"]]) * segment["length"]
|
||||
for segment in lz4_spec["segments"]
|
||||
)
|
||||
stream_spec = spec["sources"]["stream"]
|
||||
sources["stream"] = b"".join(
|
||||
f"P15-086-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode(
|
||||
"ascii"
|
||||
)
|
||||
for index in range(stream_spec["line_count"])
|
||||
)
|
||||
for name, content in sources.items():
|
||||
expected = spec["sources"][name]
|
||||
if len(content) != expected["size"] or sha256_bytes(content) != expected["sha256"]:
|
||||
raise SystemExit(f"B28 {name} source identity changed")
|
||||
directory = root / name
|
||||
directory.mkdir(parents=True)
|
||||
payload = directory / "payload.bin"
|
||||
payload.write_bytes(content)
|
||||
os.utime(payload, (0, 0))
|
||||
os.utime(directory, (0, 0))
|
||||
return sources
|
||||
|
||||
|
||||
def run(argv: list[str], cwd: Path, expected: set[int] = {0}) -> subprocess.CompletedProcess[str]:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if completed.returncode not in expected:
|
||||
raise SystemExit(
|
||||
f"command failed ({completed.returncode}): {' '.join(argv)}\n{completed.stdout}"
|
||||
)
|
||||
return completed
|
||||
|
||||
|
||||
def verify_fsck(image: Path, expected: bytes, label: str) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix=f"b28-{label}-fsck-") as temporary:
|
||||
destination = Path(temporary)
|
||||
run(["fsck.erofs", f"--extract={destination}", str(image)], image.parent)
|
||||
if (destination / "payload.bin").read_bytes() != expected:
|
||||
raise SystemExit(f"B28 {label} fsck extraction mismatch")
|
||||
|
||||
|
||||
def build_codec(
|
||||
codec: str,
|
||||
codec_spec: dict[str, object],
|
||||
sources: dict[str, bytes],
|
||||
source_root: Path,
|
||||
images: Path,
|
||||
) -> list[dict[str, object]]:
|
||||
valid = images / f"{codec}-valid.erofs"
|
||||
run(
|
||||
[
|
||||
"mkfs.erofs",
|
||||
*codec_spec["mkfs_args"],
|
||||
str(valid),
|
||||
str(source_root / codec_spec["source"]),
|
||||
],
|
||||
images,
|
||||
)
|
||||
valid_data = valid.read_bytes()
|
||||
actual_image_sha256 = sha256_bytes(valid_data)
|
||||
source = sources[codec_spec["source"]]
|
||||
verify_fsck(valid, source, f"{codec}-valid")
|
||||
|
||||
extent = codec_spec["extent"]
|
||||
start = extent["physical_offset"]
|
||||
end = start + extent["physical_length"]
|
||||
block = valid_data[start:end]
|
||||
leading = next((index for index, value in enumerate(block) if value), len(block))
|
||||
stream = block[leading:]
|
||||
if (
|
||||
leading != extent["leading_zero_bytes"]
|
||||
or len(stream) != extent["stream_bytes"]
|
||||
or not (extent["prefix_consumed"] < codec_spec["corruption_start"] < len(stream))
|
||||
):
|
||||
raise SystemExit(f"{codec} pcluster or corruption boundary changed")
|
||||
|
||||
corrupted_data = bytearray(valid_data)
|
||||
corruption = start + leading + codec_spec["corruption_start"]
|
||||
corrupted_data[corruption:end] = b"\0" * (end - corruption)
|
||||
if corrupted_data == valid_data:
|
||||
raise SystemExit(f"{codec} corruption mutation changed no bytes")
|
||||
corrupted = images / f"{codec}-corrupt.erofs"
|
||||
corrupted.write_bytes(corrupted_data)
|
||||
with tempfile.TemporaryDirectory(prefix=f"b28-{codec}-corrupt-") as temporary:
|
||||
failed = run(
|
||||
["fsck.erofs", f"--extract={temporary}", str(corrupted)],
|
||||
images,
|
||||
set(range(1, 256)),
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"class": "valid",
|
||||
"codec": codec,
|
||||
"expected_errno": 0,
|
||||
"gate_image_sha256": codec_spec["gate_image_sha256"],
|
||||
"path": valid.name,
|
||||
"sha256": actual_image_sha256,
|
||||
},
|
||||
{
|
||||
"class": "corrupt",
|
||||
"codec": codec,
|
||||
"expected_full_errno": 97,
|
||||
"expected_prefix_errno": 0 if codec_spec["decision"] == "GO" else 97,
|
||||
"fsck_exit": failed.returncode,
|
||||
"path": corrupted.name,
|
||||
"sha256": sha256_path(corrupted),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = load_spec(args.spec)
|
||||
if args.output.exists():
|
||||
raise SystemExit(f"refusing existing B28 output: {args.output}")
|
||||
for tool in ("mkfs.erofs", "fsck.erofs"):
|
||||
if shutil.which(tool) is None:
|
||||
raise SystemExit(f"{tool} is required")
|
||||
|
||||
args.output.mkdir(parents=True)
|
||||
source_root = args.output / "sources"
|
||||
images = args.output / "images"
|
||||
images.mkdir()
|
||||
sources = make_sources(source_root, spec)
|
||||
records = []
|
||||
for codec in sorted(spec["codecs"]):
|
||||
records.extend(
|
||||
build_codec(codec, spec["codecs"][codec], sources, source_root, images)
|
||||
)
|
||||
(args.output / "spec.json").write_bytes(args.spec.read_bytes())
|
||||
index = {
|
||||
"batch": "B28",
|
||||
"candidate": "P15-086",
|
||||
"fixture_count": len(records),
|
||||
"fixtures": records,
|
||||
"schema": 1,
|
||||
"source_sha256": {
|
||||
name: sha256_bytes(content) for name, content in sorted(sources.items())
|
||||
},
|
||||
"status": "READY",
|
||||
"test": "TC176-stream-runtime",
|
||||
}
|
||||
(args.output / "fixture-index.json").write_text(
|
||||
json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(json.dumps(index, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int
|
||||
parse_offset(const char *text, off_t *value)
|
||||
{
|
||||
char *end;
|
||||
long long parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtoll(text, &end, 10);
|
||||
if (errno != 0 || end == text || *end != '\0' || parsed < 0)
|
||||
return (-1);
|
||||
*value = (off_t)parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
parse_size(const char *text, size_t *value)
|
||||
{
|
||||
char *end;
|
||||
unsigned long parsed;
|
||||
|
||||
errno = 0;
|
||||
parsed = strtoul(text, &end, 10);
|
||||
if (errno != 0 || end == text || *end != '\0' || parsed == 0 ||
|
||||
parsed > 1024 * 1024)
|
||||
return (-1);
|
||||
*value = (size_t)parsed;
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
open_pair(const char *path, const char *reference, int *actual_fd,
|
||||
int *expected_fd)
|
||||
{
|
||||
|
||||
*actual_fd = open(path, O_RDONLY);
|
||||
*expected_fd = open(reference, O_RDONLY);
|
||||
if (*actual_fd >= 0 && *expected_fd >= 0)
|
||||
return (0);
|
||||
perror("open B28 range input");
|
||||
if (*actual_fd >= 0)
|
||||
close(*actual_fd);
|
||||
if (*expected_fd >= 0)
|
||||
close(*expected_fd);
|
||||
return (1);
|
||||
}
|
||||
|
||||
static int
|
||||
check_range(const char *path, const char *reference, off_t offset, size_t size)
|
||||
{
|
||||
unsigned char *actual, *expected;
|
||||
ssize_t actual_count, expected_count;
|
||||
int actual_fd, expected_fd, result;
|
||||
|
||||
actual = malloc(size);
|
||||
expected = malloc(size);
|
||||
if (actual == NULL || expected == NULL) {
|
||||
free(actual);
|
||||
free(expected);
|
||||
return (1);
|
||||
}
|
||||
if (open_pair(path, reference, &actual_fd, &expected_fd) != 0) {
|
||||
free(actual);
|
||||
free(expected);
|
||||
return (1);
|
||||
}
|
||||
actual_count = pread(actual_fd, actual, size, offset);
|
||||
expected_count = pread(expected_fd, expected, size, offset);
|
||||
result = actual_count == (ssize_t)size && expected_count == (ssize_t)size &&
|
||||
memcmp(actual, expected, size) == 0 ? 0 : 1;
|
||||
if (result != 0)
|
||||
fprintf(stderr, "range mismatch offset=%jd size=%zu actual=%zd expected=%zd errno=%d\n",
|
||||
(intmax_t)offset, size, actual_count, expected_count, errno);
|
||||
close(actual_fd);
|
||||
close(expected_fd);
|
||||
free(actual);
|
||||
free(expected);
|
||||
return (result);
|
||||
}
|
||||
|
||||
static int
|
||||
check_range_errno(const char *path, off_t offset, size_t size,
|
||||
int expected_errno)
|
||||
{
|
||||
unsigned char *buffer;
|
||||
ssize_t count;
|
||||
int descriptor, result;
|
||||
|
||||
buffer = malloc(size);
|
||||
if (buffer == NULL)
|
||||
return (1);
|
||||
descriptor = open(path, O_RDONLY);
|
||||
if (descriptor < 0) {
|
||||
perror("open B28 damaged payload");
|
||||
free(buffer);
|
||||
return (1);
|
||||
}
|
||||
errno = 0;
|
||||
count = pread(descriptor, buffer, size, offset);
|
||||
result = count < 0 && errno == expected_errno ? 0 : 1;
|
||||
if (result != 0)
|
||||
fprintf(stderr, "range errno mismatch offset=%jd size=%zu count=%zd errno=%d expected=%d\n",
|
||||
(intmax_t)offset, size, count, errno, expected_errno);
|
||||
close(descriptor);
|
||||
free(buffer);
|
||||
return (result);
|
||||
}
|
||||
|
||||
static int
|
||||
check_full_errno(const char *path, int expected_errno)
|
||||
{
|
||||
unsigned char buffer[4096];
|
||||
ssize_t count;
|
||||
int descriptor;
|
||||
|
||||
descriptor = open(path, O_RDONLY);
|
||||
if (descriptor < 0) {
|
||||
if (errno == expected_errno)
|
||||
return (0);
|
||||
perror("open B28 damaged payload");
|
||||
return (1);
|
||||
}
|
||||
for (;;) {
|
||||
errno = 0;
|
||||
count = read(descriptor, buffer, sizeof(buffer));
|
||||
if (count < 0) {
|
||||
int actual_errno = errno;
|
||||
|
||||
close(descriptor);
|
||||
if (actual_errno == expected_errno)
|
||||
return (0);
|
||||
fprintf(stderr, "full errno=%d expected=%d\n", actual_errno,
|
||||
expected_errno);
|
||||
return (1);
|
||||
}
|
||||
if (count == 0) {
|
||||
close(descriptor);
|
||||
fprintf(stderr, "damaged extent reached clean EOF\n");
|
||||
return (1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
off_t offset;
|
||||
size_t size;
|
||||
|
||||
if (argc == 6 && strcmp(argv[1], "range") == 0 &&
|
||||
parse_offset(argv[4], &offset) == 0 &&
|
||||
parse_size(argv[5], &size) == 0)
|
||||
return (check_range(argv[2], argv[3], offset, size));
|
||||
if (argc == 6 && strcmp(argv[1], "range-errno") == 0 &&
|
||||
parse_offset(argv[3], &offset) == 0 &&
|
||||
parse_size(argv[4], &size) == 0)
|
||||
return (check_range_errno(argv[2], offset, size, atoi(argv[5])));
|
||||
if (argc == 4 && strcmp(argv[1], "full-errno") == 0)
|
||||
return (check_full_errno(argv[2], atoi(argv[3])));
|
||||
fprintf(stderr, "usage: %s range PATH REFERENCE OFFSET SIZE | range-errno PATH OFFSET SIZE ERRNO | full-errno PATH ERRNO\n",
|
||||
argv[0]);
|
||||
return (2);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
{
|
||||
"baseline": "2c61589e542ae96db8a100d71f42a8dd58ced172",
|
||||
"batch": "B28",
|
||||
"candidate": "P15-086",
|
||||
"codecs": {
|
||||
"deflate": {
|
||||
"corruption_start": 4031,
|
||||
"decision": "GO",
|
||||
"gate_image_sha256": "5f98ff39be30b1396be8164e54a8ae80b144d181d5295a640b2e7814406d7a2d",
|
||||
"extent": {
|
||||
"leading_zero_bytes": 1,
|
||||
"logical_length": 22878,
|
||||
"logical_offset": 114326,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 24576,
|
||||
"prefix_consumed": 805,
|
||||
"stream_bytes": 4095
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U86100000-0000-4000-8000-000000000086",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zdeflate,level=1,dictsize=32768",
|
||||
"-C4096"
|
||||
],
|
||||
"source": "stream"
|
||||
},
|
||||
"lz4": {
|
||||
"corruption_start": 4032,
|
||||
"decision": "GO",
|
||||
"gate_image_sha256": "853649c78b159161421a6e833e0c516216382f3e8d135fc80aacd509ddb27b2c",
|
||||
"extent": {
|
||||
"leading_zero_bytes": 0,
|
||||
"logical_length": 1038906,
|
||||
"logical_offset": 0,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 4096,
|
||||
"prefix_consumed": 1032,
|
||||
"stream_bytes": 4096
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U86000000-0000-4000-8000-000000000086",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"-zlz4",
|
||||
"-C4096",
|
||||
"-Elegacy-compress"
|
||||
],
|
||||
"source": "lz4"
|
||||
},
|
||||
"lzma": {
|
||||
"corruption_start": 476,
|
||||
"decision": "GO",
|
||||
"gate_image_sha256": "887d4a9baf629533c8d512177a84ca256a20135157d29a201408534f63230889",
|
||||
"extent": {
|
||||
"leading_zero_bytes": 3556,
|
||||
"logical_length": 151552,
|
||||
"logical_offset": 0,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 4096,
|
||||
"prefix_consumed": 352,
|
||||
"stream_bytes": 540
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U86200000-0000-4000-8000-000000000086",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zlzma,level=6,dictsize=65536",
|
||||
"-C4096"
|
||||
],
|
||||
"source": "stream"
|
||||
},
|
||||
"zstd": {
|
||||
"corruption_start": 2940,
|
||||
"decision": "FULL_FALLBACK",
|
||||
"gate_image_sha256": "93f6beb46e77187ea8b0fc1ad3a5a1cb565d546d9ce40d51278f26d67187d5fa",
|
||||
"extent": {
|
||||
"leading_zero_bytes": 1092,
|
||||
"logical_length": 151552,
|
||||
"logical_offset": 0,
|
||||
"physical_length": 4096,
|
||||
"physical_offset": 4096,
|
||||
"prefix_consumed": 1457,
|
||||
"stream_bytes": 3004
|
||||
},
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U86300000-0000-4000-8000-000000000086",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zzstd,level=3,dictsize=65536",
|
||||
"-C4096"
|
||||
],
|
||||
"source": "stream"
|
||||
}
|
||||
},
|
||||
"errno": {
|
||||
"integrity": 97,
|
||||
"overflow": 84
|
||||
},
|
||||
"gate": {
|
||||
"decision_commit": "07a42b667f3b8754f84beb1d8dcccb4378c1309c",
|
||||
"evidence_commit": "2c61589e542ae96db8a100d71f42a8dd58ced172",
|
||||
"evidence_root": "planning/pre15/evidence/20260815T154915Z-G04-G05-P15-086",
|
||||
"input_sha256": "62f7f505aef846fa85085039b2f47e252e11602e6ef1bf41f56841cdc9089a2a",
|
||||
"result_sha256": "33ef86b73447cd8c8099dc33901b9c4fff6c8ecedda3837d25cac9463a369d4f",
|
||||
"script_sha256": "591e407f20a6c8ab5d506329b3bd39ff7bd7f78b66ac86b9db2c45a498b31184"
|
||||
},
|
||||
"ranges": [
|
||||
{
|
||||
"length": 4096,
|
||||
"name": "prefix",
|
||||
"offset": 0
|
||||
},
|
||||
{
|
||||
"length": 4096,
|
||||
"name": "cross-page",
|
||||
"offset": 3584
|
||||
},
|
||||
{
|
||||
"length": 4096,
|
||||
"name": "middle",
|
||||
"offset": 8192
|
||||
},
|
||||
{
|
||||
"length": 4096,
|
||||
"name": "tail",
|
||||
"offset": -4096
|
||||
}
|
||||
],
|
||||
"schema": 1,
|
||||
"sources": {
|
||||
"lz4": {
|
||||
"segments": [
|
||||
{
|
||||
"byte": 69,
|
||||
"length": 262144
|
||||
},
|
||||
{
|
||||
"byte": 82,
|
||||
"length": 262144
|
||||
},
|
||||
{
|
||||
"byte": 79,
|
||||
"length": 262144
|
||||
},
|
||||
{
|
||||
"byte": 70,
|
||||
"length": 262144
|
||||
}
|
||||
],
|
||||
"sha256": "b4802e36692c8124aa80813711cf03d5cec47729ecd25acf5cc4fd53c484ea80",
|
||||
"size": 1048576
|
||||
},
|
||||
"stream": {
|
||||
"line_count": 4096,
|
||||
"sha256": "3c6a1cd405abc8b67b0100dbceebd0f918836563c7a5f705b9cfc178b35fca28",
|
||||
"size": 151552
|
||||
}
|
||||
},
|
||||
"test": "TC176-stream-runtime"
|
||||
}
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return sha256_bytes(path.read_bytes())
|
||||
|
||||
|
||||
def run(argv: list[str], cwd: Path) -> None:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
f"command failed ({completed.returncode}): {' '.join(argv)}\n"
|
||||
f"{completed.stdout}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B32"
|
||||
or spec.get("test") != "TC168-cache-inflight"
|
||||
):
|
||||
raise SystemExit("invalid B32 cache fixture spec")
|
||||
if args.output.exists():
|
||||
raise SystemExit(f"refusing existing B32 output: {args.output}")
|
||||
if shutil.which("mkfs.erofs") is None:
|
||||
raise SystemExit("mkfs.erofs is required")
|
||||
|
||||
source = (b"Pre15-B32-cache-state:0123456789abcdef\n" * 2000)[
|
||||
: spec["source"]["size"]
|
||||
]
|
||||
if sha256_bytes(source) != spec["source"]["sha256"]:
|
||||
raise SystemExit("B32 deterministic source identity changed")
|
||||
|
||||
source_dir = args.output / "source"
|
||||
images = args.output / "images"
|
||||
source_dir.mkdir(parents=True)
|
||||
images.mkdir()
|
||||
payload = source_dir / "payload.bin"
|
||||
payload.write_bytes(source)
|
||||
os.utime(payload, (0, 0))
|
||||
os.utime(source_dir, (0, 0))
|
||||
|
||||
valid = images / "lzma-valid.erofs"
|
||||
run(["mkfs.erofs", *spec["mkfs_args"], str(valid), str(source_dir)], images)
|
||||
valid_data = valid.read_bytes()
|
||||
if len(valid_data) != 8192:
|
||||
raise SystemExit(f"unexpected B32 image size: {len(valid_data)}")
|
||||
pcluster_start = 4096
|
||||
pcluster_end = 8192
|
||||
pcluster = valid_data[pcluster_start:pcluster_end]
|
||||
leading = next(
|
||||
(index for index, value in enumerate(pcluster) if value), len(pcluster)
|
||||
)
|
||||
stream = pcluster[leading:]
|
||||
if leading == len(pcluster) or len(stream) < 2:
|
||||
raise SystemExit("B32 LZMA stream boundary is empty")
|
||||
|
||||
truncated_data = bytearray(valid_data)
|
||||
truncated_data[pcluster_start + leading : pcluster_end] = b"\0" + stream[:-1]
|
||||
if truncated_data == valid_data:
|
||||
raise SystemExit("B32 truncated mutation changed no bytes")
|
||||
truncated = images / "lzma-truncated.erofs"
|
||||
truncated.write_bytes(truncated_data)
|
||||
|
||||
records = []
|
||||
for image_class, path, expected_errno in (
|
||||
("valid", valid, 0),
|
||||
("truncated", truncated, 97),
|
||||
):
|
||||
records.append(
|
||||
{
|
||||
"class": image_class,
|
||||
"expected_errno": expected_errno,
|
||||
"path": path.name,
|
||||
"sha256": sha256_path(path),
|
||||
}
|
||||
)
|
||||
index = {
|
||||
"batch": "B32",
|
||||
"decoded_size": len(source),
|
||||
"fixture_count": len(records),
|
||||
"fixtures": records,
|
||||
"leading_zero_bytes": leading,
|
||||
"physical_length": len(pcluster),
|
||||
"physical_offset": pcluster_start,
|
||||
"schema": 1,
|
||||
"source_sha256": sha256_path(payload),
|
||||
"status": "READY",
|
||||
"test": "TC168-cache-inflight",
|
||||
}
|
||||
(args.output / "fixture-index.json").write_text(
|
||||
json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(json.dumps(index, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,541 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#define B32_WORKERS 64
|
||||
#define B32_DATA_SIZE 32
|
||||
|
||||
enum cache_state {
|
||||
CACHE_EMPTY,
|
||||
CACHE_INFLIGHT,
|
||||
CACHE_READY,
|
||||
CACHE_FAILED,
|
||||
};
|
||||
|
||||
enum cache_lookup {
|
||||
CACHE_BYPASS,
|
||||
CACHE_HIT,
|
||||
CACHE_OWNER,
|
||||
CACHE_ERROR,
|
||||
};
|
||||
|
||||
struct map_blocks {
|
||||
uint64_t m_pa;
|
||||
uint64_t m_la;
|
||||
uint64_t m_plen;
|
||||
uint64_t m_llen;
|
||||
uint16_t m_deviceid;
|
||||
uint8_t m_algorithmformat;
|
||||
unsigned int m_flags;
|
||||
};
|
||||
|
||||
struct cache_key {
|
||||
struct map_blocks map;
|
||||
uint64_t nid;
|
||||
size_t decoded_size;
|
||||
};
|
||||
|
||||
struct cache {
|
||||
pthread_mutex_t lock;
|
||||
pthread_cond_t cv;
|
||||
struct cache_key key;
|
||||
unsigned char *data;
|
||||
unsigned int waiters;
|
||||
int error;
|
||||
enum cache_state state;
|
||||
bool closing;
|
||||
};
|
||||
|
||||
struct wave {
|
||||
struct cache *cache;
|
||||
struct cache_key key;
|
||||
pthread_barrier_t barrier;
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
atomic_uint decodes;
|
||||
atomic_uint bypasses;
|
||||
int expected_error;
|
||||
int results[B32_WORKERS];
|
||||
};
|
||||
|
||||
struct worker_arg {
|
||||
struct wave *wave;
|
||||
unsigned int index;
|
||||
};
|
||||
|
||||
struct fini_arg {
|
||||
struct cache *cache;
|
||||
atomic_bool done;
|
||||
};
|
||||
|
||||
static void
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "B32 oracle failure: %s\n", message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void
|
||||
check_pthread(int error, const char *operation)
|
||||
{
|
||||
if (error != 0) {
|
||||
errno = error;
|
||||
perror(operation);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
key_equal(const struct cache_key *left, const struct cache_key *right)
|
||||
{
|
||||
return (left->nid == right->nid &&
|
||||
left->decoded_size == right->decoded_size &&
|
||||
left->map.m_pa == right->map.m_pa &&
|
||||
left->map.m_la == right->map.m_la &&
|
||||
left->map.m_plen == right->map.m_plen &&
|
||||
left->map.m_llen == right->map.m_llen &&
|
||||
left->map.m_deviceid == right->map.m_deviceid &&
|
||||
left->map.m_algorithmformat == right->map.m_algorithmformat &&
|
||||
left->map.m_flags == right->map.m_flags);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_init(struct cache *cache)
|
||||
{
|
||||
memset(cache, 0, sizeof(*cache));
|
||||
check_pthread(pthread_mutex_init(&cache->lock, NULL), "pthread_mutex_init");
|
||||
check_pthread(pthread_cond_init(&cache->cv, NULL), "pthread_cond_init");
|
||||
}
|
||||
|
||||
static enum cache_lookup
|
||||
cache_claim(struct cache *cache, const struct cache_key *key,
|
||||
unsigned char *output, int *errorp)
|
||||
{
|
||||
unsigned char *old;
|
||||
enum cache_lookup result;
|
||||
|
||||
*errorp = 0;
|
||||
old = NULL;
|
||||
check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock");
|
||||
if (cache->closing)
|
||||
goto bypass;
|
||||
if (cache->state != CACHE_EMPTY && key_equal(&cache->key, key)) {
|
||||
switch (cache->state) {
|
||||
case CACHE_READY:
|
||||
if (cache->data == NULL)
|
||||
fail("READY state has no data");
|
||||
memcpy(output, cache->data, key->decoded_size);
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock");
|
||||
return (CACHE_HIT);
|
||||
case CACHE_INFLIGHT:
|
||||
++cache->waiters;
|
||||
do {
|
||||
check_pthread(pthread_cond_wait(&cache->cv, &cache->lock),
|
||||
"pthread_cond_wait");
|
||||
} while (cache->state == CACHE_INFLIGHT);
|
||||
if (!key_equal(&cache->key, key))
|
||||
fail("inflight key changed before waiter consumed it");
|
||||
if (cache->state == CACHE_READY) {
|
||||
if (cache->data == NULL)
|
||||
fail("published success has no data");
|
||||
memcpy(output, cache->data, key->decoded_size);
|
||||
result = CACHE_HIT;
|
||||
} else {
|
||||
if (cache->state != CACHE_FAILED || cache->error <= 0)
|
||||
fail("published failure has no positive errno");
|
||||
*errorp = cache->error;
|
||||
result = CACHE_ERROR;
|
||||
}
|
||||
--cache->waiters;
|
||||
if (cache->state == CACHE_FAILED && cache->waiters == 0) {
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
if (cache->waiters == 0)
|
||||
check_pthread(pthread_cond_broadcast(&cache->cv),
|
||||
"pthread_cond_broadcast");
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock");
|
||||
return (result);
|
||||
case CACHE_FAILED:
|
||||
if (cache->waiters != 0)
|
||||
goto bypass;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
break;
|
||||
case CACHE_EMPTY:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (cache->state == CACHE_INFLIGHT || cache->waiters != 0)
|
||||
goto bypass;
|
||||
old = cache->data;
|
||||
cache->data = NULL;
|
||||
cache->key = *key;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_INFLIGHT;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
free(old);
|
||||
return (CACHE_OWNER);
|
||||
|
||||
bypass:
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
return (CACHE_BYPASS);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_complete(struct cache *cache, const struct cache_key *key,
|
||||
unsigned char *data, int error)
|
||||
{
|
||||
if ((error == 0) != (data != NULL))
|
||||
fail("owner completion is not typed");
|
||||
check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock");
|
||||
if (cache->state != CACHE_INFLIGHT || !key_equal(&cache->key, key))
|
||||
fail("owner lost its inflight key");
|
||||
if (error == 0) {
|
||||
cache->data = data;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_READY;
|
||||
} else {
|
||||
if (error < 1)
|
||||
fail("owner published a non-positive errno");
|
||||
cache->error = error;
|
||||
cache->state = CACHE_FAILED;
|
||||
if (cache->waiters == 0) {
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
}
|
||||
check_pthread(pthread_cond_broadcast(&cache->cv),
|
||||
"pthread_cond_broadcast");
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
}
|
||||
|
||||
static unsigned char *
|
||||
copy_payload(const unsigned char *payload, size_t size)
|
||||
{
|
||||
unsigned char *copy;
|
||||
|
||||
copy = malloc(size);
|
||||
if (copy == NULL)
|
||||
fail("malloc failed");
|
||||
memcpy(copy, payload, size);
|
||||
return (copy);
|
||||
}
|
||||
|
||||
static void
|
||||
wait_for_waiters(struct cache *cache, unsigned int expected)
|
||||
{
|
||||
struct timespec delay = { .tv_sec = 0, .tv_nsec = 1000000 };
|
||||
unsigned int attempt, waiters;
|
||||
|
||||
for (attempt = 0; attempt < 5000; ++attempt) {
|
||||
check_pthread(pthread_mutex_lock(&cache->lock),
|
||||
"pthread_mutex_lock");
|
||||
waiters = cache->waiters;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock),
|
||||
"pthread_mutex_unlock");
|
||||
if (waiters == expected)
|
||||
return;
|
||||
nanosleep(&delay, NULL);
|
||||
}
|
||||
fail("workers did not register as same-key waiters");
|
||||
}
|
||||
|
||||
static void *
|
||||
wave_worker(void *opaque)
|
||||
{
|
||||
struct worker_arg *arg;
|
||||
struct wave *wave;
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int barrier_result, error;
|
||||
|
||||
arg = opaque;
|
||||
wave = arg->wave;
|
||||
barrier_result = pthread_barrier_wait(&wave->barrier);
|
||||
if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD)
|
||||
check_pthread(barrier_result, "pthread_barrier_wait");
|
||||
lookup = cache_claim(wave->cache, &wave->key, output, &error);
|
||||
if (lookup == CACHE_OWNER) {
|
||||
atomic_fetch_add_explicit(&wave->decodes, 1, memory_order_relaxed);
|
||||
wait_for_waiters(wave->cache, B32_WORKERS - 1);
|
||||
if (wave->expected_error != 0) {
|
||||
cache_complete(wave->cache, &wave->key, NULL,
|
||||
wave->expected_error);
|
||||
wave->results[arg->index] = wave->expected_error;
|
||||
} else {
|
||||
memcpy(output, wave->payload, sizeof(output));
|
||||
cache_complete(wave->cache, &wave->key,
|
||||
copy_payload(wave->payload, sizeof(wave->payload)), 0);
|
||||
wave->results[arg->index] =
|
||||
memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO;
|
||||
}
|
||||
} else if (lookup == CACHE_HIT) {
|
||||
wave->results[arg->index] =
|
||||
memcmp(output, wave->payload, sizeof(output)) == 0 ? 0 : EIO;
|
||||
} else if (lookup == CACHE_ERROR) {
|
||||
wave->results[arg->index] = error;
|
||||
} else {
|
||||
atomic_fetch_add_explicit(&wave->bypasses, 1, memory_order_relaxed);
|
||||
wave->results[arg->index] = EBUSY;
|
||||
}
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
run_wave(struct cache *cache, const struct cache_key *key, int expected_error)
|
||||
{
|
||||
struct wave wave;
|
||||
struct worker_arg args[B32_WORKERS];
|
||||
pthread_t threads[B32_WORKERS];
|
||||
unsigned int index;
|
||||
|
||||
memset(&wave, 0, sizeof(wave));
|
||||
wave.cache = cache;
|
||||
wave.key = *key;
|
||||
wave.expected_error = expected_error;
|
||||
for (index = 0; index < sizeof(wave.payload); ++index)
|
||||
wave.payload[index] = (unsigned char)(0xa0U + index);
|
||||
check_pthread(pthread_barrier_init(&wave.barrier, NULL, B32_WORKERS),
|
||||
"pthread_barrier_init");
|
||||
for (index = 0; index < B32_WORKERS; ++index) {
|
||||
args[index].wave = &wave;
|
||||
args[index].index = index;
|
||||
check_pthread(pthread_create(&threads[index], NULL, wave_worker,
|
||||
&args[index]), "pthread_create");
|
||||
}
|
||||
for (index = 0; index < B32_WORKERS; ++index)
|
||||
check_pthread(pthread_join(threads[index], NULL), "pthread_join");
|
||||
check_pthread(pthread_barrier_destroy(&wave.barrier),
|
||||
"pthread_barrier_destroy");
|
||||
if (atomic_load_explicit(&wave.decodes, memory_order_relaxed) != 1)
|
||||
fail("same-key wave had more than one owner decode");
|
||||
if (atomic_load_explicit(&wave.bypasses, memory_order_relaxed) != 0)
|
||||
fail("same-key wave bypassed its owner generation");
|
||||
for (index = 0; index < B32_WORKERS; ++index) {
|
||||
if (wave.results[index] != expected_error)
|
||||
fail("same-key waiter received different bytes or errno");
|
||||
}
|
||||
}
|
||||
|
||||
static struct cache_key
|
||||
base_key(void)
|
||||
{
|
||||
return ((struct cache_key) {
|
||||
.map = {
|
||||
.m_pa = 0x1000,
|
||||
.m_la = 0x2000,
|
||||
.m_plen = 4096,
|
||||
.m_llen = B32_DATA_SIZE,
|
||||
.m_deviceid = 3,
|
||||
.m_algorithmformat = 1,
|
||||
.m_flags = 1,
|
||||
},
|
||||
.nid = 42,
|
||||
.decoded_size = B32_DATA_SIZE,
|
||||
});
|
||||
}
|
||||
|
||||
static void
|
||||
seed_ready(struct cache *cache, const struct cache_key *key)
|
||||
{
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int error;
|
||||
|
||||
memset(payload, 0x5a, sizeof(payload));
|
||||
lookup = cache_claim(cache, key, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("seed miss did not become owner");
|
||||
cache_complete(cache, key, copy_payload(payload, sizeof(payload)), 0);
|
||||
}
|
||||
|
||||
static void
|
||||
expect_distinct_key(struct cache *cache, const struct cache_key *base,
|
||||
const struct cache_key *changed)
|
||||
{
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int error;
|
||||
|
||||
lookup = cache_claim(cache, changed, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("changed key incorrectly hit or bypassed");
|
||||
memset(payload, 0x33, sizeof(payload));
|
||||
cache_complete(cache, changed, copy_payload(payload, sizeof(payload)), 0);
|
||||
lookup = cache_claim(cache, base, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("key reuse did not start a fresh owner generation");
|
||||
memset(payload, 0x5a, sizeof(payload));
|
||||
cache_complete(cache, base, copy_payload(payload, sizeof(payload)), 0);
|
||||
}
|
||||
|
||||
static void
|
||||
test_exact_key(struct cache *cache, const struct cache_key *base)
|
||||
{
|
||||
struct cache_key changed;
|
||||
|
||||
seed_ready(cache, base);
|
||||
changed = *base;
|
||||
++changed.nid;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
--changed.decoded_size;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_pa;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_la;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_plen;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_llen;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_deviceid;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
++changed.map.m_algorithmformat;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
changed = *base;
|
||||
changed.map.m_flags ^= 2U;
|
||||
expect_distinct_key(cache, base, &changed);
|
||||
}
|
||||
|
||||
static void
|
||||
test_fallback_and_waiter_generation(struct cache *cache,
|
||||
const struct cache_key *base)
|
||||
{
|
||||
struct cache_key changed;
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int error;
|
||||
|
||||
changed = *base;
|
||||
++changed.nid;
|
||||
lookup = cache_claim(cache, base, output, &error);
|
||||
if (lookup != CACHE_HIT)
|
||||
fail("ready base key was not retained");
|
||||
lookup = cache_claim(cache, &changed, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("ready eviction did not create a new owner");
|
||||
lookup = cache_claim(cache, base, output, &error);
|
||||
if (lookup != CACHE_BYPASS)
|
||||
fail("different key did not use no-cache inflight fallback");
|
||||
memset(payload, 0x7c, sizeof(payload));
|
||||
cache_complete(cache, &changed, copy_payload(payload, sizeof(payload)), 0);
|
||||
|
||||
check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock");
|
||||
cache->waiters = 1;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
lookup = cache_claim(cache, base, output, &error);
|
||||
if (lookup != CACHE_BYPASS)
|
||||
fail("published generation was evicted before waiter consumption");
|
||||
check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock");
|
||||
cache->waiters = 0;
|
||||
check_pthread(pthread_cond_broadcast(&cache->cv),
|
||||
"pthread_cond_broadcast");
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
}
|
||||
|
||||
static void *
|
||||
fini_worker(void *opaque)
|
||||
{
|
||||
struct fini_arg *arg;
|
||||
struct cache *cache;
|
||||
unsigned char *data;
|
||||
|
||||
arg = opaque;
|
||||
cache = arg->cache;
|
||||
check_pthread(pthread_mutex_lock(&cache->lock), "pthread_mutex_lock");
|
||||
cache->closing = true;
|
||||
while (cache->state == CACHE_INFLIGHT || cache->waiters != 0)
|
||||
check_pthread(pthread_cond_wait(&cache->cv, &cache->lock),
|
||||
"pthread_cond_wait");
|
||||
data = cache->data;
|
||||
cache->data = NULL;
|
||||
cache->error = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
check_pthread(pthread_mutex_unlock(&cache->lock), "pthread_mutex_unlock");
|
||||
free(data);
|
||||
atomic_store_explicit(&arg->done, true, memory_order_release);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
test_shutdown(struct cache *cache, const struct cache_key *key)
|
||||
{
|
||||
struct fini_arg arg;
|
||||
struct timespec delay = { .tv_sec = 0, .tv_nsec = 20000000 };
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
pthread_t thread;
|
||||
enum cache_lookup lookup;
|
||||
int error;
|
||||
|
||||
lookup = cache_claim(cache, key, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("shutdown setup did not create inflight owner");
|
||||
arg.cache = cache;
|
||||
atomic_init(&arg.done, false);
|
||||
check_pthread(pthread_create(&thread, NULL, fini_worker, &arg),
|
||||
"pthread_create");
|
||||
nanosleep(&delay, NULL);
|
||||
if (atomic_load_explicit(&arg.done, memory_order_acquire))
|
||||
fail("shutdown did not wait for inflight owner");
|
||||
memset(payload, 0x91, sizeof(payload));
|
||||
cache_complete(cache, key, copy_payload(payload, sizeof(payload)), 0);
|
||||
check_pthread(pthread_join(thread, NULL), "pthread_join");
|
||||
if (!atomic_load_explicit(&arg.done, memory_order_acquire))
|
||||
fail("shutdown did not finish after owner publication");
|
||||
check_pthread(pthread_cond_destroy(&cache->cv), "pthread_cond_destroy");
|
||||
check_pthread(pthread_mutex_destroy(&cache->lock), "pthread_mutex_destroy");
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
struct cache cache;
|
||||
struct cache_key key, failure_key, retry_key;
|
||||
unsigned char output[B32_DATA_SIZE];
|
||||
unsigned char payload[B32_DATA_SIZE];
|
||||
enum cache_lookup lookup;
|
||||
int error;
|
||||
|
||||
cache_init(&cache);
|
||||
key = base_key();
|
||||
run_wave(&cache, &key, 0);
|
||||
failure_key = key;
|
||||
++failure_key.nid;
|
||||
run_wave(&cache, &failure_key, EIO);
|
||||
lookup = cache_claim(&cache, &failure_key, output, &error);
|
||||
if (lookup != CACHE_OWNER)
|
||||
fail("published failure did not become retryable");
|
||||
memset(payload, 0x44, sizeof(payload));
|
||||
cache_complete(&cache, &failure_key,
|
||||
copy_payload(payload, sizeof(payload)), 0);
|
||||
|
||||
retry_key = key;
|
||||
retry_key.nid += 2;
|
||||
test_exact_key(&cache, &retry_key);
|
||||
test_fallback_and_waiter_generation(&cache, &retry_key);
|
||||
retry_key.nid += 3;
|
||||
test_shutdown(&cache, &retry_key);
|
||||
printf("TC168-cache-inflight host oracle PASS: one owner, exact key, typed failure, retry, fallback, eviction, shutdown\n");
|
||||
return (0);
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <pthread.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
struct concurrent_ctx {
|
||||
const unsigned char *expected;
|
||||
size_t length;
|
||||
off_t offset;
|
||||
long loops;
|
||||
int actual_fd;
|
||||
int expected_errno;
|
||||
pthread_barrier_t barrier;
|
||||
pthread_mutex_t lock;
|
||||
int failure;
|
||||
ssize_t mismatch_count;
|
||||
int mismatch_errno;
|
||||
};
|
||||
|
||||
struct vnode_ctx {
|
||||
const char *actual_path;
|
||||
const unsigned char *expected;
|
||||
size_t expected_size;
|
||||
long loops;
|
||||
pthread_barrier_t barrier;
|
||||
pthread_mutex_t lock;
|
||||
int failure;
|
||||
};
|
||||
|
||||
static int
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "B32 probe failure: %s\n", message);
|
||||
return (1);
|
||||
}
|
||||
|
||||
static long
|
||||
parse_long(const char *text, long minimum, long maximum)
|
||||
{
|
||||
char *end;
|
||||
long value;
|
||||
|
||||
errno = 0;
|
||||
value = strtol(text, &end, 10);
|
||||
if (errno != 0 || *text == '\0' || *end != '\0' || value < minimum ||
|
||||
value > maximum)
|
||||
return (-1);
|
||||
return (value);
|
||||
}
|
||||
|
||||
static unsigned char *
|
||||
read_expected(const char *path, off_t offset, size_t length)
|
||||
{
|
||||
unsigned char *buffer;
|
||||
ssize_t count;
|
||||
int descriptor;
|
||||
|
||||
descriptor = open(path, O_RDONLY);
|
||||
if (descriptor < 0)
|
||||
return (NULL);
|
||||
buffer = malloc(length);
|
||||
if (buffer == NULL) {
|
||||
close(descriptor);
|
||||
return (NULL);
|
||||
}
|
||||
count = pread(descriptor, buffer, length, offset);
|
||||
close(descriptor);
|
||||
if (count != (ssize_t)length) {
|
||||
free(buffer);
|
||||
return (NULL);
|
||||
}
|
||||
return (buffer);
|
||||
}
|
||||
|
||||
static void
|
||||
record_failure(pthread_mutex_t *lock, int *failure, int error)
|
||||
{
|
||||
pthread_mutex_lock(lock);
|
||||
if (*failure == 0)
|
||||
*failure = error != 0 ? error : EIO;
|
||||
pthread_mutex_unlock(lock);
|
||||
}
|
||||
|
||||
static void
|
||||
record_concurrent_failure(struct concurrent_ctx *ctx, ssize_t count,
|
||||
int saved_errno)
|
||||
{
|
||||
|
||||
pthread_mutex_lock(&ctx->lock);
|
||||
if (ctx->failure == 0) {
|
||||
ctx->failure = saved_errno != 0 ? saved_errno : EIO;
|
||||
ctx->mismatch_count = count;
|
||||
ctx->mismatch_errno = saved_errno;
|
||||
}
|
||||
pthread_mutex_unlock(&ctx->lock);
|
||||
}
|
||||
|
||||
static void *
|
||||
concurrent_worker(void *opaque)
|
||||
{
|
||||
struct concurrent_ctx *ctx;
|
||||
unsigned char *buffer;
|
||||
ssize_t count;
|
||||
long loop;
|
||||
int barrier_result, saved_errno;
|
||||
|
||||
ctx = opaque;
|
||||
buffer = malloc(ctx->length);
|
||||
if (buffer == NULL) {
|
||||
record_failure(&ctx->lock, &ctx->failure, ENOMEM);
|
||||
return (NULL);
|
||||
}
|
||||
barrier_result = pthread_barrier_wait(&ctx->barrier);
|
||||
if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD) {
|
||||
record_failure(&ctx->lock, &ctx->failure, barrier_result);
|
||||
free(buffer);
|
||||
return (NULL);
|
||||
}
|
||||
for (loop = 0; loop < ctx->loops; ++loop) {
|
||||
errno = 0;
|
||||
count = pread(ctx->actual_fd, buffer, ctx->length, ctx->offset);
|
||||
saved_errno = errno;
|
||||
if (ctx->expected_errno != 0) {
|
||||
if (count != -1 || saved_errno != ctx->expected_errno) {
|
||||
record_concurrent_failure(ctx, count, saved_errno);
|
||||
break;
|
||||
}
|
||||
} else if (count != (ssize_t)ctx->length ||
|
||||
memcmp(buffer, ctx->expected, ctx->length) != 0) {
|
||||
record_concurrent_failure(ctx, count, saved_errno);
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(buffer);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static int
|
||||
run_concurrent(const char *actual_path, const char *expected_path,
|
||||
long expected_errno, long workers, long loops, off_t offset, size_t length)
|
||||
{
|
||||
struct concurrent_ctx ctx;
|
||||
pthread_t *threads;
|
||||
unsigned char *expected;
|
||||
long index;
|
||||
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
expected = expected_errno == 0 ?
|
||||
read_expected(expected_path, offset, length) : calloc(1, length);
|
||||
if (expected == NULL)
|
||||
return (fail("could not prepare expected bytes"));
|
||||
ctx.expected = expected;
|
||||
ctx.length = length;
|
||||
ctx.offset = offset;
|
||||
ctx.loops = loops;
|
||||
ctx.expected_errno = (int)expected_errno;
|
||||
ctx.actual_fd = open(actual_path, O_RDONLY);
|
||||
if (ctx.actual_fd < 0) {
|
||||
free(expected);
|
||||
return (fail("could not open actual file"));
|
||||
}
|
||||
if (pthread_barrier_init(&ctx.barrier, NULL, (unsigned int)workers) != 0 ||
|
||||
pthread_mutex_init(&ctx.lock, NULL) != 0) {
|
||||
close(ctx.actual_fd);
|
||||
free(expected);
|
||||
return (fail("could not initialize concurrent controls"));
|
||||
}
|
||||
threads = calloc((size_t)workers, sizeof(*threads));
|
||||
if (threads == NULL)
|
||||
return (fail("could not allocate worker handles"));
|
||||
for (index = 0; index < workers; ++index) {
|
||||
if (pthread_create(&threads[index], NULL, concurrent_worker, &ctx) != 0)
|
||||
return (fail("could not create concurrent worker"));
|
||||
}
|
||||
for (index = 0; index < workers; ++index) {
|
||||
if (pthread_join(threads[index], NULL) != 0)
|
||||
return (fail("could not join concurrent worker"));
|
||||
}
|
||||
free(threads);
|
||||
pthread_barrier_destroy(&ctx.barrier);
|
||||
pthread_mutex_destroy(&ctx.lock);
|
||||
close(ctx.actual_fd);
|
||||
free(expected);
|
||||
if (ctx.failure != 0) {
|
||||
fprintf(stderr,
|
||||
"concurrent mismatch count=%zd errno=%d expected_errno=%d\n",
|
||||
ctx.mismatch_count, ctx.mismatch_errno, ctx.expected_errno);
|
||||
errno = ctx.failure;
|
||||
perror("concurrent read");
|
||||
return (1);
|
||||
}
|
||||
printf("concurrent PASS workers=%ld loops=%ld assertions=%ld errno=%ld "
|
||||
"offset=%ld length=%ld\n", workers, loops, workers * loops,
|
||||
expected_errno, (long)offset, (long)length);
|
||||
return (0);
|
||||
}
|
||||
|
||||
static void *
|
||||
vnode_worker(void *opaque)
|
||||
{
|
||||
struct vnode_ctx *ctx;
|
||||
unsigned char buffer[4096];
|
||||
struct stat status;
|
||||
ssize_t count;
|
||||
long loop;
|
||||
int barrier_result, descriptor, saved_errno;
|
||||
|
||||
ctx = opaque;
|
||||
barrier_result = pthread_barrier_wait(&ctx->barrier);
|
||||
if (barrier_result != 0 && barrier_result != PTHREAD_BARRIER_SERIAL_THREAD) {
|
||||
record_failure(&ctx->lock, &ctx->failure, barrier_result);
|
||||
return (NULL);
|
||||
}
|
||||
for (loop = 0; loop < ctx->loops; ++loop) {
|
||||
descriptor = open(ctx->actual_path, O_RDONLY);
|
||||
if (descriptor < 0) {
|
||||
record_failure(&ctx->lock, &ctx->failure, errno);
|
||||
break;
|
||||
}
|
||||
if (fstat(descriptor, &status) != 0 ||
|
||||
(size_t)status.st_size != ctx->expected_size) {
|
||||
saved_errno = errno;
|
||||
close(descriptor);
|
||||
record_failure(&ctx->lock, &ctx->failure,
|
||||
saved_errno != 0 ? saved_errno : EIO);
|
||||
break;
|
||||
}
|
||||
count = pread(descriptor, buffer, sizeof(buffer), 0);
|
||||
saved_errno = errno;
|
||||
close(descriptor);
|
||||
if (count != (ssize_t)sizeof(buffer) ||
|
||||
memcmp(buffer, ctx->expected, sizeof(buffer)) != 0) {
|
||||
record_failure(&ctx->lock, &ctx->failure,
|
||||
saved_errno != 0 ? saved_errno : EIO);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static int
|
||||
run_vnode_race(const char *actual_path, const char *expected_path, long workers,
|
||||
long loops)
|
||||
{
|
||||
struct vnode_ctx ctx;
|
||||
struct stat status;
|
||||
pthread_t *threads;
|
||||
long index;
|
||||
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
if (stat(expected_path, &status) != 0 || status.st_size < 4096)
|
||||
return (fail("could not stat vnode-race reference"));
|
||||
ctx.actual_path = actual_path;
|
||||
ctx.expected_size = (size_t)status.st_size;
|
||||
ctx.expected = read_expected(expected_path, 0, 4096);
|
||||
ctx.loops = loops;
|
||||
if (ctx.expected == NULL)
|
||||
return (fail("could not read vnode-race reference"));
|
||||
if (pthread_barrier_init(&ctx.barrier, NULL, (unsigned int)workers) != 0 ||
|
||||
pthread_mutex_init(&ctx.lock, NULL) != 0)
|
||||
return (fail("could not initialize vnode-race controls"));
|
||||
threads = calloc((size_t)workers, sizeof(*threads));
|
||||
if (threads == NULL)
|
||||
return (fail("could not allocate vnode-race handles"));
|
||||
for (index = 0; index < workers; ++index) {
|
||||
if (pthread_create(&threads[index], NULL, vnode_worker, &ctx) != 0)
|
||||
return (fail("could not create vnode-race worker"));
|
||||
}
|
||||
for (index = 0; index < workers; ++index) {
|
||||
if (pthread_join(threads[index], NULL) != 0)
|
||||
return (fail("could not join vnode-race worker"));
|
||||
}
|
||||
free(threads);
|
||||
pthread_barrier_destroy(&ctx.barrier);
|
||||
pthread_mutex_destroy(&ctx.lock);
|
||||
free((void *)ctx.expected);
|
||||
if (ctx.failure != 0) {
|
||||
errno = ctx.failure;
|
||||
perror("vnode race");
|
||||
return (1);
|
||||
}
|
||||
printf("vnode-race PASS workers=%ld loops=%ld assertions=%ld\n", workers,
|
||||
loops, workers * loops);
|
||||
return (0);
|
||||
}
|
||||
|
||||
static bool
|
||||
marker_exists(const char *directory, const char *name)
|
||||
{
|
||||
char path[1024];
|
||||
|
||||
if (snprintf(path, sizeof(path), "%s/%s", directory, name) >=
|
||||
(int)sizeof(path))
|
||||
return (false);
|
||||
return (access(path, F_OK) == 0);
|
||||
}
|
||||
|
||||
static int
|
||||
write_marker(const char *directory, const char *name, const char *content)
|
||||
{
|
||||
char path[1024];
|
||||
ssize_t length;
|
||||
int descriptor;
|
||||
|
||||
if (snprintf(path, sizeof(path), "%s/%s", directory, name) >=
|
||||
(int)sizeof(path))
|
||||
return (-1);
|
||||
descriptor = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
if (descriptor < 0)
|
||||
return (-1);
|
||||
length = (ssize_t)strlen(content);
|
||||
if (write(descriptor, content, (size_t)length) != length) {
|
||||
close(descriptor);
|
||||
return (-1);
|
||||
}
|
||||
return (close(descriptor));
|
||||
}
|
||||
|
||||
static int
|
||||
run_lifecycle(const char *path, const char *directory)
|
||||
{
|
||||
struct timespec delay = { .tv_sec = 0, .tv_nsec = 10000000 };
|
||||
char result[128];
|
||||
ssize_t count;
|
||||
long attempt;
|
||||
int descriptor, saved_errno;
|
||||
bool closed;
|
||||
|
||||
descriptor = open(path, O_RDONLY);
|
||||
if (descriptor < 0)
|
||||
return (fail("lifecycle open failed"));
|
||||
if (write_marker(directory, "ready", "ready\n") != 0)
|
||||
return (fail("lifecycle ready marker failed"));
|
||||
closed = false;
|
||||
for (attempt = 0; attempt < 18000; ++attempt) {
|
||||
if (!closed && marker_exists(directory, "close")) {
|
||||
if (close(descriptor) != 0)
|
||||
return (fail("lifecycle close failed"));
|
||||
closed = true;
|
||||
if (write_marker(directory, "closed", "closed\n") != 0)
|
||||
return (fail("lifecycle closed marker failed"));
|
||||
}
|
||||
if (marker_exists(directory, "read")) {
|
||||
errno = 0;
|
||||
count = pread(descriptor, result, sizeof(result), 0);
|
||||
saved_errno = errno;
|
||||
if (snprintf(result, sizeof(result), "%zd %d\n", count,
|
||||
saved_errno) >= (int)sizeof(result) ||
|
||||
write_marker(directory, "result", result) != 0)
|
||||
return (fail("lifecycle result marker failed"));
|
||||
if (!closed)
|
||||
close(descriptor);
|
||||
return (0);
|
||||
}
|
||||
if (marker_exists(directory, "exit")) {
|
||||
if (!closed)
|
||||
close(descriptor);
|
||||
return (0);
|
||||
}
|
||||
nanosleep(&delay, NULL);
|
||||
}
|
||||
if (!closed)
|
||||
close(descriptor);
|
||||
return (fail("lifecycle control timed out"));
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
long expected_errno, workers, loops, offset, length;
|
||||
|
||||
if (argc == 9 && strcmp(argv[1], "concurrent") == 0) {
|
||||
expected_errno = parse_long(argv[4], 0, 255);
|
||||
workers = parse_long(argv[5], 1, 256);
|
||||
loops = parse_long(argv[6], 1, 10000);
|
||||
offset = parse_long(argv[7], 0, INT64_MAX);
|
||||
length = parse_long(argv[8], 1, 1048576);
|
||||
if (expected_errno < 0 || workers < 0 || loops < 0 || offset < 0 ||
|
||||
length < 0)
|
||||
return (fail("invalid concurrent arguments"));
|
||||
return (run_concurrent(argv[2], argv[3], expected_errno, workers,
|
||||
loops, (off_t)offset, (size_t)length));
|
||||
}
|
||||
if (argc == 6 && strcmp(argv[1], "vnode-race") == 0) {
|
||||
workers = parse_long(argv[4], 1, 256);
|
||||
loops = parse_long(argv[5], 1, 10000);
|
||||
if (workers < 0 || loops < 0)
|
||||
return (fail("invalid vnode-race arguments"));
|
||||
return (run_vnode_race(argv[2], argv[3], workers, loops));
|
||||
}
|
||||
if (argc == 4 && strcmp(argv[1], "lifecycle") == 0)
|
||||
return (run_lifecycle(argv[2], argv[3]));
|
||||
return (fail("usage: concurrent ACTUAL EXPECTED ERRNO WORKERS LOOPS OFFSET LENGTH | vnode-race ACTUAL EXPECTED WORKERS LOOPS | lifecycle PATH CONTROL_DIR"));
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"baseline": "3000e04b230601dd39b6747032f7ad5dcced0ca4",
|
||||
"batch": "B32",
|
||||
"candidate": [
|
||||
"P15-075",
|
||||
"P15-090"
|
||||
],
|
||||
"key_fields": [
|
||||
"nid",
|
||||
"decoded_size",
|
||||
"m_pa",
|
||||
"m_la",
|
||||
"m_plen",
|
||||
"m_llen",
|
||||
"m_deviceid",
|
||||
"m_algorithmformat",
|
||||
"m_flags"
|
||||
],
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U32000000-0000-4000-8000-000000000075",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zlzma,level=6,dictsize=65536",
|
||||
"-C4096"
|
||||
],
|
||||
"schema": 1,
|
||||
"source": {
|
||||
"sha256": "082153d3c28ac1e3588deaabd9607a7b274d093ab9b9900060888ffc806934bb",
|
||||
"size": 65536
|
||||
},
|
||||
"states": [
|
||||
"EMPTY",
|
||||
"INFLIGHT",
|
||||
"READY",
|
||||
"FAILED"
|
||||
],
|
||||
"test": "TC168-cache-inflight"
|
||||
}
|
||||
Executable
+306
@@ -0,0 +1,306 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B32-qemu-run.sh
|
||||
|
||||
if test "${1:-}" != --worker; then
|
||||
test "$#" -eq 1 || {
|
||||
printf '%s\n' 'usage: B32-qemu-run.sh TC168-cache-inflight|TC184-vnode-races' >&2
|
||||
exit 20
|
||||
}
|
||||
case "$1" in
|
||||
TC168-cache-inflight|TC184-vnode-races) ;;
|
||||
*) printf 'unknown B32 QEMU scenario: %s\n' "$1" >&2; exit 20 ;;
|
||||
esac
|
||||
total=${PRE15_QEMU_TIMEOUT:-1200}
|
||||
case "$total" in
|
||||
''|*[!0-9]*|0) printf '%s\n' 'invalid PRE15_QEMU_TIMEOUT' >&2; exit 20 ;;
|
||||
esac
|
||||
exec timeout -k 30 "$total" "$script" --worker "$1"
|
||||
fi
|
||||
|
||||
shift
|
||||
test "$#" -eq 1 || exit 20
|
||||
scenario=$1
|
||||
: "${PRE15_EVIDENCE_ROOT:?PRE15_EVIDENCE_ROOT is required}"
|
||||
: "${PRE15_QEMU_BASE_IMAGE:?PRE15_QEMU_BASE_IMAGE is required}"
|
||||
: "${PRE15_QEMU_SSH_KEY:?PRE15_QEMU_SSH_KEY is required}"
|
||||
|
||||
dut=$(CDPATH= cd -- "$(dirname -- "$script")/../../.." && pwd -P)
|
||||
root=$(CDPATH= cd -- "$dut/.." && pwd -P)
|
||||
lib=$dut/tests/pre15/lib
|
||||
case_script=$dut/tests/pre15/cases/B32-cache-state.sh
|
||||
freebsd_src=${PRE15_FREEBSD_SRC:-${FREEBSD_SRC:-/work/build/freebsd-src}}
|
||||
user=${PRE15_QEMU_SSH_USER:-root}
|
||||
boot_timeout=${PRE15_QEMU_BOOT_TIMEOUT:-180}
|
||||
guest_timeout=${PRE15_GUEST_COMMAND_TIMEOUT:-60}
|
||||
case_timeout=${PRE15_B32_QEMU_CASE_TIMEOUT:-900}
|
||||
run_id=$(date -u '+%Y%m%dT%H%M%SZ')-qemu-B32-$scenario-$$
|
||||
run=$PRE15_EVIDENCE_ROOT/$run_id
|
||||
case_run=$run/case
|
||||
case_tmp=$run/case-tmp
|
||||
overlay=$run/guest-overlay.qcow2
|
||||
control=$run/ssh-control
|
||||
serial=$run/serial.log
|
||||
qemu_log=$run/qemu.log
|
||||
ownership=$case_run/ownership.tsv
|
||||
cleanup_log=$case_run/cleanup.log
|
||||
target_marker=$case_run/target.marker
|
||||
reason_file=$case_run/reason.txt
|
||||
fixture_hashes=$case_run/fixture-hashes.tsv
|
||||
module_hash=$case_run/module-hash.tsv
|
||||
qemu_pid=
|
||||
port=
|
||||
ssh_ready=0
|
||||
cleanup_done=0
|
||||
cleanup_status=PASS
|
||||
|
||||
mkdir -p "$case_run" "$case_tmp"
|
||||
: >"$ownership"
|
||||
: >"$fixture_hashes"
|
||||
: >"$module_hash"
|
||||
: >"$reason_file"
|
||||
|
||||
cleanup()
|
||||
{
|
||||
test "$cleanup_done" -eq 0 || return
|
||||
cleanup_done=1
|
||||
if test "$ssh_ready" -eq 1; then
|
||||
PRE15_QEMU_SSH_PORT=$port
|
||||
PRE15_QEMU_SSH_USER=$user
|
||||
PRE15_QEMU_SSH_KEY=$PRE15_QEMU_SSH_KEY
|
||||
PRE15_QEMU_CONTROL_PATH=$control
|
||||
PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout
|
||||
PRE15_OWNERSHIP_FILE=$ownership
|
||||
PRE15_CLEANUP_LOG=$cleanup_log
|
||||
PRE15_RUN_DIR=$case_run
|
||||
export PRE15_QEMU_SSH_PORT PRE15_QEMU_SSH_USER PRE15_QEMU_SSH_KEY
|
||||
export PRE15_QEMU_CONTROL_PATH PRE15_GUEST_COMMAND_TIMEOUT
|
||||
export PRE15_OWNERSHIP_FILE PRE15_CLEANUP_LOG PRE15_RUN_DIR
|
||||
. "$lib/runner.sh"
|
||||
pre15_cleanup_owned || cleanup_status=FAIL
|
||||
test "${PRE15_CLEANUP_STATUS_RESULT:-FAIL}" = PASS || cleanup_status=FAIL
|
||||
else
|
||||
printf '%s\n' 'PASS no guest resources reached before SSH readiness' \
|
||||
>"$cleanup_log"
|
||||
fi
|
||||
if test -S "$control"; then
|
||||
timeout -k 5 15 ssh -S "$control" -O exit -p "${port:-22}" \
|
||||
"$user@127.0.0.1" >>"$run/host-cleanup.log" 2>&1 || true
|
||||
fi
|
||||
if test -n "$qemu_pid" && test "$qemu_pid" != 26318 && \
|
||||
kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
kill -TERM "$qemu_pid" 2>/dev/null || cleanup_status=FAIL
|
||||
wait_count=0
|
||||
while kill -0 "$qemu_pid" 2>/dev/null && test "$wait_count" -lt 100; do
|
||||
sleep 0.1
|
||||
wait_count=$((wait_count + 1))
|
||||
done
|
||||
if kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
kill -KILL "$qemu_pid" 2>/dev/null || cleanup_status=FAIL
|
||||
sleep 0.2
|
||||
fi
|
||||
fi
|
||||
if test -n "$qemu_pid" && kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
cleanup_status=FAIL
|
||||
printf 'FAIL owned QEMU PID survived: %s\n' "$qemu_pid" \
|
||||
>>"$run/host-cleanup.log"
|
||||
else
|
||||
printf 'PASS owned QEMU PID stopped: %s\n' "${qemu_pid:-not-started}" \
|
||||
>>"$run/host-cleanup.log"
|
||||
fi
|
||||
if test -f "$overlay"; then
|
||||
rm -f -- "$overlay" || cleanup_status=FAIL
|
||||
fi
|
||||
test ! -e "$overlay" || cleanup_status=FAIL
|
||||
printf 'overlay_removed=%s\n' "$(test ! -e "$overlay" && printf yes || printf no)" \
|
||||
>>"$run/host-cleanup.log"
|
||||
rm -rf -- "$case_tmp"
|
||||
stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \
|
||||
"$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-after.txt" || \
|
||||
cleanup_status=FAIL
|
||||
if ! cmp -s "$run/base-identity-before.txt" "$run/base-identity-after.txt"; then
|
||||
cleanup_status=FAIL
|
||||
printf '%s\n' 'FAIL protected base metadata changed' \
|
||||
>>"$run/host-cleanup.log"
|
||||
else
|
||||
printf '%s\n' 'PASS protected base metadata unchanged' \
|
||||
>>"$run/host-cleanup.log"
|
||||
fi
|
||||
if test -n "$port"; then
|
||||
if timeout 10s python3 -B - "$port" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(0.2)
|
||||
raise SystemExit(0 if sock.connect_ex(("127.0.0.1", int(sys.argv[1]))) != 0 else 1)
|
||||
PY
|
||||
then
|
||||
printf 'PASS dynamic port free: %s\n' "$port" \
|
||||
>>"$run/host-cleanup.log"
|
||||
else
|
||||
cleanup_status=FAIL
|
||||
printf 'FAIL dynamic port remains open: %s\n' "$port" \
|
||||
>>"$run/host-cleanup.log"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$cleanup_status" >"$run/cleanup.status"
|
||||
}
|
||||
|
||||
finish()
|
||||
{
|
||||
rc=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
cleanup
|
||||
if test "$cleanup_status" != PASS; then
|
||||
printf '%s\n' 'RUNNER_FAIL owned cleanup or base metadata check failed' \
|
||||
>>"$run/RESULT.txt"
|
||||
rc=20
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
trap finish EXIT HUP INT TERM
|
||||
|
||||
for tool in cmp qemu-img qemu-system-x86_64 ssh stat timeout; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'INFRA_BLOCKED missing QEMU tool: %s\n' "$tool" >"$run/RESULT.txt"
|
||||
exit 21
|
||||
}
|
||||
done
|
||||
test -f "$PRE15_QEMU_BASE_IMAGE" || {
|
||||
printf '%s\n' 'INFRA_BLOCKED protected base image is absent' >"$run/RESULT.txt"
|
||||
exit 21
|
||||
}
|
||||
test -f "$PRE15_QEMU_SSH_KEY" || {
|
||||
printf '%s\n' 'INFRA_BLOCKED SSH key is absent' >"$run/RESULT.txt"
|
||||
exit 21
|
||||
}
|
||||
|
||||
stat -c 'path=%n size=%s mode=%a inode=%i mtime=%Y ctime=%Z' \
|
||||
"$PRE15_QEMU_BASE_IMAGE" >"$run/base-identity-before.txt"
|
||||
timeout -k 5 30 qemu-img info --output=json "$PRE15_QEMU_BASE_IMAGE" \
|
||||
>"$run/base-qemu-img-info.json"
|
||||
timeout -k 5 30 qemu-img create -f qcow2 -F qcow2 \
|
||||
-b "$PRE15_QEMU_BASE_IMAGE" "$overlay" >"$run/qemu-img-create.stdout" \
|
||||
2>"$run/qemu-img-create.stderr"
|
||||
|
||||
port=$(timeout 10s python3 -B - <<'PY'
|
||||
import socket
|
||||
|
||||
while True:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
if port != 9222:
|
||||
print(port)
|
||||
break
|
||||
PY
|
||||
)
|
||||
python3 -B - "$run/qemu-argv.json" "$overlay" "$port" "$serial" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
argv = [
|
||||
"qemu-system-x86_64", "-accel", "tcg,thread=multi", "-cpu", "qemu64",
|
||||
"-m", "6144", "-smp", "4",
|
||||
"-drive", f"file={sys.argv[2]},if=virtio,format=qcow2",
|
||||
"-netdev", f"user,id=net0,hostfwd=tcp:127.0.0.1:{sys.argv[3]}-:22",
|
||||
"-device", "virtio-net-pci,netdev=net0", "-display", "none",
|
||||
"-serial", f"file:{sys.argv[4]}", "-monitor", "none",
|
||||
]
|
||||
Path(sys.argv[1]).write_text(json.dumps(argv, separators=(",", ":")) + "\n", encoding="ascii")
|
||||
PY
|
||||
|
||||
qemu-system-x86_64 -accel tcg,thread=multi -cpu qemu64 -m 6144 -smp 4 \
|
||||
-drive "file=$overlay,if=virtio,format=qcow2" \
|
||||
-netdev "user,id=net0,hostfwd=tcp:127.0.0.1:$port-:22" \
|
||||
-device virtio-net-pci,netdev=net0 -display none -serial "file:$serial" \
|
||||
-monitor none >"$run/qemu.stdout" 2>"$qemu_log" &
|
||||
qemu_pid=$!
|
||||
printf '%s\n' "$qemu_pid" >"$run/qemu.pid"
|
||||
test "$qemu_pid" != 26318 || {
|
||||
printf '%s\n' 'RUNNER_FAIL QEMU PID collided with protected PID' >"$run/RESULT.txt"
|
||||
exit 20
|
||||
}
|
||||
|
||||
boot_marker=$run/ssh-ready
|
||||
boot_start=$(date -u '+%s')
|
||||
set +e
|
||||
timeout -s KILL "$boot_timeout" env B32_QEMU_PID="$qemu_pid" \
|
||||
B32_QEMU_PORT="$port" B32_QEMU_USER="$user" \
|
||||
B32_QEMU_KEY="$PRE15_QEMU_SSH_KEY" B32_QEMU_MARKER="$boot_marker" \
|
||||
/bin/sh -c '
|
||||
while kill -0 "$B32_QEMU_PID" 2>/dev/null; do
|
||||
if timeout -s KILL 8 ssh -o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -i "$B32_QEMU_KEY" \
|
||||
-p "$B32_QEMU_PORT" "$B32_QEMU_USER@127.0.0.1" true \
|
||||
>/dev/null 2>&1; then
|
||||
: >"$B32_QEMU_MARKER"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
exit 21
|
||||
'
|
||||
boot_rc=$?
|
||||
set -e
|
||||
boot_end=$(date -u '+%s')
|
||||
printf 'start_epoch=%s\nend_epoch=%s\nelapsed_seconds=%s\ndeadline_seconds=%s\nexit=%s\n' \
|
||||
"$boot_start" "$boot_end" "$((boot_end - boot_start))" "$boot_timeout" \
|
||||
"$boot_rc" >"$run/boot-timing.txt"
|
||||
if test -f "$boot_marker"; then
|
||||
ssh_ready=1
|
||||
elif ! kill -0 "$qemu_pid" 2>/dev/null; then
|
||||
printf '%s\n' 'INFRA_BLOCKED QEMU exited before guest SSH' >"$run/RESULT.txt"
|
||||
exit 21
|
||||
else
|
||||
printf 'INFRA_BLOCKED guest SSH not ready within absolute %ss boot deadline\n' \
|
||||
"$boot_timeout" >"$run/RESULT.txt"
|
||||
exit 21
|
||||
fi
|
||||
|
||||
if ! timeout -k 5 30 ssh -MNf -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o ControlMaster=yes \
|
||||
-o "ControlPath=$control" -i "$PRE15_QEMU_SSH_KEY" -p "$port" \
|
||||
"$user@127.0.0.1"; then
|
||||
printf '%s\n' 'INFRA_BLOCKED SSH ControlMaster failed' >"$run/RESULT.txt"
|
||||
exit 21
|
||||
fi
|
||||
|
||||
export PRE15_MODE=qemu
|
||||
export PRE15_DUT=$dut PRE15_ROOT=$root PRE15_LIB_DIR=$lib
|
||||
export PRE15_CASE_TMP=$case_tmp PRE15_RUN_DIR=$case_run
|
||||
export PRE15_FREEBSD_SRC=$freebsd_src PRE15_B32_SCENARIO=$scenario
|
||||
export PRE15_QEMU_CONTROL_PATH=$control PRE15_QEMU_SSH_PORT=$port
|
||||
export PRE15_QEMU_SSH_USER=$user PRE15_QEMU_SSH_KEY
|
||||
export PRE15_GUEST_COMMAND_TIMEOUT=$guest_timeout
|
||||
export PRE15_OWNERSHIP_FILE=$ownership PRE15_CLEANUP_LOG=$cleanup_log
|
||||
export PRE15_TARGET_MARKER_FILE=$target_marker PRE15_REASON_FILE=$reason_file
|
||||
export PRE15_FIXTURE_HASHES=$fixture_hashes PRE15_MODULE_HASH=$module_hash
|
||||
|
||||
set +e
|
||||
timeout -k 10 "$case_timeout" /bin/sh "$case_script" \
|
||||
>"$case_run/stdout.log" 2>"$case_run/stderr.log"
|
||||
case_rc=$?
|
||||
set -e
|
||||
case "$case_rc" in
|
||||
0)
|
||||
if test -f "$target_marker"; then
|
||||
printf '%s PASS target=reached\n' "$scenario" >"$run/RESULT.txt"
|
||||
else
|
||||
printf '%s\n' 'RUNNER_FAIL case exited zero without target marker' \
|
||||
>"$run/RESULT.txt"
|
||||
case_rc=20
|
||||
fi
|
||||
;;
|
||||
10) printf '%s DUT_FAIL\n' "$scenario" >"$run/RESULT.txt" ;;
|
||||
20) printf '%s RUNNER_FAIL\n' "$scenario" >"$run/RESULT.txt" ;;
|
||||
21) printf '%s INFRA_BLOCKED\n' "$scenario" >"$run/RESULT.txt" ;;
|
||||
124|137) printf '%s RUNNER_FAIL case-timeout=%ss\n' "$scenario" "$case_timeout" >"$run/RESULT.txt"; case_rc=20 ;;
|
||||
*) printf '%s RUNNER_FAIL exit=%s\n' "$scenario" "$case_rc" >"$run/RESULT.txt"; case_rc=20 ;;
|
||||
esac
|
||||
exit "$case_rc"
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
EXTENT_RE = re.compile(
|
||||
r"^\s*0:\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*"
|
||||
r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def run(argv: list[str], cwd: Path) -> str:
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(
|
||||
f"command failed ({completed.returncode}): {' '.join(argv)}\n"
|
||||
f"{completed.stdout}"
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--spec", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
spec = json.loads(args.spec.read_text(encoding="ascii"))
|
||||
if (
|
||||
spec.get("schema") != 1
|
||||
or spec.get("batch") != "B33"
|
||||
or spec.get("test") != "TC168-cache-inflight"
|
||||
):
|
||||
raise SystemExit("invalid B33 cache fixture spec")
|
||||
if args.output.exists():
|
||||
raise SystemExit(f"refusing existing B33 output: {args.output}")
|
||||
for tool in ("dump.erofs", "mkfs.erofs"):
|
||||
if shutil.which(tool) is None:
|
||||
raise SystemExit(f"{tool} is required")
|
||||
|
||||
pattern = b"Pre15-B33-cache-policy:0123456789abcdef\n"
|
||||
expected = spec["source"]
|
||||
source = (pattern * ((expected["size"] + len(pattern) - 1) // len(pattern)))[
|
||||
: expected["size"]
|
||||
]
|
||||
if hashlib.sha256(source).hexdigest() != expected["sha256"]:
|
||||
raise SystemExit("B33 deterministic source identity changed")
|
||||
|
||||
source_dir = args.output / "source"
|
||||
images = args.output / "images"
|
||||
source_dir.mkdir(parents=True)
|
||||
images.mkdir()
|
||||
payload = source_dir / "payload.bin"
|
||||
payload.write_bytes(source)
|
||||
os.utime(payload, (0, 0))
|
||||
os.utime(source_dir, (0, 0))
|
||||
|
||||
valid = images / "lz4-valid.erofs"
|
||||
run(["mkfs.erofs", *spec["mkfs_args"], str(valid), str(source_dir)], images)
|
||||
extent_text = run(
|
||||
["dump.erofs", "-e", "--path=/payload.bin", str(valid)], images
|
||||
)
|
||||
match = EXTENT_RE.search(extent_text)
|
||||
if match is None:
|
||||
raise SystemExit("B33 single extent could not be parsed")
|
||||
logical, logical_end, logical_length, physical, physical_end, physical_length = (
|
||||
map(int, match.groups())
|
||||
)
|
||||
if (
|
||||
logical != 0
|
||||
or logical_end != expected["size"]
|
||||
or logical_length != expected["size"]
|
||||
or physical_end - physical != physical_length
|
||||
or physical_length != 4096
|
||||
):
|
||||
raise SystemExit("B33 fixture is not one 64 KiB logical pcluster")
|
||||
|
||||
valid_data = valid.read_bytes()
|
||||
if physical + physical_length > len(valid_data):
|
||||
raise SystemExit("B33 physical extent exceeds image")
|
||||
pcluster = valid_data[physical : physical + physical_length]
|
||||
leading = next(
|
||||
(index for index, value in enumerate(pcluster) if value), len(pcluster)
|
||||
)
|
||||
stream = pcluster[leading:]
|
||||
if leading == len(pcluster) or len(stream) < 2:
|
||||
raise SystemExit("B33 LZ4 stream boundary is empty")
|
||||
truncated_data = bytearray(valid_data)
|
||||
truncated_data[physical + leading : physical + physical_length] = (
|
||||
b"\0" + stream[:-1]
|
||||
)
|
||||
if truncated_data == valid_data:
|
||||
raise SystemExit("B33 truncated mutation changed no bytes")
|
||||
truncated = images / "lz4-truncated.erofs"
|
||||
truncated.write_bytes(truncated_data)
|
||||
|
||||
records = []
|
||||
for image_class, path, expected_errno in (
|
||||
("valid", valid, 0),
|
||||
("truncated", truncated, 97),
|
||||
):
|
||||
records.append(
|
||||
{
|
||||
"class": image_class,
|
||||
"expected_errno": expected_errno,
|
||||
"path": path.name,
|
||||
"sha256": sha256_path(path),
|
||||
}
|
||||
)
|
||||
index = {
|
||||
"batch": "B33",
|
||||
"decoded_size": len(source),
|
||||
"fixture_count": len(records),
|
||||
"fixtures": records,
|
||||
"leading_zero_bytes": leading,
|
||||
"physical_length": physical_length,
|
||||
"physical_offset": physical,
|
||||
"schema": 1,
|
||||
"source_sha256": sha256_path(payload),
|
||||
"status": "READY",
|
||||
"test": "TC168-cache-inflight",
|
||||
}
|
||||
(args.output / "fixture-index.json").write_text(
|
||||
json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="ascii"
|
||||
)
|
||||
print(json.dumps(index, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,418 @@
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define CODEC_COUNT 4
|
||||
#define MOUNT_BUDGET (256U * 1024)
|
||||
#define GLOBAL_BUDGET (512U * 1024)
|
||||
#define MINIMUM_WORK (128U * 1024)
|
||||
|
||||
enum cache_state {
|
||||
CACHE_EMPTY,
|
||||
CACHE_INFLIGHT,
|
||||
CACHE_READY,
|
||||
};
|
||||
|
||||
enum cache_lookup {
|
||||
CACHE_BYPASS,
|
||||
CACHE_HIT,
|
||||
CACHE_OWNER,
|
||||
};
|
||||
|
||||
struct metric {
|
||||
uint64_t hits;
|
||||
uint64_t misses;
|
||||
uint64_t bypasses;
|
||||
uint64_t evictions;
|
||||
uint64_t reclaims;
|
||||
size_t resident_bytes;
|
||||
};
|
||||
|
||||
struct budget {
|
||||
size_t limit;
|
||||
size_t used;
|
||||
size_t peak;
|
||||
};
|
||||
|
||||
struct key {
|
||||
uint64_t identity;
|
||||
uint8_t codec;
|
||||
};
|
||||
|
||||
struct cache {
|
||||
struct budget *global;
|
||||
struct metric metrics[CODEC_COUNT];
|
||||
struct key key;
|
||||
unsigned char *data;
|
||||
size_t charged_bytes;
|
||||
size_t budget_bytes;
|
||||
size_t peak_bytes;
|
||||
enum cache_state state;
|
||||
bool enabled;
|
||||
};
|
||||
|
||||
static void
|
||||
fail(const char *message)
|
||||
{
|
||||
fprintf(stderr, "B33 cache oracle failure: %s\n", message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static uint64_t
|
||||
hash_bytes(const unsigned char *data, size_t size)
|
||||
{
|
||||
uint64_t hash;
|
||||
size_t index;
|
||||
|
||||
hash = UINT64_C(1469598103934665603);
|
||||
for (index = 0; index < size; ++index) {
|
||||
hash ^= data[index];
|
||||
hash *= UINT64_C(1099511628211);
|
||||
}
|
||||
return (hash);
|
||||
}
|
||||
|
||||
static void
|
||||
fill(unsigned char *data, size_t size, const struct key *key)
|
||||
{
|
||||
size_t index;
|
||||
|
||||
for (index = 0; index < size; ++index)
|
||||
data[index] = (unsigned char)(key->identity + key->codec * 31 +
|
||||
index * 17);
|
||||
}
|
||||
|
||||
static bool
|
||||
key_equal(const struct key *left, const struct key *right)
|
||||
{
|
||||
|
||||
return (left->identity == right->identity && left->codec == right->codec);
|
||||
}
|
||||
|
||||
static bool
|
||||
reserve(struct budget *budget, size_t bytes)
|
||||
{
|
||||
|
||||
if (bytes > budget->limit || budget->used > budget->limit - bytes)
|
||||
return (false);
|
||||
budget->used += bytes;
|
||||
if (budget->used > budget->peak)
|
||||
budget->peak = budget->used;
|
||||
return (true);
|
||||
}
|
||||
|
||||
static void
|
||||
release(struct budget *budget, size_t bytes)
|
||||
{
|
||||
|
||||
if (bytes > budget->used)
|
||||
fail("global budget underflow");
|
||||
budget->used -= bytes;
|
||||
}
|
||||
|
||||
static void
|
||||
cache_init(struct cache *cache, struct budget *global, bool enabled)
|
||||
{
|
||||
|
||||
memset(cache, 0, sizeof(*cache));
|
||||
cache->global = global;
|
||||
cache->budget_bytes = MOUNT_BUDGET;
|
||||
cache->enabled = enabled;
|
||||
}
|
||||
|
||||
static bool
|
||||
policy_admit(const struct cache *cache, uint8_t codec, size_t decoded_size,
|
||||
size_t compressed_size)
|
||||
{
|
||||
|
||||
return (cache->enabled && codec < CODEC_COUNT &&
|
||||
decoded_size <= cache->budget_bytes &&
|
||||
compressed_size + decoded_size >= MINIMUM_WORK);
|
||||
}
|
||||
|
||||
static void
|
||||
drop(struct cache *cache, bool evicted, bool reclaimed)
|
||||
{
|
||||
struct metric *metric;
|
||||
|
||||
if (cache->charged_bytes == 0)
|
||||
return;
|
||||
if (cache->key.codec >= CODEC_COUNT)
|
||||
fail("charged unknown codec");
|
||||
metric = &cache->metrics[cache->key.codec];
|
||||
if (cache->state == CACHE_READY) {
|
||||
if (metric->resident_bytes != cache->charged_bytes)
|
||||
fail("codec resident accounting mismatch");
|
||||
metric->resident_bytes = 0;
|
||||
if (evicted)
|
||||
++metric->evictions;
|
||||
if (reclaimed)
|
||||
++metric->reclaims;
|
||||
}
|
||||
release(cache->global, cache->charged_bytes);
|
||||
cache->charged_bytes = 0;
|
||||
free(cache->data);
|
||||
cache->data = NULL;
|
||||
}
|
||||
|
||||
static enum cache_lookup
|
||||
claim(struct cache *cache, const struct key *key, size_t decoded_size,
|
||||
size_t compressed_size, unsigned char *output)
|
||||
{
|
||||
struct metric *metric;
|
||||
|
||||
if (key->codec >= CODEC_COUNT)
|
||||
fail("unknown requested codec");
|
||||
metric = &cache->metrics[key->codec];
|
||||
if (!policy_admit(cache, key->codec, decoded_size, compressed_size)) {
|
||||
++metric->bypasses;
|
||||
return (CACHE_BYPASS);
|
||||
}
|
||||
if (cache->state == CACHE_READY && key_equal(&cache->key, key)) {
|
||||
memcpy(output, cache->data, decoded_size);
|
||||
++metric->hits;
|
||||
return (CACHE_HIT);
|
||||
}
|
||||
if (cache->state == CACHE_INFLIGHT) {
|
||||
++metric->bypasses;
|
||||
return (CACHE_BYPASS);
|
||||
}
|
||||
if (cache->state == CACHE_READY)
|
||||
drop(cache, true, false);
|
||||
cache->state = CACHE_EMPTY;
|
||||
if (!reserve(cache->global, decoded_size)) {
|
||||
++metric->bypasses;
|
||||
return (CACHE_BYPASS);
|
||||
}
|
||||
cache->charged_bytes = decoded_size;
|
||||
if (decoded_size > cache->peak_bytes)
|
||||
cache->peak_bytes = decoded_size;
|
||||
cache->key = *key;
|
||||
cache->state = CACHE_INFLIGHT;
|
||||
++metric->misses;
|
||||
return (CACHE_OWNER);
|
||||
}
|
||||
|
||||
static void
|
||||
complete(struct cache *cache, const struct key *key, unsigned char *data,
|
||||
int error)
|
||||
{
|
||||
struct metric *metric;
|
||||
|
||||
if (cache->state != CACHE_INFLIGHT || !key_equal(&cache->key, key))
|
||||
fail("owner lost key");
|
||||
metric = &cache->metrics[key->codec];
|
||||
if (error != 0) {
|
||||
if (data != NULL || error < 1)
|
||||
fail("failure publication is untyped");
|
||||
release(cache->global, cache->charged_bytes);
|
||||
cache->charged_bytes = 0;
|
||||
cache->state = CACHE_EMPTY;
|
||||
return;
|
||||
}
|
||||
if (data == NULL || metric->resident_bytes != 0)
|
||||
fail("success publication is untyped");
|
||||
cache->data = data;
|
||||
metric->resident_bytes = cache->charged_bytes;
|
||||
cache->state = CACHE_READY;
|
||||
}
|
||||
|
||||
static bool
|
||||
reclaim(struct cache *cache)
|
||||
{
|
||||
|
||||
if (cache->state == CACHE_INFLIGHT)
|
||||
return (false);
|
||||
if (cache->state == CACHE_READY) {
|
||||
drop(cache, true, true);
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
static void
|
||||
cache_fini(struct cache *cache)
|
||||
{
|
||||
|
||||
if (cache->state == CACHE_INFLIGHT)
|
||||
fail("unmount did not drain owner");
|
||||
drop(cache, false, false);
|
||||
cache->state = CACHE_EMPTY;
|
||||
}
|
||||
|
||||
static unsigned char *
|
||||
make_data(const struct key *key, size_t size)
|
||||
{
|
||||
unsigned char *data;
|
||||
|
||||
data = malloc(size);
|
||||
if (data == NULL)
|
||||
fail("decoded allocation");
|
||||
fill(data, size, key);
|
||||
return (data);
|
||||
}
|
||||
|
||||
static void
|
||||
test_codec_accounting(struct budget *global, struct metric output[CODEC_COUNT],
|
||||
uint64_t *eviction_hash)
|
||||
{
|
||||
struct cache cache;
|
||||
struct key key;
|
||||
unsigned char *expected, *readback;
|
||||
uint64_t before, after;
|
||||
unsigned int codec;
|
||||
|
||||
cache_init(&cache, global, true);
|
||||
expected = malloc(MOUNT_BUDGET);
|
||||
readback = malloc(MOUNT_BUDGET);
|
||||
if (expected == NULL || readback == NULL)
|
||||
fail("accounting buffers");
|
||||
before = 0;
|
||||
for (codec = 0; codec < CODEC_COUNT; ++codec) {
|
||||
key = (struct key){ .identity = 100 + codec, .codec = codec };
|
||||
fill(expected, MOUNT_BUDGET, &key);
|
||||
if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_OWNER)
|
||||
fail("codec-neutral request was not admitted");
|
||||
complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0);
|
||||
if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_HIT ||
|
||||
memcmp(readback, expected, MOUNT_BUDGET) != 0)
|
||||
fail("codec cache hit changed bytes");
|
||||
if (codec == 0)
|
||||
before = hash_bytes(readback, MOUNT_BUDGET);
|
||||
}
|
||||
key = (struct key){ .identity = 100, .codec = 0 };
|
||||
if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_OWNER)
|
||||
fail("evicted codec key was not reusable");
|
||||
complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0);
|
||||
if (claim(&cache, &key, MOUNT_BUDGET, 4096, readback) != CACHE_HIT)
|
||||
fail("reused codec key did not hit");
|
||||
after = hash_bytes(readback, MOUNT_BUDGET);
|
||||
if (before != after)
|
||||
fail("eviction changed decoded hash");
|
||||
if (!reclaim(&cache) || global->used != 0)
|
||||
fail("ready reclaim did not release bytes");
|
||||
if (cache.peak_bytes != MOUNT_BUDGET)
|
||||
fail("mount peak did not equal hard budget");
|
||||
memcpy(output, cache.metrics, sizeof(cache.metrics));
|
||||
*eviction_hash = after;
|
||||
free(readback);
|
||||
free(expected);
|
||||
}
|
||||
|
||||
static uint64_t
|
||||
test_global_exhaustion(struct budget *global)
|
||||
{
|
||||
struct cache first, second, third;
|
||||
struct key first_key = { .identity = 201, .codec = 0 };
|
||||
struct key second_key = { .identity = 202, .codec = 2 };
|
||||
struct key third_key = { .identity = 203, .codec = 3 };
|
||||
unsigned char *scratch;
|
||||
uint64_t fallback_hash;
|
||||
|
||||
cache_init(&first, global, true);
|
||||
cache_init(&second, global, true);
|
||||
cache_init(&third, global, true);
|
||||
scratch = malloc(MOUNT_BUDGET);
|
||||
if (scratch == NULL)
|
||||
fail("global fallback allocation");
|
||||
if (claim(&first, &first_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER ||
|
||||
claim(&second, &second_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER)
|
||||
fail("two mounts did not fill global budget");
|
||||
complete(&first, &first_key, make_data(&first_key, MOUNT_BUDGET), 0);
|
||||
complete(&second, &second_key, make_data(&second_key, MOUNT_BUDGET), 0);
|
||||
if (global->used != GLOBAL_BUDGET || global->peak != GLOBAL_BUDGET)
|
||||
fail("global hard budget was not reached exactly");
|
||||
if (claim(&third, &third_key, MOUNT_BUDGET, 4096, scratch) != CACHE_BYPASS)
|
||||
fail("global exhaustion did not bypass");
|
||||
fill(scratch, MOUNT_BUDGET, &third_key);
|
||||
fallback_hash = hash_bytes(scratch, MOUNT_BUDGET);
|
||||
if (!reclaim(&first) ||
|
||||
claim(&third, &third_key, MOUNT_BUDGET, 4096, scratch) != CACHE_OWNER)
|
||||
fail("reclaimed global bytes were not reusable");
|
||||
complete(&third, &third_key, make_data(&third_key, MOUNT_BUDGET), 0);
|
||||
cache_fini(&second);
|
||||
cache_fini(&third);
|
||||
if (global->used != 0)
|
||||
fail("unmount did not release global bytes");
|
||||
free(scratch);
|
||||
return (fallback_hash);
|
||||
}
|
||||
|
||||
static void
|
||||
test_failure_and_policy(struct budget *global)
|
||||
{
|
||||
struct cache cache, disabled;
|
||||
struct key key = { .identity = 301, .codec = 1 };
|
||||
unsigned char output[MOUNT_BUDGET];
|
||||
unsigned int codec;
|
||||
|
||||
cache_init(&cache, global, true);
|
||||
if (claim(&cache, &key, MOUNT_BUDGET, 4096, output) != CACHE_OWNER)
|
||||
fail("failure setup was not admitted");
|
||||
if (reclaim(&cache))
|
||||
fail("inflight reclaim did not report busy");
|
||||
complete(&cache, &key, NULL, EIO);
|
||||
if (global->used != 0 ||
|
||||
claim(&cache, &key, MOUNT_BUDGET, 4096, output) != CACHE_OWNER)
|
||||
fail("failure did not release budget for retry");
|
||||
complete(&cache, &key, make_data(&key, MOUNT_BUDGET), 0);
|
||||
cache_fini(&cache);
|
||||
|
||||
cache_init(&disabled, global, false);
|
||||
for (codec = 0; codec < CODEC_COUNT; ++codec) {
|
||||
key.codec = codec;
|
||||
if (claim(&disabled, &key, MOUNT_BUDGET, 4096, output) != CACHE_BYPASS)
|
||||
fail("disabled codec policy did not bypass");
|
||||
disabled.enabled = true;
|
||||
if (claim(&disabled, &key, 64U * 1024, 1, output) != CACHE_BYPASS)
|
||||
fail("low-work policy did not bypass");
|
||||
disabled.enabled = false;
|
||||
}
|
||||
disabled.enabled = true;
|
||||
if (claim(&disabled, &key, MOUNT_BUDGET + 1, 4096, output) != CACHE_BYPASS)
|
||||
fail("oversized policy did not bypass");
|
||||
cache_fini(&disabled);
|
||||
if (global->used != 0)
|
||||
fail("policy tests leaked global bytes");
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
struct budget global = { .limit = GLOBAL_BUDGET };
|
||||
struct metric metrics[CODEC_COUNT];
|
||||
uint64_t eviction_hash, fallback_hash;
|
||||
unsigned int codec;
|
||||
|
||||
test_codec_accounting(&global, metrics, &eviction_hash);
|
||||
fallback_hash = test_global_exhaustion(&global);
|
||||
test_failure_and_policy(&global);
|
||||
for (codec = 0; codec < CODEC_COUNT; ++codec) {
|
||||
if (metrics[codec].hits == 0 || metrics[codec].misses == 0 ||
|
||||
metrics[codec].evictions == 0 || metrics[codec].resident_bytes != 0)
|
||||
fail("four-codec metrics are incomplete");
|
||||
}
|
||||
printf("{\"status\":\"PASS\",\"global_limit\":%u,"
|
||||
"\"global_peak\":%zu,\"global_remaining\":%zu,"
|
||||
"\"mount_limit\":%u,\"eviction_hash\":\"%016" PRIx64 "\","
|
||||
"\"fallback_hash\":\"%016" PRIx64 "\",\"metrics\":[",
|
||||
GLOBAL_BUDGET, global.peak, global.used, MOUNT_BUDGET,
|
||||
eviction_hash, fallback_hash);
|
||||
for (codec = 0; codec < CODEC_COUNT; ++codec) {
|
||||
if (codec != 0)
|
||||
printf(",");
|
||||
printf("{\"codec\":%u,\"hits\":%" PRIu64
|
||||
",\"misses\":%" PRIu64 ",\"bypasses\":%" PRIu64
|
||||
",\"evictions\":%" PRIu64 ",\"reclaims\":%" PRIu64
|
||||
",\"resident_bytes\":%zu}", codec, metrics[codec].hits,
|
||||
metrics[codec].misses, metrics[codec].bypasses,
|
||||
metrics[codec].evictions, metrics[codec].reclaims,
|
||||
metrics[codec].resident_bytes);
|
||||
}
|
||||
printf("]}\n");
|
||||
return (0);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"batch": "B33",
|
||||
"mkfs_args": [
|
||||
"-T0",
|
||||
"-U33000000-0000-4000-8000-000000000038",
|
||||
"--all-root",
|
||||
"-x-1",
|
||||
"--workers=1",
|
||||
"-zlz4",
|
||||
"-C4096",
|
||||
"-Elegacy-compress"
|
||||
],
|
||||
"schema": 1,
|
||||
"source": {
|
||||
"sha256": "55e7e309853ecccb5c891b611b630c9c764d07e340331ff62080075e1ddad89c",
|
||||
"size": 65536
|
||||
},
|
||||
"test": "TC168-cache-inflight"
|
||||
}
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B33-qemu-run.sh
|
||||
source_runner=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/B32-qemu-run.sh
|
||||
|
||||
if test "${1:-}" != --worker; then
|
||||
test "$#" -eq 1 && test "$1" = TC168-cache-inflight || {
|
||||
printf '%s\n' 'usage: B33-qemu-run.sh TC168-cache-inflight' >&2
|
||||
exit 20
|
||||
}
|
||||
total=${PRE15_QEMU_TIMEOUT:-1200}
|
||||
case "$total" in
|
||||
''|*[!0-9]*|0) printf '%s\n' 'invalid PRE15_QEMU_TIMEOUT' >&2; exit 20 ;;
|
||||
esac
|
||||
exec timeout -k 30 "$total" "$script" --worker "$1"
|
||||
fi
|
||||
|
||||
shift
|
||||
test "$#" -eq 1 && test "$1" = TC168-cache-inflight || exit 20
|
||||
test -f "$source_runner" || {
|
||||
printf '%s\n' 'INFRA_BLOCKED B32 absolute-timeout runner is absent' >&2
|
||||
exit 21
|
||||
}
|
||||
|
||||
sed \
|
||||
-e "s|^script=.*$|script=$source_runner|" \
|
||||
-e 's|case_script=$dut/tests/pre15/cases/B32-cache-state.sh|case_script=$dut/tests/pre15/cases/B33-cache-policy.sh|' \
|
||||
-e 's/-qemu-B32-/-qemu-B33-/' \
|
||||
"$source_runner" | /bin/sh -s -- --worker "$1"
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
arguments = parser.parse_args()
|
||||
root = arguments.output
|
||||
root.mkdir(parents=True)
|
||||
(root / "nested").mkdir()
|
||||
payload = bytearray()
|
||||
for offset in range(384 * 1024):
|
||||
payload.append(((offset * 131) ^ (offset >> 3) ^ 0x5A) & 0xFF)
|
||||
(root / "payload.bin").write_bytes(payload)
|
||||
(root / "nested" / "alpha.txt").write_text(
|
||||
"Pre15 deterministic smoke fixture\n", encoding="ascii"
|
||||
)
|
||||
(root / "empty").write_bytes(b"")
|
||||
files = []
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
files.append(
|
||||
{
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"sha256": digest(path),
|
||||
"size": path.stat().st_size,
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"files": files,
|
||||
"nested_entries": ["alpha.txt"],
|
||||
"root_entries": ["empty", "nested", "payload.bin"],
|
||||
"status": "READY",
|
||||
}
|
||||
print(json.dumps(manifest, indent=2, sort_keys=True))
|
||||
@@ -0,0 +1,32 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/linker.h>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
struct kld_sym_lookup lookup;
|
||||
int failed;
|
||||
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "usage: SMOKE-kldsym SYMBOL...\n");
|
||||
return (2);
|
||||
}
|
||||
failed = 0;
|
||||
for (int index = 1; index < argc; ++index) {
|
||||
memset(&lookup, 0, sizeof(lookup));
|
||||
lookup.version = sizeof(lookup);
|
||||
lookup.symname = argv[index];
|
||||
if (kldsym(0, KLDSYM_LOOKUP, &lookup) == -1) {
|
||||
printf("MISSING %s\n", argv[index]);
|
||||
failed = 1;
|
||||
} else {
|
||||
printf("PRESENT %s 0x%jx\n", argv[index],
|
||||
(uintmax_t)lookup.symvalue);
|
||||
}
|
||||
}
|
||||
return (failed);
|
||||
}
|
||||
Executable
+1142
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user