imrcpan/erofs-freebsd-out-tree
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
imrcpan
2026-08-18 09:20:44 +02:00
commit b826cd721a
522 changed files with 93730 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/pre15/gates/P15-005-input.json
+416
View File
@@ -0,0 +1,416 @@
{
"schema": 1,
"gate": "G02",
"candidate": "P15-005",
"required_base": "edf098905a34de764185e72fc7e92d7b8f4e7285",
"title": "metadata buffer ownership and frozen map tuple oracle",
"required_categories": [
"plain",
"inline",
"hole",
"chunk",
"multidevice",
"compressed",
"fragment",
"partial-reference",
"post-EOF",
"bounds",
"overflow"
],
"tuple_fields": [
"m_la",
"m_pa",
"m_llen",
"m_plen",
"m_deviceid",
"m_flags",
"errno",
"acquire_count",
"release_count"
],
"errno": {
"EIO": 5,
"EINVAL": 22,
"EOPNOTSUPP": 45,
"EOVERFLOW": 84,
"EINTEGRITY": 97
},
"source_paths": [
"repo-pre-15/src/internal.h",
"repo-pre-15/src/data.c",
"repo-pre-15/src/decompressor.c",
"repo-pre-15/src/inode.c",
"repo-pre-15/src/super.c",
"repo-pre-15/src/xattr.c",
"repo-pre-15/src/zdata.c",
"repo-pre-15/src/zmap.c",
"repo-pre-15/src/erofs_fs.h",
"src-linux/internal.h",
"src-linux/data.c",
"src-linux/decompressor.c",
"src-linux/inode.c",
"src-linux/super.c",
"src-linux/xattr.c",
"src-linux/zdata.c",
"src-linux/zmap.c"
],
"linux_sha256": {
"src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405",
"src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a",
"src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709",
"src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3",
"src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9",
"src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a",
"src-linux/zdata.c": "358869da60dcdafd13bc1ff0cd28e864c13b46a49d9f556d9b37139f7e0f4e79",
"src-linux/zmap.c": "4fb34c024bfaec9d2294ba2f85069bb2f727b378ebf4848814e081f91ea022cc"
},
"linux_semantic_anchors": {
"src-linux/internal.h": [
"struct erofs_buf {",
"#define __EROFS_BUF_INITIALIZER",
"void erofs_put_metabuf(struct erofs_buf *buf);"
],
"src-linux/data.c": [
"void erofs_put_metabuf(struct erofs_buf *buf)",
"folio_put(page_folio(buf->page));",
"buf->page = NULL;",
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/decompressor.c": [
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/inode.c": [
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/super.c": [
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/xattr.c": [
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/zdata.c": [
"struct erofs_buf buf = __EROFS_BUF_INITIALIZER;",
"erofs_put_metabuf(&buf);"
],
"src-linux/zmap.c": [
"erofs_put_metabuf(&map.buf);"
]
},
"baseline_function_sha256": {
"data.c:erofs_map_blocks_chunk": "b850ab686f61e6265ef0780c1363f570ceb6afab004b5d9c527d728f00a0fa0e",
"data.c:erofs_read_metadata": "1ea2f9bad3c60019c0f373c98444dcd1dd4bb178875a170313868a56cd314436",
"data.c:erofs_read_data": "9d962f6086f5fc4a66bbfd9b27d05aa71234bb8266a3c383ec693298966e5e63",
"data.c:erofs_read_uio": "05d3ad9e8b25b7e9b6d62f4fd836ab204bc57b90781fc140cc498d1ab8fe0a78",
"decompressor.c:z_erofs_read_cfg": "77ddaf24f5317e0b83b72d85f64aa624cbd15b71a9602d2217b11dd382e1f719",
"decompressor.c:z_erofs_parse_cfgs": "d800c9a52b9f9333f57da2f67e27d238320866b0f8816929b32f119c84a69eb4",
"inode.c:erofs_read_inode": "ed65faffe7a607e95bca50de7e28cf592a0500b7363935991de959269a200fe1",
"super.c:erofs_load_generation_seed": "98299634491a22b427820a28a7ec1d23268ab1fe02d1fc0515774da70900f071",
"super.c:erofs_superblock_csum_verify": "cd56c29a6cf795de359f6642509a4a152e5a1860ec32a997e91ebf81e7877096",
"super.c:erofs_scan_devices": "2743af3b9d9d7e53c0102bc197026adf58ef6bf7adbf5cdbe8716c9e25b2a547",
"super.c:erofs_read_superblock": "d605349d9604d89d4bb413c42abc71f13b8fafa43b1fda671fbd4613f2a3ed6f",
"xattr.c:erofs_xattr_read_backing": "df65cf9e391d2f17f79beeecda24fbf6c1be495dd3baf65121c226c4054ccf75",
"xattr.c:erofs_xattr_read_metadata": "ed726755b28f1c1a82a07a956b889bd633da45cdd6671c91c7bc4db9d4006096",
"xattr.c:erofs_xattr_load_body": "c8684629f78920cefb6a6d61ddad5ed42ba499d817798f81086282c827e54aa5",
"xattr.c:erofs_xattr_load_shared_entry": "e47145a775685cfaeb9341a7cc1cc742c73145e34f4b81d8fc5dac35d765e17f",
"xattr.c:erofs_inode_has_noacl": "8098be9fef8148498a2080e02db2876d9592f1af10ca551278902e3aa4c3428e",
"xattr.c:erofs_xattr_iter_shared": "1ec7756ef31dd6c51e04d92656440a9dbca5e9e9fe38c359eed744ad16b39b2c",
"xattr.c:erofs_getxattr": "9ee7d7effcfa3d23ab4d4ce1e3e7d638c048085368b2f3a2db3dfdadf911cd34",
"xattr.c:erofs_listxattr": "6f75c9f8c50c331daba1bb0c1b757390c3ff873ee7419b2dd2b35dfc3db84ea6",
"xattr.c:erofs_xattr_prefixes_init": "7aaade8116337ad1026da04777bf1d9f8aefaa246623d6c32080af153fb93abd",
"zdata.c:z_erofs_read_extent": "327e3a22b9c89c410d1ecef6934dcdaf194a4b6962737ea5f3e4f21d10694b13",
"zdata.c:z_erofs_do_read": "2096f98b6d19e1c5f80edb6de9fbfe15bb27f006e2510bcb068f920c23aa38ec",
"zmap.c:z_erofs_read_index": "cf8ca75c1d600f2f708e73e723a7abaee7c98f9813eb8f432e1f96dd1602d82b",
"zmap.c:z_erofs_load_full_lcluster": "751d456964fcdcf66f20d22c1f22adef209767aef1bab9d0857f36df7d2c37b2",
"zmap.c:z_erofs_load_compact_lcluster": "4c23a8a966262b5a365bc874718316daa616cc97dfe3e2711570f81e7e3b953c",
"zmap.c:z_erofs_read_extent": "454064ca59dfd0674af5520c4a965be659d2b7769403cd8d84bd9657e0145f27",
"zmap.c:z_erofs_map_blocks_ext": "31c9310f801395fe01d38515ee42ef49c1d07ceba0b86b612bb140ec180238b0",
"zmap.c:z_erofs_fill_inode": "125f0c3492c152415978226919e782d355a32fc49cf8f49456eb398a4c721032"
},
"direct_metadata_functions": [
"data.c:erofs_map_blocks_chunk",
"data.c:erofs_read_data",
"data.c:erofs_read_uio",
"decompressor.c:z_erofs_read_cfg",
"inode.c:erofs_read_inode",
"super.c:erofs_load_generation_seed",
"super.c:erofs_superblock_csum_verify",
"super.c:erofs_scan_devices",
"super.c:erofs_read_superblock",
"xattr.c:erofs_xattr_read_backing",
"zdata.c:z_erofs_read_extent",
"zmap.c:z_erofs_fill_inode",
"zmap.c:z_erofs_map_blocks_ext",
"zmap.c:z_erofs_read_extent",
"zmap.c:z_erofs_read_index"
],
"candidate_ownership_contract": {
"object_local_counts": {
"data.c:erofs_map_blocks_chunk": 1,
"data.c:erofs_read_metadata": 0,
"data.c:erofs_read_data": 1,
"data.c:erofs_read_uio": 1,
"decompressor.c:z_erofs_read_cfg": 1,
"decompressor.c:z_erofs_parse_cfgs": 1,
"inode.c:erofs_read_inode": 1,
"super.c:erofs_load_generation_seed": 1,
"super.c:erofs_superblock_csum_verify": 1,
"super.c:erofs_scan_devices": 1,
"super.c:erofs_read_superblock": 1,
"xattr.c:erofs_xattr_read_backing": 0,
"xattr.c:erofs_xattr_read_metadata": 1,
"xattr.c:erofs_xattr_load_body": 1,
"xattr.c:erofs_xattr_load_shared_entry": 1,
"xattr.c:erofs_inode_has_noacl": 1,
"xattr.c:erofs_xattr_iter_shared": 1,
"xattr.c:erofs_getxattr": 1,
"xattr.c:erofs_listxattr": 1,
"xattr.c:erofs_xattr_prefixes_init": 1,
"zdata.c:z_erofs_read_extent": 1,
"zdata.c:z_erofs_do_read": 0,
"zmap.c:z_erofs_read_index": 0,
"zmap.c:z_erofs_load_full_lcluster": 1,
"zmap.c:z_erofs_load_compact_lcluster": 1,
"zmap.c:z_erofs_read_extent": 1,
"zmap.c:z_erofs_map_blocks_ext": 1,
"zmap.c:z_erofs_fill_inode": 1
},
"put_counts": {
"data.c:erofs_map_blocks_chunk": 1,
"data.c:erofs_read_metadata": 0,
"data.c:erofs_read_data": 1,
"data.c:erofs_read_uio": 1,
"decompressor.c:z_erofs_read_cfg": 1,
"decompressor.c:z_erofs_parse_cfgs": 1,
"inode.c:erofs_read_inode": 18,
"super.c:erofs_load_generation_seed": 1,
"super.c:erofs_superblock_csum_verify": 1,
"super.c:erofs_scan_devices": 2,
"super.c:erofs_read_superblock": 1,
"xattr.c:erofs_xattr_read_backing": 0,
"xattr.c:erofs_xattr_read_metadata": 1,
"xattr.c:erofs_xattr_load_body": 1,
"xattr.c:erofs_xattr_load_shared_entry": 1,
"xattr.c:erofs_inode_has_noacl": 1,
"xattr.c:erofs_xattr_iter_shared": 1,
"xattr.c:erofs_getxattr": 1,
"xattr.c:erofs_listxattr": 1,
"xattr.c:erofs_xattr_prefixes_init": 2,
"zdata.c:z_erofs_read_extent": 1,
"zdata.c:z_erofs_do_read": 0,
"zmap.c:z_erofs_read_index": 0,
"zmap.c:z_erofs_load_full_lcluster": 2,
"zmap.c:z_erofs_load_compact_lcluster": 7,
"zmap.c:z_erofs_read_extent": 1,
"zmap.c:z_erofs_map_blocks_ext": 1,
"zmap.c:z_erofs_fill_inode": 6
},
"legacy_release_counts": {
"data.c:erofs_read_data": 1,
"data.c:erofs_read_uio": 1,
"zdata.c:z_erofs_read_extent": 1,
"zdata.c:z_erofs_do_read": 1
},
"raw_bread_counts": {
"data.c:erofs_read_metadata": 1,
"xattr.c:erofs_xattr_read_backing": 1
},
"acquire_apis": [
"erofs_read_metadata",
"z_erofs_read_cfg",
"erofs_xattr_read_backing",
"erofs_xattr_read_metadata",
"erofs_xattr_load_body",
"erofs_xattr_load_shared_entry",
"z_erofs_read_index"
],
"transfer_functions": [
"data.c:erofs_read_metadata",
"decompressor.c:z_erofs_read_cfg",
"xattr.c:erofs_xattr_read_backing",
"xattr.c:erofs_xattr_read_metadata",
"zmap.c:z_erofs_read_index"
]
},
"tuple_cases": [
{
"id": "plain-block-edge",
"engine": "data",
"category": "plain",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 12288, "datalayout": 0, "startblk": 32, "inode_off": 0, "inode_isize": 32, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 0},
"request": 4095,
"reader": {"mode": "unused", "bytes_hex": ""},
"expected": {"m_la": 4095, "m_pa": 135167, "m_llen": 1, "m_plen": 1, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 0, "release_count": 0}
},
{
"id": "inline-tail",
"engine": "data",
"category": "inline",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 5000, "datalayout": 2, "startblk": 40, "inode_off": 8192, "inode_isize": 64, "xattr_isize": 32, "chunkbits": 12, "chunkformat": 0},
"request": 4500,
"reader": {"mode": "unused", "bytes_hex": ""},
"expected": {"m_la": 4500, "m_pa": 8692, "m_llen": 500, "m_plen": 500, "m_deviceid": 0, "m_flags": 3, "errno": 0, "acquire_count": 0, "release_count": 0}
},
{
"id": "plain-hole",
"engine": "data",
"category": "hole",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 64, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 4096, "datalayout": 0, "startblk": 18446744073709551615, "inode_off": 0, "inode_isize": 32, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 0},
"request": 512,
"reader": {"mode": "unused", "bytes_hex": ""},
"expected": {"m_la": 512, "m_pa": 0, "m_llen": 3584, "m_plen": 3584, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0}
},
{
"id": "chunk-index",
"engine": "data",
"category": "chunk",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 20000, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 13, "chunkformat": 32},
"request": 9000,
"reader": {"mode": "ok", "bytes_hex": "0000000000000000000000004d000000"},
"expected": {"m_la": 9000, "m_pa": 316200, "m_llen": 7384, "m_plen": 7384, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}
},
{
"id": "chunk-48bit-device2",
"engine": "data",
"category": "multidevice",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 24576, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 13, "chunkformat": 96},
"request": 12345,
"reader": {"mode": "ok", "bytes_hex": "00000000000000000100020005000000"},
"expected": {"m_la": 12345, "m_pa": 17592186069049, "m_llen": 4039, "m_plen": 4039, "m_deviceid": 2, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}
},
{
"id": "chunk-index-out-of-provider",
"engine": "data",
"category": "bounds",
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 2, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 8192, "inode_isize": 0, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 32},
"request": 0,
"reader": {"mode": "unused", "bytes_hex": ""},
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 97, "acquire_count": 0, "release_count": 0}
},
{
"id": "chunk-shift-overflow-after-acquire",
"engine": "data",
"category": "overflow",
"sbi": {"block_size": 131072, "blkszbits": 17, "blocks": 1, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 96},
"request": 0,
"reader": {"mode": "ok", "bytes_hex": "0080000000000000"},
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 4096, "m_plen": 4096, "m_deviceid": 0, "m_flags": 0, "errno": 84, "acquire_count": 1, "release_count": 1}
},
{
"id": "chunk-index-short-read",
"engine": "data",
"category": "bounds",
"supplemental": true,
"sbi": {"block_size": 4096, "blkszbits": 12, "blocks": 128, "device_id_mask": 65535},
"inode": {"nid": 1, "size": 4096, "datalayout": 4, "startblk": 0, "inode_off": 0, "inode_isize": 64, "xattr_isize": 0, "chunkbits": 12, "chunkformat": 32},
"request": 0,
"reader": {"mode": "short", "bytes_hex": "00000000"},
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0}
},
{
"id": "explicit-compressed",
"engine": "zmap",
"category": "compressed",
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1},
"request": 0,
"reader": {"mode": "ok", "base": 16, "records": [{"plen": 268437504, "pstart": 74565, "lstart": 0}]},
"expected": {"m_la": 0, "m_pa": 74565, "m_llen": 4096, "m_plen": 2048, "m_deviceid": 0, "m_flags": 1, "errno": 0, "acquire_count": 1, "release_count": 1}
},
{
"id": "explicit-partial-reference",
"engine": "zmap",
"category": "partial-reference",
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1},
"request": 0,
"reader": {"mode": "ok", "base": 16, "records": [{"plen": 402654208, "pstart": 262144, "lstart": 0}]},
"expected": {"m_la": 0, "m_pa": 262144, "m_llen": 4096, "m_plen": 1024, "m_deviceid": 0, "m_flags": 9, "errno": 0, "acquire_count": 1, "release_count": 1}
},
{
"id": "explicit-fragment-tail",
"engine": "zmap",
"category": "fragment",
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 6144, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 37, "z_lclusterbits": 12, "z_extents": 1},
"request": 4096,
"reader": {"mode": "ok", "base": 16, "records": [{"plen": 8192, "pstart": 0, "lstart": 4096}]},
"expected": {"m_la": 4096, "m_pa": 0, "m_llen": 2048, "m_plen": 8192, "m_deviceid": 0, "m_flags": 16, "errno": 0, "acquire_count": 1, "release_count": 1}
},
{
"id": "compressed-post-eof",
"engine": "zmap",
"category": "post-EOF",
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1},
"request": 4113,
"reader": {"mode": "unused", "base": 16, "records": []},
"expected": {"m_la": 4096, "m_pa": 0, "m_llen": 18, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 0, "acquire_count": 0, "release_count": 0}
},
{
"id": "compressed-index-short-read",
"engine": "zmap",
"category": "compressed",
"supplemental": true,
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1},
"request": 0,
"reader": {"mode": "short", "base": 16, "bytes_hex": "0000000000000000"},
"expected": {"m_la": 0, "m_pa": 0, "m_llen": 0, "m_plen": 0, "m_deviceid": 0, "m_flags": 0, "errno": 5, "acquire_count": 0, "release_count": 0}
},
{
"id": "compressed-sanity-error-after-acquire",
"engine": "zmap",
"category": "compressed",
"supplemental": true,
"sbi": {"block_size": 4096, "blkszbits": 12, "available_compr_algs": 1, "packed_size": 16384},
"inode": {"nid": 2, "size": 4096, "datalayout": 1, "inode_off": 0, "inode_isize": 0, "xattr_isize": 0, "z_advise": 5, "z_lclusterbits": 12, "z_extents": 1},
"request": 0,
"reader": {"mode": "ok", "base": 16, "records": [{"plen": 1879049216, "pstart": 74565, "lstart": 0}]},
"expected": {"m_la": 0, "m_pa": 74565, "m_llen": 0, "m_plen": 1024, "m_deviceid": 0, "m_flags": 1, "errno": 45, "acquire_count": 1, "release_count": 1}
}
],
"ownership_paths": [
{"id": "plain-inline-success", "consumer": "plain", "function": "data.c:erofs_read_data", "path": "metadata read succeeds; copied raw output is caller-owned", "acquire_count": 1, "release_count": 1},
{"id": "plain-inline-short-read", "consumer": "plain", "function": "data.c:erofs_read_data", "path": "metadata read fails before ownership transfer", "acquire_count": 0, "release_count": 0},
{"id": "map-chunk-success", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "index acquired, decoded, and released", "acquire_count": 1, "release_count": 1},
{"id": "map-chunk-post-acquire-error", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "index released before physical shift overflow", "acquire_count": 1, "release_count": 1},
{"id": "map-chunk-short-read", "consumer": "map", "function": "data.c:erofs_map_blocks_chunk", "path": "short index fails before acquire", "acquire_count": 0, "release_count": 0},
{"id": "compressed-config-success", "consumer": "compressed", "function": "decompressor.c:z_erofs_read_cfg", "path": "length released; payload transferred then released by parse_cfgs", "acquire_count": 2, "release_count": 2},
{"id": "compressed-config-second-short", "consumer": "compressed", "function": "decompressor.c:z_erofs_read_cfg", "path": "length released; payload not acquired", "acquire_count": 1, "release_count": 1},
{"id": "compressed-data-decode-error", "consumer": "compressed", "function": "zdata.c:z_erofs_read_extent", "path": "metadata input released after decoder error", "acquire_count": 1, "release_count": 1},
{"id": "compressed-map-success", "consumer": "compressed", "function": "zmap.c:z_erofs_read_extent", "path": "extent record copied then released", "acquire_count": 1, "release_count": 1},
{"id": "compressed-map-short", "consumer": "compressed", "function": "zmap.c:z_erofs_read_extent", "path": "short record fails before acquire", "acquire_count": 0, "release_count": 0},
{"id": "inode-compact-success", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact inode decoded and released", "acquire_count": 1, "release_count": 1},
{"id": "inode-invalid-format", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "first acquired inode released on format rejection", "acquire_count": 1, "release_count": 1},
{"id": "inode-extended-second-short", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact probe released; extended read not acquired", "acquire_count": 1, "release_count": 1},
{"id": "inode-extended-success", "consumer": "inode", "function": "inode.c:erofs_read_inode", "path": "compact probe and extended inode each released", "acquire_count": 2, "release_count": 2},
{"id": "super-read-success", "consumer": "super", "function": "super.c:erofs_read_superblock", "path": "super bytes copied and released before validation", "acquire_count": 1, "release_count": 1},
{"id": "super-read-short", "consumer": "super", "function": "super.c:erofs_read_superblock", "path": "short provider read fails before acquire", "acquire_count": 0, "release_count": 0},
{"id": "super-device-validation-error", "consumer": "super", "function": "super.c:erofs_scan_devices", "path": "device table released through out label", "acquire_count": 1, "release_count": 1},
{"id": "xattr-prefix-success", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length released; prefix transferred then released by caller", "acquire_count": 2, "release_count": 2},
{"id": "xattr-prefix-invalid-length", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length record released before validation error", "acquire_count": 1, "release_count": 1},
{"id": "xattr-prefix-second-short", "consumer": "xattr", "function": "xattr.c:erofs_xattr_read_metadata", "path": "length released; prefix not acquired", "acquire_count": 1, "release_count": 1},
{"id": "xattr-body-validation-error", "consumer": "xattr", "function": "xattr.c:erofs_xattr_load_body", "path": "body released through fail label", "acquire_count": 1, "release_count": 1},
{"id": "xattr-get-iterator-error", "consumer": "xattr", "function": "xattr.c:erofs_getxattr", "path": "body released after inline/shared iterator error", "acquire_count": 1, "release_count": 1}
],
"decision": {
"go": "all required categories observed; every legal tuple equals expected by field; each successful acquisition has one release; short reads acquire nothing; Linux ownership anchors and all audited consumer bodies are present",
"stop": "missing category, legal tuple delta, unexplained errno, incomplete direct consumer inventory, or any acquire/release mismatch",
"h07": "success and request before EOF implies positive m_llen"
},
"timeouts": {"host_seconds": 240, "qemu_seconds": 1200},
"qemu_required_for_this_candidate_gate": false,
"full_feature_suite_required": false
}
tests/pre15/gates/P15-005.sh
Executable
+1018
View File
File diff suppressed because it is too large Load Diff
tests/pre15/gates/P15-006-input.json
+119
View File
@@ -0,0 +1,119 @@
{
"schema": 1,
"gate": "G02",
"candidate": "P15-006",
"required_base": "6673f51152a5195a8a8903aa801f820abce7936e",
"title": "independent common map tuple oracle",
"required_categories": [
"plain",
"inline",
"hole",
"chunk",
"multidevice",
"compressed",
"fragment",
"partial-reference",
"post-EOF",
"bounds",
"overflow",
"invalid"
],
"required_subcategories": [
"plain-boundaries",
"plain-overflow",
"inline-boundaries",
"inline-overflow",
"chunk-raw32",
"chunk-index32",
"chunk-index48",
"chunk-holes",
"chunk-bounds",
"chunk-overflow",
"device-mask",
"device-resolution",
"extent-4",
"extent-8",
"extent-16",
"extent-32",
"compressed-flags",
"compressed-bounds",
"compressed-overflow",
"compressed-invalid",
"cleanup-before-acquire",
"cleanup-after-acquire"
],
"tuple_fields": [
"m_la",
"m_pa",
"m_llen",
"m_plen",
"m_deviceid",
"m_flags",
"m_algorithmformat",
"errno",
"acquire_count",
"release_count"
],
"tuple_byte_layout": "<QQQQH2xIiiII",
"map_flags": {
"EROFS_MAP_MAPPED": 1,
"EROFS_MAP_META": 2,
"EROFS_MAP_PARTIAL_MAPPED": 4,
"EROFS_MAP_PARTIAL_REF": 8,
"EROFS_MAP_FRAGMENT": 16
},
"required_adapter_flag_mask": 31,
"errno": {
"EIO": 5,
"ENODEV": 19,
"EINVAL": 22,
"EOPNOTSUPP": 45,
"EOVERFLOW": 84,
"EINTEGRITY": 97
},
"source_paths": [
"repo-pre-15/src/internal.h",
"repo-pre-15/src/data.c",
"repo-pre-15/src/erofs_fs.h",
"repo-pre-15/src/zmap.c"
],
"protected_function_sha256": {
"data.c:erofs_inline_tail_start": "a2f41930abc764619a0657e293eb47f134c000fb6d95b4437c70cce5234919ae",
"data.c:erofs_check_device_range": "a5bea19931762c755e3112ff1c4bb1c8123133b80fc84ff1f755795ad1d93037",
"data.c:erofs_fill_from_devinfo": "ae9078e6b3dd7416304279bd9420655e19845b606ef79c93af169cb22f153e7a",
"data.c:erofs_map_dev": "de77c2b1f5f12a6a7a9bc0414c42bf1ce1745a60331b0abed8c8df87f5b7dcd7",
"data.c:erofs_map_blocks_chunk": "932ae2d67edc5a3f36c05fa5c374c2e48a0bac48b7a51bf447160df0ac653b43",
"data.c:erofs_bread_device": "c274cd0274ecf79ceea214e123c9d70f3e1f41c6a56449470be554a291ffa808",
"data.c:erofs_bread": "74ae2446f1ce8629642402ff63b83d3258d768fdc20dc2196643e3e1bd0382a3",
"data.c:erofs_brelse": "1d50316f726878cf3390e92eaf6c27072ca8fccc3efceb14fde11f796886e1c5",
"data.c:erofs_put_metabuf": "7f0c546971c5a4bca38582fa21a523d33a8b9231daa3354042b84573b1c0be35",
"data.c:erofs_read_metadata": "b0e00f06d1d008adaab72368fbf593be4ad3b7bea79e086c667b4beeef9b98d7",
"data.c:erofs_read_physical": "aa33e6fcf99ebf46c1eda54da4308ca160a4948f5afbb14038630ce7ce964e09",
"zmap.c:z_erofs_extent_add": "778aa5ad60ed9c3b1affa1e7137966940536c3a05352f14ec04d3e37795d3aa9",
"zmap.c:z_erofs_extent_roundup": "882c4f0d77fdcddd0776699508e80d1061ac788410ed661b21f65902c5bcfb40",
"zmap.c:z_erofs_extent_table_pos": "288999e2dc5cbff5915ec05916229e0d78f01fde0f23da577309c0b31530c173",
"zmap.c:z_erofs_extent_record_pos": "08a5106164bd3dfb5abfec556655659dead8b7a9df86bf699e7828cec1d15920",
"zmap.c:z_erofs_read_extent": "e178fd7aa556ed9b94fcce85d44e52abf4a5410e1036a7d8d14752cecd55d6b0",
"zmap.c:z_erofs_extent_lstart": "5f7b83d23aa9b8128f2f5d509067032e4856e0cff23e6ec35ec4fd703deecb34",
"zmap.c:z_erofs_map_blocks_ext": "996aa16dd7e0a19cce6de9740493a801a41fab7be03a1d2f5717d0e172ee955b",
"zmap.c:z_erofs_map_sanity_check": "83c707d56b42c3fd5e461685889f0dc490ff66ad2b5a34ce7f5bc62c8eddaa1f",
"zmap.c:z_erofs_map_blocks_iter": "b11768dad797a9a0d2ea33a93ea9e3fe3558a1bf9d89f1b22103a75b25a2bd3d"
},
"baseline_legacy_map_sha256": "cac2527aa1399e9d8a803445d8b52f1052569537d5d58d7a76fabe33f53cb4cc",
"candidate_legacy_map_sha256": "f24bdaa8f9508cc4bf55b34f67b748ccdf33743660153d4b27c0b7f32deb9d25",
"map_struct_sha256": "ae883c958dfad3a3df9c6950af0e24723c64523908b901d400c2a3c5eeaf515e",
"expected_map_case_count": 80,
"expected_device_case_count": 13,
"expected_model_sha256": "effcf0b6cc6a6e916eae89a14ad52d7b96273a453556647b6eabf2926fcb4abb",
"decision": {
"go": "all required categories and subcategories are observed; every source-extracted tuple equals the independent model field-by-field and byte-for-byte; device resolution and positive errno match; every metadata acquire is released exactly once; protected producer and GEOM bodies are unchanged",
"stop": "missing chunk, multidevice, bounds, overflow, invalid, compressed, device, flag, errno, or cleanup coverage; model/source tuple difference; byte difference; protected body difference; or non-EOF successful zero run",
"h07": "success and request before EOF implies positive m_llen"
},
"timeouts": {
"host_seconds": 240,
"qemu_seconds": 1200
},
"qemu_required_for_this_candidate_gate": false,
"full_feature_suite_required": false
}
tests/pre15/gates/P15-006.sh
Executable
+1712
View File
File diff suppressed because it is too large Load Diff
tests/pre15/gates/P15-019-input.json
+96
View File
@@ -0,0 +1,96 @@
{
"candidate": "P15-019",
"errno": {
"EINTEGRITY": 97,
"ENAMETOOLONG": 63
},
"expected_fixture_set_sha256": "5e99e5ad9112508991e78ae6e65928973d6b0da78285a059894bed06010cf373",
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"maxpathlen": 1024,
"sha256": {
"sys/kern/vfs_lookup.c": "7958081eda9a137a9cd913b959e86b9ed61eff2f9d6cdb5ce5bcfc8211307f24",
"sys/kern/vfs_syscalls.c": "a37e268b4909fe382650a624eeaef8b6dada182390c90be7fdad8e548c3fd5e7",
"sys/kern/vnode_if.src": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b",
"sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c",
"sys/sys/param.h": "cc451dd2de4d6a7cc97b928ddb9e5b19475cba4ab418153ab7f6e6baa661b917",
"sys/sys/syslimits.h": "7142ae0f262668e984dd462025726bc9eb0d3e8997dda73b02914d1f7b25da3e",
"sys/sys/uio.h": "adb36abeb680d940bbce8566bf82f7dd545e01c519f4e71fa9a2c1a0e1cbbbe8"
}
},
"gate": "G03",
"layouts": {
"chunk": {
"expected_layout": 4,
"short_length": 31
},
"compressed": {
"expected_layout": 3,
"short_length": 64
},
"fragment": {
"expected_layout": 1,
"short_length": 31
},
"inline": {
"expected_layout": 2,
"short_length": 31
},
"plain": {
"expected_layout": 0,
"short_length": 31
}
},
"protected": {
"base_image": "/work/debug-qemu/local/vm-freebsd-build.qcow2.bp",
"pid": 26318,
"port": 9222
},
"required_base": "d645feb720c7022d2138d2a62eb72c022eb75351",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a",
"repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1",
"repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64",
"repo-pre-15/src/inode.c": "dee361837ea104f455c851fe4c268bf5d7e10cd9dd2ef58645aea819137639f6",
"repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9",
"repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242",
"repo-pre-15/src/zmap.c": "95432d49c20f3049e0fb5af21904283494325570673dbb037767c1397c77c1f3",
"src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a",
"src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3",
"src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405"
},
"tools": {
"dump.erofs": {
"path": "/usr/bin/dump.erofs",
"sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c",
"version": "mkfs.erofs (erofs-utils) 1.8.6"
}
},
"utils": {
"head": "7db78788b000999e2de88decd2ba90654f26171c",
"sha256": {
"include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915",
"lib/compress.c": "c38562ebf7b642ee98611d8ba86a782cfcb18fc2abe55c4cbd9a89f3921e7c9e",
"lib/fragments.c": "4d0df4889a3db564e67a4f714584a08b76b0654ef571fa5f1c53cab49212ef5b",
"lib/inode.c": "382bf7ccc22436ab9997a3fa417a0424216a6f3b58a95d7c9a1d182a9e228bc1"
}
},
"uuid": "00000000-0000-0000-0000-000000000019",
"write_set": [
"repo-pre-15/src/internal.h",
"repo-pre-15/src/inode.c",
"repo-pre-15/src/data.c",
"repo-pre-15/src/erofs_vnops.c",
"repo-pre-15/tests/pre15/cases/B16-symlink.sh",
"repo-pre-15/tests/pre15/fixtures/B16-*"
]
}
tests/pre15/gates/P15-019.sh
Executable
+964
View File
@@ -0,0 +1,964 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-019-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output/host"
python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" "$utils_src" <<'PY'
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import stat
import struct
import subprocess
import sys
import tempfile
import traceback
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
UTILS_SRC = Path(sys.argv[6])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
DUT = ROOT / "repo-pre-15"
SUPER = 1024
MAGIC = 0xE0F5E1E2
CRC32C_POLY = 0x82F63B78
FEATURE_COMPAT_SB_CHKSUM = 0x00000001
FEATURE_INCOMPAT_CHUNKED_FILE = 0x00000004
EROFS_FT_REG_FILE = 1
EROFS_FT_SYMLINK = 7
S_IFMT = 0o170000
S_IFREG = 0o100000
S_IFLNK = 0o120000
CHUNK_FORMAT_INDEXES = 0x0020
LAYOUT_NAMES = {
0: "plain",
1: "compressed-full",
2: "inline",
3: "compressed-compact",
4: "chunk",
}
class GateStop(Exception):
pass
class InfraBlocked(Exception):
pass
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def git(path: Path, *args: str) -> str:
completed = subprocess.run(
["git", "-C", str(path), *args],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
if completed.returncode != 0:
raise InfraBlocked(f"git {' '.join(args)} failed: {completed.stdout.strip()}")
return completed.stdout.strip()
COMMANDS: list[dict[str, Any]] = []
def display_arg(value: str, work: Path) -> str:
replacements = (
(str(work), "OWNED_WORK"),
(str(OUTPUT), "GATE_OUTPUT"),
(str(ROOT), "ROOT"),
)
for old, new in replacements:
value = value.replace(old, new)
return value
def run(argv: list[str], work: Path, timeout: int = 30) -> subprocess.CompletedProcess[str]:
try:
completed = subprocess.run(
argv,
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=timeout,
)
except subprocess.TimeoutExpired as error:
raise InfraBlocked(f"command timed out: {' '.join(argv)}") from error
COMMANDS.append(
{
"argv": [display_arg(item, work) for item in argv],
"exit": completed.returncode,
"stdout_sha256": sha256_bytes(completed.stdout.encode("utf-8")),
}
)
return completed
def run_required(argv: list[str], work: Path, timeout: int = 30) -> str:
completed = run(argv, work, timeout)
if completed.returncode != 0:
raise InfraBlocked(
f"command failed ({completed.returncode}): {' '.join(argv)}: "
f"{completed.stdout.strip()}"
)
return completed.stdout
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
value = seed
for byte in data:
value ^= byte
for _ in range(8):
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
return value & 0xFFFFFFFF
class Inode:
def __init__(
self,
nid: int,
offset: int,
inode_format: int,
inode_size: int,
xattr_size: int,
layout: int,
mode: int,
size: int,
start_block: int,
):
self.nid = nid
self.offset = offset
self.inode_format = inode_format
self.inode_size = inode_size
self.xattr_size = xattr_size
self.layout = layout
self.mode = mode
self.size = size
self.start_block = start_block
class DirectoryEntry:
def __init__(self, nid: int, offset: int, file_type: int, name: bytes):
self.nid = nid
self.offset = offset
self.file_type = file_type
self.name = name
class Image:
def __init__(self, data: bytes | bytearray):
self.data = bytearray(data)
if len(self.data) < SUPER + 128 or self.u32(SUPER) != MAGIC:
raise GateStop("fixture is not an EROFS image")
self.block_bits = self.data[SUPER + 12]
if self.block_bits < 9 or self.block_bits > 16:
raise GateStop("fixture has invalid block bits")
self.block_size = 1 << self.block_bits
self.meta_blkaddr = self.u32(SUPER + 40)
self.feature_compat = self.u32(SUPER + 8)
self.feature_incompat = self.u32(SUPER + 80)
self.blocks = self.u32(SUPER + 36)
self.root_nid = self.u16(SUPER + 14)
self.packed_nid = self.u64(SUPER + 96)
if self.blocks == 0 or self.blocks << self.block_bits > len(self.data):
raise GateStop("fixture primary image bounds are invalid")
self.verify_checksum()
@classmethod
def load(cls, path: Path) -> "Image":
return cls(path.read_bytes())
def u16(self, offset: int) -> int:
if offset < 0 or offset + 2 > len(self.data):
raise GateStop("u16 read outside image")
return struct.unpack_from("<H", self.data, offset)[0]
def u32(self, offset: int) -> int:
if offset < 0 or offset + 4 > len(self.data):
raise GateStop("u32 read outside image")
return struct.unpack_from("<I", self.data, offset)[0]
def u64(self, offset: int) -> int:
if offset < 0 or offset + 8 > len(self.data):
raise GateStop("u64 read outside image")
return struct.unpack_from("<Q", self.data, offset)[0]
def put_u16(self, offset: int, value: int) -> None:
struct.pack_into("<H", self.data, offset, value)
def put_u32(self, offset: int, value: int) -> None:
struct.pack_into("<I", self.data, offset, value)
def put_u64(self, offset: int, value: int) -> None:
struct.pack_into("<Q", self.data, offset, value)
@property
def checksum_end(self) -> int:
span = self.block_size - SUPER if self.block_size > SUPER else self.block_size
end = SUPER + span
if end > len(self.data):
raise GateStop("checksum span exceeds image")
return end
def calculated_checksum(self) -> int:
block = bytearray(self.data[SUPER : self.checksum_end])
block[4:8] = bytes(4)
return crc32c(block)
def verify_checksum(self) -> None:
if self.feature_compat & FEATURE_COMPAT_SB_CHKSUM:
if self.u32(SUPER + 4) != self.calculated_checksum():
raise GateStop("fixture checksum is invalid")
def update_checksum(self) -> None:
if not self.feature_compat & FEATURE_COMPAT_SB_CHKSUM:
raise GateStop("fixture does not advertise a checksum")
self.put_u32(SUPER + 4, 0)
self.put_u32(SUPER + 4, self.calculated_checksum())
self.verify_checksum()
def inode(self, nid: int) -> Inode:
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
if offset > len(self.data) - 32:
raise GateStop(f"nid {nid} lies outside the primary image")
inode_format = self.u16(offset)
inode_size = 64 if inode_format & 1 else 32
if offset > len(self.data) - inode_size:
raise GateStop(f"nid {nid} is truncated")
xattr_count = self.u16(offset + 2)
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
size = self.u64(offset + 8) if inode_size == 64 else self.u32(offset + 8)
return Inode(
nid=nid,
offset=offset,
inode_format=inode_format,
inode_size=inode_size,
xattr_size=xattr_size,
layout=(inode_format >> 1) & 7,
mode=self.u16(offset + 4),
size=size,
start_block=self.u32(offset + 16),
)
def uncompressed_data(self, inode: Inode, blob: bytes | None = None) -> bytes:
if inode.layout == 2:
offset = inode.offset + inode.inode_size + inode.xattr_size
if (offset & (self.block_size - 1)) + inode.size > self.block_size:
raise GateStop("inline fixture crosses its metadata block")
source = self.data
elif inode.layout == 0:
offset = inode.start_block << self.block_bits
source = self.data
elif inode.layout == 4:
if blob is None:
raise GateStop("chunk fixture has no external blob")
chunk_format = self.u16(inode.offset + 16)
entry_size = 8 if chunk_format & CHUNK_FORMAT_INDEXES else 4
index_base = (inode.offset + inode.inode_size + inode.xattr_size + entry_size - 1) & ~(entry_size - 1)
output = bytearray()
logical = 0
chunk_size = 1 << (self.block_bits + (chunk_format & 0x1F))
while logical < inode.size:
index = logical // chunk_size
position = index_base + index * entry_size
if entry_size == 8:
high, device_id, low = struct.unpack_from("<HHI", self.data, position)
block = low | high << 32
if device_id != 1:
raise GateStop(f"chunk fixture device id is {device_id}, not 1")
source = blob
else:
block = self.u32(position)
source = self.data
count = min(chunk_size, inode.size - logical)
offset = block << self.block_bits
if offset > len(source) or count > len(source) - offset:
raise GateStop("chunk payload exceeds its device")
output.extend(source[offset : offset + count])
logical += count
return bytes(output)
else:
raise GateStop(f"layout {inode.layout} is not uncompressed")
if offset > len(source) or inode.size > len(source) - offset:
raise GateStop("uncompressed payload exceeds its image")
return bytes(source[offset : offset + inode.size])
def directory_entries(self, inode: Inode) -> list[DirectoryEntry]:
data = self.uncompressed_data(inode)
if len(data) < 12:
raise GateStop("root directory is too short")
first_name = struct.unpack_from("<H", data, 8)[0]
if first_name < 12 or first_name % 12 != 0 or first_name >= len(data):
raise GateStop("root directory has an invalid first name offset")
count = first_name // 12
entries = []
for index in range(count):
slot = index * 12
name_start = struct.unpack_from("<H", data, slot + 8)[0]
name_end = (
struct.unpack_from("<H", data, slot + 20)[0]
if index + 1 < count
else len(data)
)
if name_start >= name_end or name_end > len(data):
raise GateStop("root directory name bounds are invalid")
name = data[name_start:name_end].split(b"\0", 1)[0]
data_offset = (
inode.offset + inode.inode_size + inode.xattr_size
if inode.layout == 2
else inode.start_block << self.block_bits
)
entries.append(
DirectoryEntry(
nid=struct.unpack_from("<Q", data, slot)[0],
offset=data_offset + slot,
file_type=data[slot + 10],
name=name,
)
)
return entries
def resolve(self, name: bytes = b"link") -> tuple[Inode, DirectoryEntry, Inode]:
root = self.inode(self.root_nid)
entry = next((item for item in self.directory_entries(root) if item.name == name), None)
if entry is None:
raise GateStop("fixture root has no /link entry")
return root, entry, self.inode(entry.nid)
def lz4_decode_exact(source: bytes, output_size: int) -> bytes:
ip = 0
output = bytearray()
while ip < len(source):
token = source[ip]
ip += 1
literal_length = token >> 4
if literal_length == 15:
while True:
if ip >= len(source):
raise ValueError("truncated literal length")
value = source[ip]
ip += 1
literal_length += value
if value != 255:
break
if ip + literal_length > len(source):
raise ValueError("truncated literals")
output.extend(source[ip : ip + literal_length])
ip += literal_length
if ip == len(source):
break
if ip + 2 > len(source):
raise ValueError("truncated match offset")
offset = source[ip] | source[ip + 1] << 8
ip += 2
if offset == 0 or offset > len(output):
raise ValueError("invalid match offset")
match_length = token & 15
if match_length == 15:
while True:
if ip >= len(source):
raise ValueError("truncated match length")
value = source[ip]
ip += 1
match_length += value
if value != 255:
break
for _ in range(match_length + 4):
output.append(output[-offset])
if len(output) > output_size:
raise ValueError("decoded output exceeds inode size")
if ip != len(source) or len(output) != output_size:
raise ValueError("raw LZ4 size mismatch")
return bytes(output)
def read_compressed_inline(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]:
header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7
if header > len(image.data) - 8:
raise GateStop("compressed map header exceeds image")
raw0, advise, algorithm, clusterbits = struct.unpack_from("<IHBB", image.data, header)
idata_size = raw0 >> 16
if inode.layout != 3 or not advise & 0x0008 or idata_size == 0:
raise GateStop("compressed fixture is not a compact inline-pcluster inode")
block_end = min((inode.offset | (image.block_size - 1)) + 1, len(image.data))
matches = []
for position in range(header + 8, block_end - idata_size + 1):
encoded = bytes(image.data[position : position + idata_size])
try:
decoded = lz4_decode_exact(encoded, inode.size)
except ValueError:
continue
matches.append((position, decoded))
if len(matches) != 1:
raise GateStop(f"compressed fixture has {len(matches)} independent decode candidates")
return matches[0][1], {
"map_header": header,
"advise": advise,
"algorithm": algorithm & 0x0F,
"clusterbits": clusterbits,
"idata_size": idata_size,
"encoded_offset": matches[0][0],
}
def read_fragment(image: Image, inode: Inode) -> tuple[bytes, dict[str, int]]:
header = (inode.offset + inode.inode_size + inode.xattr_size + 7) & ~7
raw = image.u64(header)
if inode.layout != 1 or raw & (1 << 63) == 0:
raise GateStop("fragment fixture lacks the whole-fragment header")
if image.packed_nid == 0 or image.packed_nid == inode.nid:
raise GateStop("fragment fixture has an invalid packed nid")
fragment_offset = raw ^ (1 << 63)
packed = image.inode(image.packed_nid)
packed_data = image.uncompressed_data(packed)
if fragment_offset > len(packed_data) or inode.size > len(packed_data) - fragment_offset:
raise GateStop("fragment target exceeds packed inode")
return bytes(packed_data[fragment_offset : fragment_offset + inode.size]), {
"map_header": header,
"fragment_offset": fragment_offset,
"packed_nid": image.packed_nid,
"packed_layout": packed.layout,
"packed_size": packed.size,
}
def target_bytes(length: int, nul_position: str | None = None) -> bytes:
pattern = b"p15-019-safe-target/"
target = bytearray((pattern * ((length + len(pattern) - 1) // len(pattern)))[:length])
if nul_position == "first":
target[0] = 0
elif nul_position == "middle":
target[length // 2] = 0
elif nul_position == "last":
target[-1] = 0
return bytes(target)
def protected_state() -> dict[str, Any]:
protected = SPEC["protected"]
base = Path(protected["base_image"])
base_stat = base.stat() if base.exists() else None
pid_path = Path("/proc") / str(protected["pid"])
command = None
if (pid_path / "cmdline").exists():
command = (pid_path / "cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip()
port_hex = f"{protected['port']:04X}"
listeners = 0
for table in (Path("/proc/net/tcp"), Path("/proc/net/tcp6")):
if not table.exists():
continue
for line in table.read_text(encoding="ascii").splitlines()[1:]:
fields = line.split()
if len(fields) >= 4 and fields[1].endswith(f":{port_hex}") and fields[3] == "0A":
listeners += 1
return {
"base": None if base_stat is None else {
"device": base_stat.st_dev,
"inode": base_stat.st_ino,
"mtime_ns": base_stat.st_mtime_ns,
"size": base_stat.st_size,
},
"pid": protected["pid"],
"pid_command_sha256": None if command is None else sha256_bytes(command.encode("utf-8")),
"port": protected["port"],
"port_listener_count": listeners,
}
def verify_identities() -> dict[str, Any]:
required = SPEC["required_base"]
resolved = git(ROOT, "rev-parse", REQUESTED_BASE)
if resolved != required:
raise GateStop(f"requested base {resolved} is not required base {required}")
source_dirty = git(ROOT, "diff", "--name-only", required, "--", "repo-pre-15/src")
if source_dirty:
raise GateStop(f"production source differs from frozen base: {source_dirty}")
source_hashes = {}
for relative, expected in SPEC["source_sha256"].items():
content = subprocess.run(
["git", "-C", str(ROOT), "show", f"{required}:{relative}"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if content.returncode != 0:
raise InfraBlocked(f"cannot read frozen source {relative}")
actual = sha256_bytes(content.stdout)
if actual != expected:
raise GateStop(f"frozen source hash differs for {relative}")
source_hashes[relative] = actual
if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd"]["head"]:
raise GateStop("FreeBSD source HEAD differs")
for relative, expected in SPEC["freebsd"]["sha256"].items():
if sha256_path(FREEBSD_SRC / relative) != expected:
raise GateStop(f"FreeBSD source hash differs for {relative}")
if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils"]["head"]:
raise GateStop("erofs-utils source HEAD differs")
for relative, expected in SPEC["utils"]["sha256"].items():
actual = sha256_path(UTILS_SRC / relative)
if actual != expected:
raise GateStop(f"erofs-utils source hash differs for {relative}")
for name, tool in SPEC["tools"].items():
path = Path(tool["path"])
if sha256_path(path) != tool["sha256"]:
raise GateStop(f"tool hash differs for {name}")
version = subprocess.run(
[SPEC["tools"]["mkfs.erofs"]["path"], "-V"],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
).stdout.splitlines()[0]
if version != SPEC["tools"]["mkfs.erofs"]["version"]:
raise GateStop(f"mkfs version differs: {version}")
return {"base": resolved, "source_sha256": source_hashes, "mkfs_version": version}
def verify_semantics() -> dict[str, Any]:
linux = (ROOT / "src-linux/inode.c").read_text(encoding="utf-8")
freebsd_inode = (DUT / "src/inode.c").read_text(encoding="utf-8")
freebsd_data = (DUT / "src/data.c").read_text(encoding="utf-8")
freebsd_vnops = (DUT / "src/erofs_vnops.c").read_text(encoding="utf-8")
lookup = (FREEBSD_SRC / "sys/kern/vfs_lookup.c").read_text(encoding="utf-8")
errno_h = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
syslimits = (FREEBSD_SRC / "sys/sys/syslimits.h").read_text(encoding="utf-8")
linux_markers = (
"vi->datalayout == EROFS_INODE_FLAT_INLINE",
"kmemdup_nul(bptr + ofs, inode->i_size, GFP_KERNEL)",
"!inode->i_size || strlen(link) != inode->i_size",
"return -EFSCORRUPTED",
".get_link = page_get_link",
".get_link = simple_get_link",
)
freebsd_markers = (
"return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio));",
"return (erofs_readlink_target(ap->a_vp, ap->a_uio));",
".vop_readlink = erofs_readlink",
)
if not all(marker in linux for marker in linux_markers):
raise GateStop("Linux fast/page symlink path anchors changed")
if not all(marker in freebsd_data + freebsd_vnops for marker in freebsd_markers):
raise GateStop("FreeBSD vnode/readlink path anchors changed")
if "erofs_validate_symlink" in freebsd_inode + freebsd_data + freebsd_vnops:
raise GateStop("P15-019 production implementation already exists at the gate base")
if "aiov.iov_len = MAXPATHLEN;" not in lookup or "linklen == 0" not in lookup:
raise GateStop("FreeBSD namei readlink boundary anchors changed")
expected_errno = SPEC["errno"]
for name, value in expected_errno.items():
if re.search(rf"#define\s+{name}\s+{value}\b", errno_h) is None:
raise GateStop(f"FreeBSD errno {name} is not {value}")
maxpath = SPEC["freebsd"]["maxpathlen"]
if re.search(rf"#define\s+PATH_MAX\s+{maxpath}\b", syslimits) is None:
raise GateStop("FreeBSD PATH_MAX differs")
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", freebsd_inode + freebsd_data + freebsd_vnops):
raise GateStop("FreeBSD EROFS source contains Linux negative errno returns")
return {
"linux": {
"fast_inline_validation": True,
"noninline_get_link": "page_get_link",
"errno_convention": "negative Linux errno",
},
"freebsd": {
"entry": "VOP_READLINK",
"path": "vnode -> erofs_readlink_target -> map/read -> GEOM or compressed backing",
"maxpathlen": maxpath,
"empty_namei_baseline": "ENOENT after zero-byte VOP_READLINK success",
"errno_convention": "positive FreeBSD errno",
},
"candidate": {
"empty": "EINTEGRITY",
"embedded_nul": "EINTEGRITY before uiomove",
"oversize": "ENAMETOOLONG before target I/O",
"normal": "exact byte target; no trailing NUL required",
},
}
def make_source(source: Path, payload: bytes, native_symlink: bool) -> None:
source.mkdir()
source.chmod(0o755)
link = source / "link"
if native_symlink:
os.symlink(payload.decode("ascii"), link)
else:
link.write_bytes(payload)
link.chmod(0o644)
os.utime(link, (0, 0), follow_symlinks=False)
os.utime(source, (0, 0), follow_symlinks=False)
def mkfs_case(
layout: str,
case_id: str,
payload: bytes,
native_symlink: bool,
work: Path,
fixtures: Path,
) -> tuple[Path, Path | None, list[int]]:
source = work / f"source-{layout}-{case_id}"
make_source(source, payload, native_symlink)
image_path = fixtures / f"{layout}-{case_id}.erofs"
blob_path = fixtures / f"{layout}-{case_id}.blob" if layout == "chunk" else None
options: list[str] = []
if layout == "plain":
options = ["-E^inline_data"]
elif layout == "chunk":
assert blob_path is not None
blob_path.write_bytes(b"")
options = ["--chunksize=4096", f"--blobdev={blob_path}"]
elif layout == "compressed":
options = ["-zlz4", "-C4096", "-Eztailpacking"]
elif layout == "fragment":
options = ["-zlz4", "-C4096", "-Eall-fragments"]
command = [
SPEC["tools"]["mkfs.erofs"]["path"],
"-T0",
"--all-time",
"--all-root",
"--workers=1",
"--sort=path",
"-x-1",
f"-U{SPEC['uuid']}",
*options,
str(image_path),
str(source),
]
run_required(command, work)
before = image_path.read_bytes()
image = Image(before)
_, entry, inode = image.resolve()
changed = []
if stat.S_IFMT(inode.mode) == S_IFREG:
image.put_u16(inode.offset + 4, S_IFLNK | 0o777)
image.data[entry.offset + 10] = EROFS_FT_SYMLINK
elif stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK:
raise GateStop(f"{layout}/{case_id}: source inode type is unexpected")
if case_id == "empty":
if inode.inode_size == 64:
image.put_u64(inode.offset + 8, 0)
else:
image.put_u32(inode.offset + 8, 0)
image.update_checksum()
after = bytes(image.data)
for offset, (old, new) in enumerate(zip(before, after)):
if old != new:
changed.append(offset)
allowed = set(range(SUPER + 4, SUPER + 8))
allowed.update(range(inode.offset + 4, inode.offset + 6))
allowed.add(entry.offset + 10)
if case_id == "empty":
allowed.update(range(inode.offset + 8, inode.offset + (16 if inode.inode_size == 64 else 12)))
unexpected = sorted(set(changed) - allowed)
if unexpected:
raise GateStop(f"{layout}/{case_id}: transform changed unexpected offsets {unexpected}")
image_path.write_bytes(after)
return image_path, blob_path, changed
def expected_case(layout: str, case_id: str) -> tuple[int, bytes, str | None, str]:
short = SPEC["layouts"][layout]["short_length"]
if case_id == "normal-short":
return short, target_bytes(short), None, "PASS"
if case_id == "normal-max":
limit = SPEC["freebsd"]["maxpathlen"]
return limit, target_bytes(limit), None, "PASS"
if case_id == "empty":
return 0, target_bytes(short), None, "EINTEGRITY"
if case_id.startswith("nul-"):
position = case_id.removeprefix("nul-")
limit = SPEC["freebsd"]["maxpathlen"]
return limit, target_bytes(limit, position), position, "EINTEGRITY"
if case_id == "too-long":
length = SPEC["freebsd"]["maxpathlen"] + 1
return length, target_bytes(length), None, "ENAMETOOLONG"
raise AssertionError(case_id)
def independent_target(image: Image, inode: Inode, blob: bytes | None) -> tuple[bytes, dict[str, Any]]:
if inode.layout in (0, 2, 4):
return image.uncompressed_data(inode, blob), {}
if inode.layout == 3:
return read_compressed_inline(image, inode)
if inode.layout == 1:
return read_fragment(image, inode)
raise GateStop(f"unsupported target layout {inode.layout}")
def oracle(image: Image, inode: Inode, blob: bytes | None) -> tuple[str, bytes | None, dict[str, Any]]:
if inode.size == 0:
return "EINTEGRITY", None, {"point": "symlink.empty"}
if inode.size > SPEC["freebsd"]["maxpathlen"]:
return "ENAMETOOLONG", None, {"point": "symlink.maxpathlen"}
target, details = independent_target(image, inode, blob)
nul = target.find(b"\0")
if nul >= 0:
return "EINTEGRITY", target, {"point": "symlink.embedded-nul", "nul_offset": nul, **details}
return "PASS", target, {"point": "symlink.valid", **details}
def fixture_set_hash(fixtures: Path) -> tuple[str, dict[str, str]]:
hashes = {}
for path in sorted(item for item in fixtures.iterdir() if item.is_file()):
if path.name == "SHA256SUMS":
continue
hashes[path.name] = sha256_path(path)
lines = "".join(f"{digest} {name}\n" for name, digest in hashes.items())
(fixtures / "SHA256SUMS").write_text(lines, encoding="ascii")
return sha256_bytes(lines.encode("ascii")), hashes
def execute() -> dict[str, Any]:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-019" or SPEC.get("gate") != "G03":
raise InfraBlocked("input identity is invalid")
identity = verify_identities()
semantics = verify_semantics()
write_json(OUTPUT / "semantics.json", semantics)
fixtures = OUTPUT / "fixtures"
fixtures.mkdir()
cases = (
"normal-short",
"normal-max",
"empty",
"nul-first",
"nul-middle",
"nul-last",
"too-long",
)
records = []
with tempfile.TemporaryDirectory(prefix=".p15-019-work-", dir=OUTPUT) as raw_work:
work = Path(raw_work)
for layout, layout_spec in sorted(SPEC["layouts"].items()):
for case_id in cases:
expected_size, payload, nul_position, expected = expected_case(layout, case_id)
native = layout == "inline" and nul_position is None
image_path, blob_path, changed = mkfs_case(
layout, case_id, payload, native, work, fixtures
)
image = Image.load(image_path)
_, entry, inode = image.resolve()
if inode.layout != layout_spec["expected_layout"]:
raise GateStop(
f"{layout}/{case_id}: layout {inode.layout} != {layout_spec['expected_layout']}"
)
if stat.S_IFMT(inode.mode) != S_IFLNK or entry.file_type != EROFS_FT_SYMLINK:
raise GateStop(f"{layout}/{case_id}: transformed inode is not a symlink")
if inode.size != expected_size:
raise GateStop(f"{layout}/{case_id}: inode size {inode.size} != {expected_size}")
blob = blob_path.read_bytes() if blob_path is not None else None
observed, target, details = oracle(image, inode, blob)
if observed != expected:
raise GateStop(f"{layout}/{case_id}: oracle {observed} != {expected}")
if expected != "EINTEGRITY" or case_id == "empty":
if case_id != "empty" and target is not None and target != payload:
raise GateStop(f"{layout}/{case_id}: target bytes differ from source")
if case_id.startswith("nul-"):
expected_nul = {"first": 0, "middle": len(payload) // 2, "last": len(payload) - 1}[nul_position]
if details.get("nul_offset") != expected_nul:
raise GateStop(f"{layout}/{case_id}: NUL offset differs")
dump = run(
[SPEC["tools"]["dump.erofs"]["path"], "--path=/link", str(image_path)],
work,
)
dump_layout = re.search(r"Layout:\s+(\d+)", dump.stdout)
dump_mode = "symlink file" in dump.stdout
if dump.returncode != 0 or dump_layout is None or int(dump_layout.group(1)) != inode.layout or not dump_mode:
raise GateStop(f"{layout}/{case_id}: dump.erofs cross-check failed")
fsck_argv = [SPEC["tools"]["fsck.erofs"]["path"], "-d0"]
if blob_path is not None:
fsck_argv.append(f"--device={blob_path}")
fsck_argv.append(str(image_path))
fsck = run(fsck_argv, work)
extracted = False
if expected == "PASS":
extract = work / f"extract-{layout}-{case_id}"
extract.mkdir()
extract_argv = [
SPEC["tools"]["fsck.erofs"]["path"],
f"--extract={extract}",
"--no-preserve",
]
if blob_path is not None:
extract_argv.append(f"--device={blob_path}")
extract_argv.append(str(image_path))
run_required(extract_argv, work)
extracted_link = extract / "link"
if not extracted_link.is_symlink() or os.readlink(extracted_link).encode("ascii") != payload:
raise GateStop(f"{layout}/{case_id}: fsck extraction target differs")
extracted = True
records.append(
{
"case": case_id,
"changed_offsets": changed,
"dump_cross_check": "PASS",
"expected": expected,
"fsck_check_exit": fsck.returncode,
"fsck_extract": extracted,
"image": image_path.name,
"inode": {
"layout": inode.layout,
"layout_name": LAYOUT_NAMES[inode.layout],
"mode": oct(inode.mode),
"nid": inode.nid,
"size": inode.size,
},
"layout": layout,
"oracle": details,
"source_payload_sha256": sha256_bytes(payload),
"target_sha256": None if target is None else sha256_bytes(target),
}
)
fixture_sha256, hashes = fixture_set_hash(fixtures)
expected_fixture = SPEC.get("expected_fixture_set_sha256", "")
if expected_fixture and fixture_sha256 != expected_fixture:
raise GateStop(
f"fixture set hash {fixture_sha256} != frozen {expected_fixture}"
)
write_json(OUTPUT / "cases.json", records)
write_json(OUTPUT / "commands.json", COMMANDS)
by_layout = {
layout: {
record["case"]: record["expected"]
for record in records
if record["layout"] == layout
}
for layout in sorted(SPEC["layouts"])
}
required = set(cases)
if any(set(result) != required for result in by_layout.values()):
raise GateStop("one or more layouts lack a complete oracle")
return {
"candidate": "P15-019",
"case_count": len(records),
"decision": "GO",
"fixture_file_count": len(hashes),
"fixture_set_sha256": fixture_sha256,
"gate": "G03",
"identity": identity,
"layout_results": by_layout,
"qemu": "NOT_RUN: Stage0 host fixture/oracle is complete",
"source_modified": False,
}
before_protected = protected_state()
status = "INFRA_BLOCKED"
exit_code = 2
result: dict[str, Any]
try:
result = execute()
status = "GO"
exit_code = 0
except GateStop as error:
status = "STOP"
exit_code = 1
result = {"candidate": "P15-019", "decision": status, "reason": str(error)}
except (InfraBlocked, OSError, ValueError, KeyError, json.JSONDecodeError) as error:
result = {"candidate": "P15-019", "decision": status, "reason": str(error)}
except Exception as error:
result = {
"candidate": "P15-019",
"decision": status,
"reason": f"unexpected gate error: {error}",
"traceback": traceback.format_exc(),
}
after_protected = protected_state()
cleanup = {
"owned_temp_remaining": sorted(path.name for path in OUTPUT.glob(".p15-019-work-*")),
"protected_before": before_protected,
"protected_after": after_protected,
"protected_unchanged": before_protected == after_protected,
}
write_json(OUTPUT / "cleanup.json", cleanup)
if cleanup["owned_temp_remaining"] or not cleanup["protected_unchanged"]:
status = "INFRA_BLOCKED"
exit_code = 2
result = {
"candidate": "P15-019",
"decision": status,
"reason": "cleanup or protected-resource identity changed",
}
result["cleanup_sha256"] = sha256_path(OUTPUT / "cleanup.json")
result["input_sha256"] = sha256_path(INPUT)
write_json(OUTPUT / "result.json", result)
print(f"P15-019 G03: {status}")
if "fixture_set_sha256" in result:
print(f"fixture_set_sha256={result['fixture_set_sha256']}")
if "reason" in result:
print(f"reason={result['reason']}")
sys.exit(exit_code)
PY
tests/pre15/gates/P15-021-input.json
+116
View File
@@ -0,0 +1,116 @@
{
"batch": "B19b",
"candidate": "P15-021",
"concurrency": {
"loops_per_worker": 20,
"workers": 64
},
"errno": {
"EINTEGRITY": 97,
"ENOATTR": 87
},
"fixture": {
"attribute_count": 8,
"attribute_value_bytes": 96,
"peer_count": 64,
"target": "/target.bin",
"uuid": "00000000-0000-0000-0000-000000000021"
},
"format": {
"bits": 32,
"feature_compat": 4,
"filter_default": 4294967295,
"seed": 633069711
},
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"public_xxh32_api": false,
"sha256": {
"sys/contrib/openzfs/module/zstd/lib/common/xxhash.h": "ba65364fb0532ef2848b35c35fbbb5172f5037f56717cb2ac562812993385f27",
"sys/contrib/zstd/lib/common/xxhash.c": "8444d064922f434b67b708a987981aa21dfeece735b48c82d31f7027d0ccef03",
"sys/contrib/zstd/lib/common/xxhash.h": "8cb837b21a8fe9a6b9dbcd0961ab16e733bfcbfa9e003f3a496ce07ae80aa8ee",
"sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806",
"sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c",
"sys/sys/libkern.h": "61f74a2ca87b0be5deb3f1aa7a29f188709331a98d5bd5a744ecd9d795a8f91b"
}
},
"gate": "G03",
"implementation": {
"kind": "file-local namespaced",
"symbol": "erofs_xxh32"
},
"linux_source_sha256": {
"src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93",
"src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a"
},
"random": {
"count": 1000000,
"seed": 20260815
},
"required_base": "666e52f710363df07f7c93919eb835d41092d011",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1",
"repo-pre-15/src/internal.h": "3a9f8bc80250a6fdccb0f0ebb2e13935edfe4b416a4d710104415b3991093bcf",
"repo-pre-15/src/xattr.c": "29d1cfe572a4dee9c003cf27f1d44301d707018087e92412781d977b752d3b6b"
},
"thresholds": {
"cold_loops_per_sample": 200,
"cold_samples": 5,
"minimum_provider_metadata_read_reduction_percent": 25.0
},
"tools": {
"cc": {
"path": "/usr/bin/cc",
"sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f"
},
"dump.erofs": {
"path": "/usr/bin/dump.erofs",
"sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"libxxhash": {
"path": "/usr/lib/x86_64-linux-gnu/libxxhash.so.0.8.3",
"sha256": "7dd49b353facbee50c371d0559ce29db5afee5ac33249ca5c7068420fb642762"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c",
"version": "mkfs.erofs (erofs-utils) 1.8.6"
}
},
"utils": {
"head": "7db78788b000999e2de88decd2ba90654f26171c",
"sha256": {
"include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915",
"lib/xattr.c": "8e1625b858f6183756ff8a2e1a58d6e10a8408c8845c1c505abb68120daf832b",
"lib/xxhash.c": "41859bc203785cfc8035b63b024ad23c9f099890a0a33d9c13dcb5401e6b9106"
}
},
"vectors": [
{"hash": 3935488656, "index": 1, "name_hex": "", "bit": 16},
{"hash": 924486869, "index": 1, "name_hex": "61", "bit": 21},
{"hash": 3124601740, "index": 1, "name_hex": "616c706861", "bit": 12},
{"hash": 1029047478, "index": 4, "name_hex": "747275737465642e6e616d65", "bit": 22},
{"hash": 2585031559, "index": 6, "name_hex": "30313233343536373839616263646566", "bit": 7},
{"hash": 1765035109, "index": 2, "name_hex": "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "bit": 5},
{"hash": 596111791, "index": 3, "name_hex": "00010203fcfdfeff", "bit": 15}
],
"write_set": {
"gate": [
"repo-pre-15/docs/pre15-stage0/P15-021.md",
"repo-pre-15/tests/pre15/gates/P15-021-input.json",
"repo-pre-15/tests/pre15/gates/P15-021.sh"
],
"source": [
"repo-pre-15/src/erofs_fs.h",
"repo-pre-15/src/internal.h",
"repo-pre-15/src/xattr.c",
"repo-pre-15/tests/pre15/cases/B19b-xattr-bloom.sh",
"repo-pre-15/tests/pre15/fixtures/B19b-*"
]
}
}
tests/pre15/gates/P15-021.sh
Executable
+892
View File
@@ -0,0 +1,892 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-021-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-master}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
for tool in cc fsck.erofs git mkfs.erofs python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 -B - "$root" "$input" "$base" "$output" "$freebsd_src" "$utils_src" <<'PY'
from __future__ import annotations
from array import array
from concurrent.futures import ThreadPoolExecutor
import ctypes
import hashlib
import json
import os
from pathlib import Path
import random
import shutil
import statistics
import struct
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
UTILS_SRC = Path(sys.argv[6])
DUT = ROOT / "repo-pre-15"
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
SUPER = 1024
MAGIC = 0xE0F5E1E2
CRC32C_POLY = 0x82F63B78
class GateStop(RuntimeError):
pass
class InfraBlocked(RuntimeError):
pass
class Reject(RuntimeError):
def __init__(self, errno_name: str, point: str):
super().__init__(f"{errno_name} at {point}")
self.errno_name = errno_name
self.point = point
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def write_json(path: Path, value: Any) -> None:
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def run(argv: list[str], *, input_bytes: bytes | None = None) -> subprocess.CompletedProcess:
completed = subprocess.run(
argv,
input=input_bytes,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=120,
)
if completed.returncode != 0:
output = (completed.stdout + completed.stderr).decode("utf-8", "replace")
raise InfraBlocked(f"command failed ({completed.returncode}): {' '.join(argv)}: {output.strip()}")
return completed
def git(path: Path, *args: str) -> str:
return run(["git", "-C", str(path), *args]).stdout.decode().strip()
def source_at(commit: str, relative: str) -> bytes:
return run(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]).stdout
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
value = seed
for byte in data:
value ^= byte
for _ in range(8):
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
return value & 0xFFFFFFFF
def rotl32(value: int, count: int) -> int:
return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF
def oracle_xxh32(data: bytes, seed: int) -> int:
prime1 = 2654435761
prime2 = 2246822519
prime3 = 3266489917
prime4 = 668265263
prime5 = 374761393
cursor = 0
end = len(data)
def round32(accumulator: int, lane: int) -> int:
accumulator = (accumulator + lane * prime2) & 0xFFFFFFFF
return (rotl32(accumulator, 13) * prime1) & 0xFFFFFFFF
if end >= 16:
accumulator1 = (seed + prime1 + prime2) & 0xFFFFFFFF
accumulator2 = (seed + prime2) & 0xFFFFFFFF
accumulator3 = seed & 0xFFFFFFFF
accumulator4 = (seed - prime1) & 0xFFFFFFFF
limit = end - 16
while cursor <= limit:
accumulator1 = round32(accumulator1, int.from_bytes(data[cursor:cursor + 4], "little"))
accumulator2 = round32(accumulator2, int.from_bytes(data[cursor + 4:cursor + 8], "little"))
accumulator3 = round32(accumulator3, int.from_bytes(data[cursor + 8:cursor + 12], "little"))
accumulator4 = round32(accumulator4, int.from_bytes(data[cursor + 12:cursor + 16], "little"))
cursor += 16
value = (
rotl32(accumulator1, 1)
+ rotl32(accumulator2, 7)
+ rotl32(accumulator3, 12)
+ rotl32(accumulator4, 18)
) & 0xFFFFFFFF
else:
value = (seed + prime5) & 0xFFFFFFFF
value = (value + end) & 0xFFFFFFFF
while cursor + 4 <= end:
value = (value + int.from_bytes(data[cursor:cursor + 4], "little") * prime3) & 0xFFFFFFFF
value = (rotl32(value, 17) * prime4) & 0xFFFFFFFF
cursor += 4
while cursor < end:
value = (value + data[cursor] * prime5) & 0xFFFFFFFF
value = (rotl32(value, 11) * prime1) & 0xFFFFFFFF
cursor += 1
value ^= value >> 15
value = (value * prime2) & 0xFFFFFFFF
value ^= value >> 13
value = (value * prime3) & 0xFFFFFFFF
value ^= value >> 16
return value & 0xFFFFFFFF
CANDIDATE_SOURCE = r'''#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
static uint32_t
erofs_xxh32_rotl(uint32_t value, unsigned int count)
{
return ((value << count) | (value >> (32 - count)));
}
static uint32_t
erofs_xxh32_round(uint32_t seed, uint32_t input)
{
seed += input * UINT32_C(2246822519);
seed = erofs_xxh32_rotl(seed, 13);
return (seed * UINT32_C(2654435761));
}
static uint32_t
erofs_xxh32_le32(const uint8_t *input)
{
return ((uint32_t)input[0] | (uint32_t)input[1] << 8 |
(uint32_t)input[2] << 16 | (uint32_t)input[3] << 24);
}
static uint32_t
erofs_xxh32(const void *input, size_t length, uint32_t seed)
{
const uint8_t *cursor = input;
const uint8_t *end = cursor + length;
uint32_t hash;
if (length >= 16) {
const uint8_t *limit = end - 16;
uint32_t v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519);
uint32_t v2 = seed + UINT32_C(2246822519);
uint32_t v3 = seed;
uint32_t v4 = seed - UINT32_C(2654435761);
do {
v1 = erofs_xxh32_round(v1, erofs_xxh32_le32(cursor));
cursor += 4;
v2 = erofs_xxh32_round(v2, erofs_xxh32_le32(cursor));
cursor += 4;
v3 = erofs_xxh32_round(v3, erofs_xxh32_le32(cursor));
cursor += 4;
v4 = erofs_xxh32_round(v4, erofs_xxh32_le32(cursor));
cursor += 4;
} while (cursor <= limit);
hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) +
erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18);
} else {
hash = seed + UINT32_C(374761393);
}
hash += (uint32_t)length;
while (cursor + 4 <= end) {
hash += erofs_xxh32_le32(cursor) * UINT32_C(3266489917);
hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263);
cursor += 4;
}
while (cursor < end) {
hash += *cursor++ * UINT32_C(374761393);
hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761);
}
hash ^= hash >> 15;
hash *= UINT32_C(2246822519);
hash ^= hash >> 13;
hash *= UINT32_C(3266489917);
hash ^= hash >> 16;
return (hash);
}
int
main(void)
{
uint8_t header[3], *name;
uint32_t hash, seed;
size_t length;
while (fread(header, sizeof(header), 1, stdin) == 1) {
length = (size_t)header[1] | (size_t)header[2] << 8;
name = malloc(length == 0 ? 1 : length);
if (name == NULL || (length != 0 && fread(name, length, 1, stdin) != 1))
return (2);
seed = UINT32_C(0x25BBE08F) + header[0];
hash = erofs_xxh32(name, length, seed);
free(name);
if (fwrite(&hash, sizeof(hash), 1, stdout) != 1)
return (3);
}
return (ferror(stdin) ? 4 : 0);
}
'''
def candidate_hashes(binary: Path, records: list[tuple[int, bytes]]) -> list[int]:
payload = bytearray()
for index, name in records:
if len(name) > 65535:
raise GateStop("candidate vector name exceeds protocol")
payload.extend((index, len(name) & 0xFF, len(name) >> 8))
payload.extend(name)
output = run([str(binary)], input_bytes=bytes(payload)).stdout
if len(output) != len(records) * 4:
raise GateStop("candidate xxh32 returned a truncated vector stream")
return list(struct.unpack(f"<{len(records)}I", output))
def verify_identity() -> dict[str, Any]:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-021" or SPEC.get("gate") != "G03":
raise InfraBlocked("invalid P15-021 input identity")
resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise InfraBlocked(f"P15-021 must replay {SPEC['required_base']}, got {resolved}")
status_lines = git(ROOT, "status", "--porcelain=v1", "--untracked-files=all").splitlines()
changed = sorted(line[3:] for line in status_lines if len(line) >= 4)
if changed not in ([], sorted(SPEC["write_set"]["gate"])):
raise InfraBlocked(f"gate worktree write set differs: {changed}")
source_hashes = {relative: sha256_bytes(source_at(resolved, relative)) for relative in SPEC["source_sha256"]}
if source_hashes != SPEC["source_sha256"]:
raise InfraBlocked("frozen DUT source identity changed")
linux_hashes = {relative: sha256_bytes(source_at(resolved, relative)) for relative in SPEC["linux_source_sha256"]}
if linux_hashes != SPEC["linux_source_sha256"]:
raise InfraBlocked("frozen Linux source identity changed")
freebsd_head = git(FREEBSD_SRC, "rev-parse", "HEAD")
freebsd_hashes = {relative: sha256_path(FREEBSD_SRC / relative) for relative in SPEC["freebsd"]["sha256"]}
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
raise InfraBlocked("frozen FreeBSD source identity changed")
utils_head = git(UTILS_SRC, "rev-parse", "HEAD")
utils_hashes = {relative: sha256_path(UTILS_SRC / relative) for relative in SPEC["utils"]["sha256"]}
if utils_head != SPEC["utils"]["head"] or utils_hashes != SPEC["utils"]["sha256"]:
raise InfraBlocked("frozen erofs-utils identity changed")
tool_hashes = {name: sha256_path(Path(item["path"])) for name, item in SPEC["tools"].items()}
expected_tools = {name: item["sha256"] for name, item in SPEC["tools"].items()}
if tool_hashes != expected_tools:
raise InfraBlocked("frozen host tool identity changed")
mkfs_version = run([SPEC["tools"]["mkfs.erofs"]["path"], "-V"]).stdout.decode().splitlines()[0]
if mkfs_version != SPEC["tools"]["mkfs.erofs"]["version"]:
raise InfraBlocked("mkfs.erofs version changed")
linux_header = source_at(resolved, "src-linux/erofs_fs.h").decode("utf-8")
linux_xattr = source_at(resolved, "src-linux/xattr.c").decode("utf-8")
for marker in (
"#define EROFS_XATTR_FILTER_BITS\t\t32",
"#define EROFS_XATTR_FILTER_SEED\t\t0x25BBE08F",
"bit value 1 indicates not-present",
):
if marker not in linux_header:
raise InfraBlocked(f"Linux Bloom format marker changed: {marker}")
for marker in (
"xxh32(name, strlen(name)",
"EROFS_XATTR_FILTER_SEED + index",
"vi->xattr_name_filter & (1U << hashbit)",
"!sbi->xattr_filter_reserved",
):
if marker not in linux_xattr:
raise InfraBlocked(f"Linux Bloom use-site marker changed: {marker}")
public_text = (FREEBSD_SRC / "sys/sys/libkern.h").read_text(encoding="utf-8")
if "xxh32" in public_text.lower():
raise InfraBlocked("FreeBSD gained a public xxh32 API; re-audit namespacing")
return {
"base": resolved,
"freebsd_head": freebsd_head,
"freebsd_sha256": freebsd_hashes,
"implementation": SPEC["implementation"],
"linux_sha256": linux_hashes,
"mkfs_version": mkfs_version,
"source_sha256": source_hashes,
"tool_sha256": tool_hashes,
"utils_head": utils_head,
"utils_sha256": utils_hashes,
}
def create_source(path: Path) -> None:
path.mkdir(parents=True)
paths = [path / f"peer-{index:03d}.bin" for index in range(SPEC["fixture"]["peer_count"])]
paths.append(path / "target.bin")
for entry in paths:
entry.write_bytes(b"P15-021\n")
entry.chmod(0o644)
for index in range(SPEC["fixture"]["attribute_count"]):
name = f"user.attr{index:02d}".encode()
prefix = f"value-{index:02d}-".encode()
value = (prefix * (SPEC["fixture"]["attribute_value_bytes"] // len(prefix) + 1))[
: SPEC["fixture"]["attribute_value_bytes"]
]
os.setxattr(entry, name, value)
os.utime(entry, (0, 0), follow_symlinks=False)
os.utime(path, (0, 0), follow_symlinks=False)
def build_fixture(source: Path, image: Path) -> list[str]:
command = [
SPEC["tools"]["mkfs.erofs"]["path"],
"-d0",
"-T0",
"--all-time",
"--all-root",
"--workers=1",
"--sort=path",
f"-U{SPEC['fixture']['uuid']}",
"-x2",
"-Exattr-name-filter,force-inode-extended",
str(image),
str(source),
]
run(command)
return command
class Image:
def __init__(self, path: Path):
self.path = path
self.fd = os.open(path, os.O_RDONLY)
self.size = os.fstat(self.fd).st_size
self.calls = 0
self.bytes = 0
self.blocks: set[int] = set()
header = self.read(SUPER, 144, "super")
if struct.unpack_from("<I", header)[0] != MAGIC:
raise Reject("EINTEGRITY", "super.magic")
self.feature_compat = struct.unpack_from("<I", header, 8)[0]
self.block_bits = header[12]
self.block_size = 1 << self.block_bits
self.root_nid = struct.unpack_from("<H", header, 14)[0]
self.blocks_count = struct.unpack_from("<I", header, 36)[0]
self.limit = self.blocks_count << self.block_bits
self.meta_blkaddr = struct.unpack_from("<I", header, 40)[0]
self.xattr_blkaddr = struct.unpack_from("<I", header, 44)[0]
self.filter_reserved = header[104]
if self.limit > self.size:
raise Reject("EINTEGRITY", "super.bounds")
def close(self) -> None:
os.close(self.fd)
def __enter__(self) -> "Image":
return self
def __exit__(self, *_: object) -> None:
self.close()
def reset_reads(self) -> None:
self.calls = 0
self.bytes = 0
self.blocks.clear()
if hasattr(os, "posix_fadvise"):
os.posix_fadvise(self.fd, 0, 0, os.POSIX_FADV_DONTNEED)
def read(self, offset: int, length: int, point: str) -> bytes:
limit = self.limit if hasattr(self, "limit") else self.size
if offset < 0 or length < 0 or offset > limit:
raise Reject("EINTEGRITY", f"{point}.bounds")
if length > limit - offset:
raise Reject("EINTEGRITY", f"{point}.bounds")
data = os.pread(self.fd, length, offset)
if len(data) != length:
raise Reject("EINTEGRITY", f"{point}.short")
self.calls += 1
self.bytes += length
if length:
first = offset // 4096
last = (offset + length - 1) // 4096
self.blocks.update(range(first, last + 1))
return data
def inode(self, nid: int) -> dict[str, int]:
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
raw = self.read(offset, 64, "inode")
inode_format, xattr_count = struct.unpack_from("<HH", raw)
inode_size = 64 if inode_format & 1 else 32
xattr_size = 0 if xattr_count == 0 else 12 + 4 * (xattr_count - 1)
size = struct.unpack_from("<Q" if inode_size == 64 else "<I", raw, 8)[0]
return {
"nid": nid,
"offset": offset,
"inode_size": inode_size,
"xattr_size": xattr_size,
"layout": (inode_format >> 1) & 7,
"size": size,
"start_block": struct.unpack_from("<I", raw, 16)[0],
}
def inode_data(self, inode: dict[str, int], logical: int, length: int) -> bytes:
if logical > inode["size"] or length > inode["size"] - logical:
raise Reject("EINTEGRITY", "inode.data-bounds")
if inode["layout"] == 2:
physical = inode["offset"] + inode["inode_size"] + inode["xattr_size"] + logical
elif inode["layout"] == 0:
physical = (inode["start_block"] << self.block_bits) + logical
else:
raise Reject("EINTEGRITY", "inode.unsupported-layout")
return self.read(physical, length, "inode.data")
def directory_entries(self, inode: dict[str, int]) -> list[tuple[bytes, int]]:
result = []
logical = 0
while logical < inode["size"]:
length = min(self.block_size, inode["size"] - logical)
data = self.inode_data(inode, logical, length)
if len(data) < 12:
raise Reject("EINTEGRITY", "directory.header")
first_name = struct.unpack_from("<H", data, 8)[0]
if first_name == 0 or first_name % 12 or first_name > len(data):
raise Reject("EINTEGRITY", "directory.name-offset")
count = first_name // 12
for index in range(count):
entry = index * 12
nid = struct.unpack_from("<Q", data, entry)[0]
start = struct.unpack_from("<H", data, entry + 8)[0]
end = struct.unpack_from("<H", data, entry + 20)[0] if index + 1 < count else len(data)
if start > end or end > len(data):
raise Reject("EINTEGRITY", "directory.name-bounds")
result.append((data[start:end].split(b"\0", 1)[0], nid))
logical += length
return result
def resolve(self, path: str) -> dict[str, int]:
inode = self.inode(self.root_nid)
for component in path.strip("/").encode().split(b"/"):
candidates = [nid for name, nid in self.directory_entries(inode) if name == component]
if len(candidates) != 1:
raise Reject("ENOATTR", "path.lookup")
inode = self.inode(candidates[0])
return inode
def body_offset(self, inode: dict[str, int]) -> int:
return inode["offset"] + inode["inode_size"]
def body_header(self, inode: dict[str, int]) -> tuple[int, int, list[int]]:
if inode["xattr_size"] < 12:
raise Reject("EINTEGRITY", "ibody.header")
raw = self.read(self.body_offset(inode), 12, "ibody.header")
name_filter = struct.unpack_from("<I", raw)[0]
shared_count = raw[4]
header_size = 12 + shared_count * 4
if header_size > inode["xattr_size"]:
raise Reject("EINTEGRITY", "ibody.shared-count")
if inode["xattr_size"] == 12:
raise Reject("EOPNOTSUPP", "ibody.header-only")
shared = []
if shared_count:
ids = self.read(self.body_offset(inode) + 12, shared_count * 4, "ibody.shared-ids")
shared = list(struct.unpack(f"<{shared_count}I", ids))
return name_filter, header_size, shared
def entry(self, offset: int, limit: int, kind: str) -> tuple[int, bytes, bytes, int]:
header = self.read(offset, 4, f"{kind}.header")
name_length, name_index, value_length = struct.unpack("<BBH", header)
total = (4 + name_length + value_length + 3) & ~3
if total > limit - offset:
raise Reject("EINTEGRITY", f"{kind}.bounds")
raw = self.read(offset, total, f"{kind}.entry")
name = raw[4:4 + name_length]
if b"\0" in name:
raise Reject("EINTEGRITY", f"{kind}.name-nul")
return name_index, name, raw[4 + name_length:4 + name_length + value_length], total
def lookup(self, inode: dict[str, int], name: bytes, candidate: bool) -> tuple[str, bytes | None, dict[str, Any]]:
self.reset_reads()
if candidate and self.feature_compat & SPEC["format"]["feature_compat"] and self.filter_reserved == 0:
name_filter, _, _ = self.body_header(inode)
bit = oracle_xxh32(name, SPEC["format"]["seed"] + 1) & (SPEC["format"]["bits"] - 1)
if name_filter & (1 << bit):
return "ENOATTR", None, self.read_stats(False, bit)
body = self.read(self.body_offset(inode), inode["xattr_size"], "ibody")
name_filter = struct.unpack_from("<I", body)[0]
shared_count = body[4]
header_size = 12 + shared_count * 4
if header_size > len(body):
raise Reject("EINTEGRITY", "ibody.shared-count")
cursor = header_size
while cursor < len(body):
name_index = body[cursor + 1] if cursor + 1 < len(body) else 0
name_length = body[cursor] if cursor < len(body) else 0
value_length = struct.unpack_from("<H", body, cursor + 2)[0] if cursor + 4 <= len(body) else 0
total = (4 + name_length + value_length + 3) & ~3
if total > len(body) - cursor:
raise Reject("EINTEGRITY", "inline.bounds")
actual = body[cursor + 4:cursor + 4 + name_length]
if b"\0" in actual:
raise Reject("EINTEGRITY", "inline.name-nul")
if name_index == 1 and actual == name:
value = body[cursor + 4 + name_length:cursor + 4 + name_length + value_length]
return "PASS", value, self.read_stats(True, None)
cursor += total
for index in range(shared_count):
shared_id = struct.unpack_from("<I", body, 12 + index * 4)[0]
offset = (self.xattr_blkaddr << self.block_bits) + shared_id * 4
name_index, actual, value, _ = self.entry(offset, self.limit, "shared")
if name_index == 1 and actual == name:
return "PASS", value, self.read_stats(True, None)
return "ENOATTR", None, self.read_stats(True, None)
def read_stats(self, scanned: bool, bit: int | None) -> dict[str, Any]:
return {
"bytes": self.bytes,
"calls": self.calls,
"filter_bit": bit,
"provider_blocks": len(self.blocks),
"scanned": scanned,
}
def mutate_image(source: Path, output: Path, mutation: str, target: dict[str, int]) -> None:
data = bytearray(source.read_bytes())
block_bits = data[SUPER + 12]
body = target["offset"] + target["inode_size"]
if mutation == "unknown-filter":
data[SUPER + 104] = 1
elif mutation == "feature-off":
compat = struct.unpack_from("<I", data, SUPER + 8)[0]
struct.pack_into("<I", data, SUPER + 8, compat & ~SPEC["format"]["feature_compat"])
elif mutation == "corrupt-shared-count":
data[body + 4] = 255
elif mutation == "corrupt-shared-id":
if data[body + 4] == 0:
raise GateStop("fixture has no shared xattr ID to corrupt")
struct.pack_into("<I", data, body + 12, 0xFFFFFFFF)
else:
raise GateStop(f"unknown fixture mutation: {mutation}")
compat = struct.unpack_from("<I", data, SUPER + 8)[0]
if compat & 1:
struct.pack_into("<I", data, SUPER + 4, 0)
end = 1 << block_bits
struct.pack_into("<I", data, SUPER + 4, crc32c(data[SUPER:end]))
output.write_bytes(data)
def lookup_case(path: Path, name: bytes, candidate: bool) -> dict[str, Any]:
try:
with Image(path) as image:
inode = image.resolve(SPEC["fixture"]["target"])
status, value, reads = image.lookup(inode, name, candidate)
return {"errno": 0 if status == "PASS" else SPEC["errno"][status], "status": status, "value_hex": None if value is None else value.hex(), "reads": reads}
except Reject as error:
return {"errno": SPEC["errno"].get(error.errno_name, -1), "status": error.errno_name, "point": error.point, "value_hex": None}
def main(work: Path) -> dict[str, Any]:
identity = verify_identity()
write_json(OUTPUT / "identity.json", identity)
candidate_source = work / "candidate.c"
candidate_binary = work / "candidate"
candidate_source.write_text(CANDIDATE_SOURCE, encoding="ascii")
run([SPEC["tools"]["cc"]["path"], "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", str(candidate_source), "-o", str(candidate_binary)])
write_json(OUTPUT / "prototype.json", {
"binary_sha256": sha256_path(candidate_binary),
"linked_into_dut_kld": False,
"source_sha256": sha256_path(candidate_source),
"status": "PASS",
"symbol": SPEC["implementation"]["symbol"],
})
library = ctypes.CDLL(SPEC["tools"]["libxxhash"]["path"])
library.XXH32.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32]
library.XXH32.restype = ctypes.c_uint32
vector_records = [(item["index"], bytes.fromhex(item["name_hex"])) for item in SPEC["vectors"]]
vector_candidate = candidate_hashes(candidate_binary, vector_records)
vector_report = []
for item, (_, name), candidate_value in zip(SPEC["vectors"], vector_records, vector_candidate):
seed = (SPEC["format"]["seed"] + item["index"]) & 0xFFFFFFFF
python_value = oracle_xxh32(name, seed)
buffer = ctypes.create_string_buffer(name if name else b"\0")
library_value = int(library.XXH32(buffer, len(name), seed))
if candidate_value != item["hash"] or python_value != item["hash"] or library_value != item["hash"] or candidate_value & 31 != item["bit"]:
raise GateStop("Linux seed/endianness vector mismatch")
vector_report.append({**item, "candidate": candidate_value, "libxxhash": library_value, "python": python_value})
write_json(OUTPUT / "vectors.json", {"status": "PASS", "vectors": vector_report})
rng = random.Random(SPEC["random"]["seed"])
random_records = []
expected = array("I")
filters = array("I", [SPEC["format"]["filter_default"]] * 1024)
indexes = (1, 2, 3, 4, 6)
for number in range(SPEC["random"]["count"]):
name = rng.randbytes(16).hex().encode("ascii")
index = indexes[number % len(indexes)]
value = oracle_xxh32(name, (SPEC["format"]["seed"] + index) & 0xFFFFFFFF)
expected.append(value)
filters[number % len(filters)] &= ~(1 << (value & 31))
random_records.append((index, name))
actual = candidate_hashes(candidate_binary, random_records)
if len(actual) != len(expected) or any(left != right for left, right in zip(actual, expected)):
raise GateStop("one-million-name candidate/oracle mismatch")
for number, value in enumerate(actual):
if filters[number % len(filters)] & (1 << (value & 31)):
raise GateStop("one-million-name valid filter produced a false negative")
write_json(OUTPUT / "million.json", {"count": len(actual), "filter_count": len(filters), "seed": SPEC["random"]["seed"], "status": "PASS"})
del random_records, actual, expected, filters
source = work / "source"
create_source(source)
fixtures = OUTPUT / "fixtures"
fixtures.mkdir()
first = fixtures / "valid.erofs"
repeat = work / "repeat.erofs"
first_command = build_fixture(source, first)
repeat_command = build_fixture(source, repeat)
if sha256_path(first) != sha256_path(repeat):
raise GateStop("mkfs EROFS Bloom fixture is not byte reproducible")
run([SPEC["tools"]["fsck.erofs"]["path"], str(first)])
with Image(first) as image:
target = image.resolve(SPEC["fixture"]["target"])
image.reset_reads()
name_filter, _, shared = image.body_header(target)
status, _, _ = image.lookup(target, b"attr00", False)
if status != "PASS" or not shared:
raise GateStop("real EROFS fixture lacks required shared xattrs")
actual_names = [f"attr{index:02d}".encode() for index in range(SPEC["fixture"]["attribute_count"])]
for name in actual_names:
bit = oracle_xxh32(name, SPEC["format"]["seed"] + 1) & 31
if name_filter & (1 << bit):
raise GateStop("mkfs filter contains a false negative")
miss = collision = None
for number in range(100000):
candidate = f"absent-{number:05d}".encode()
bit = oracle_xxh32(candidate, SPEC["format"]["seed"] + 1) & 31
if name_filter & (1 << bit) and miss is None:
miss = candidate
if not name_filter & (1 << bit) and collision is None:
collision = candidate
if miss is not None and collision is not None:
break
if miss is None or collision is None:
raise GateStop("real fixture cannot provide both miss and collision names")
mutated = {}
for mutation in ("unknown-filter", "feature-off", "corrupt-shared-count", "corrupt-shared-id"):
path = fixtures / f"{mutation}.erofs"
mutate_image(first, path, mutation, target)
mutated[mutation] = path
for mutation in ("unknown-filter", "feature-off"):
run([SPEC["tools"]["fsck.erofs"]["path"], str(mutated[mutation])])
cases = [
("hit", first, b"attr00", "PASS", True),
("miss", first, miss, "ENOATTR", False),
("false-positive", first, collision, "ENOATTR", True),
("unknown-filter", mutated["unknown-filter"], miss, "ENOATTR", True),
("feature-off", mutated["feature-off"], miss, "ENOATTR", True),
("corrupt-shared-count", mutated["corrupt-shared-count"], miss, "EINTEGRITY", None),
("corrupt-shared-id", mutated["corrupt-shared-id"], b"attr00", "EINTEGRITY", None),
]
case_report = []
for identifier, path, name, expected_status, expected_scan in cases:
baseline = lookup_case(path, name, False)
candidate = lookup_case(path, name, True)
if candidate["status"] != expected_status or candidate["errno"] < 0:
raise GateStop(f"{identifier} candidate status/positive errno mismatch: {candidate}")
if identifier not in ("corrupt-shared-count",) and baseline["status"] != expected_status:
raise GateStop(f"{identifier} baseline oracle mismatch: {baseline}")
if expected_scan is not None and candidate["reads"]["scanned"] != expected_scan:
raise GateStop(f"{identifier} scan/fallback mismatch")
case_report.append({"id": identifier, "name": name.decode(), "baseline": baseline, "candidate": candidate})
write_json(OUTPUT / "cases.json", {"cases": case_report, "status": "PASS"})
samples = SPEC["thresholds"]["cold_samples"]
loops = SPEC["thresholds"]["cold_loops_per_sample"]
baseline_calls = []
candidate_calls = []
baseline_blocks = []
candidate_blocks = []
for _ in range(samples):
totals = {"baseline_calls": 0, "candidate_calls": 0, "baseline_blocks": 0, "candidate_blocks": 0}
for _ in range(loops):
baseline = lookup_case(first, miss, False)["reads"]
candidate = lookup_case(first, miss, True)["reads"]
totals["baseline_calls"] += baseline["calls"]
totals["candidate_calls"] += candidate["calls"]
totals["baseline_blocks"] += baseline["provider_blocks"]
totals["candidate_blocks"] += candidate["provider_blocks"]
baseline_calls.append(totals["baseline_calls"])
candidate_calls.append(totals["candidate_calls"])
baseline_blocks.append(totals["baseline_blocks"])
candidate_blocks.append(totals["candidate_blocks"])
call_reduction = 100.0 * (statistics.median(baseline_calls) - statistics.median(candidate_calls)) / statistics.median(baseline_calls)
block_reduction = 100.0 * (statistics.median(baseline_blocks) - statistics.median(candidate_blocks)) / statistics.median(baseline_blocks)
threshold = SPEC["thresholds"]["minimum_provider_metadata_read_reduction_percent"]
if call_reduction < threshold or block_reduction < threshold:
raise GateStop(f"cold metadata read reduction is below {threshold}: calls={call_reduction}, blocks={block_reduction}")
benchmark = {
"baseline_calls": baseline_calls,
"baseline_provider_blocks": baseline_blocks,
"call_reduction_percent": call_reduction,
"candidate_calls": candidate_calls,
"candidate_provider_blocks": candidate_blocks,
"provider_block_reduction_percent": block_reduction,
"samples": samples,
"status": "PASS",
"threshold_percent": threshold,
}
write_json(OUTPUT / "benchmark.json", benchmark)
expected_cases = {item[0]: item[3] for item in cases}
def worker(worker_id: int) -> int:
completed = 0
for iteration in range(SPEC["concurrency"]["loops_per_worker"]):
identifier, path, name, _, _ = cases[(worker_id + iteration) % len(cases)]
result = lookup_case(path, name, True)
if result["status"] != expected_cases[identifier] or result["errno"] < 0:
raise GateStop(f"concurrent {identifier} mismatch")
completed += 1
return completed
with ThreadPoolExecutor(max_workers=SPEC["concurrency"]["workers"]) as executor:
completed = sum(executor.map(worker, range(SPEC["concurrency"]["workers"])))
concurrency = {"completed": completed, **SPEC["concurrency"], "status": "PASS"}
write_json(OUTPUT / "concurrency.json", concurrency)
fixture_hashes = {path.name: sha256_path(path) for path in sorted(fixtures.iterdir())}
write_json(OUTPUT / "fixture.json", {
"command": first_command,
"feature_filter": True,
"filter": name_filter,
"fixture_sha256": fixture_hashes,
"miss": miss.decode(),
"collision": collision.decode(),
"shared_count": len(shared),
"status": "PASS",
"target": SPEC["fixture"]["target"],
})
write_json(OUTPUT / "commands.json", {
"candidate_compile": [SPEC["tools"]["cc"]["path"], "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "candidate.c", "-o", "candidate"],
"fixture_first": first_command,
"fixture_repeat": repeat_command,
"gate": ["timeout", "-k", "10", "240", "tests/pre15/gates/P15-021.sh", "--base", REQUESTED_BASE, "--output", str(OUTPUT)],
})
return {
"batch": "B19b",
"benchmark": benchmark,
"candidate": "P15-021",
"case_count": len(cases),
"concurrency": concurrency,
"decision": "GO",
"fixture_sha256": fixture_hashes,
"full_feature_suite": "NOT_RUN",
"qemu": "NOT_RUN",
"random_name_count": SPEC["random"]["count"],
"source_modified": False,
"status": "GO",
}
exit_code = 21
work = Path(tempfile.mkdtemp(prefix=".P15-021-work-", dir=OUTPUT))
try:
result = main(work)
exit_code = 0
except GateStop as error:
result = {
"batch": "B19b",
"candidate": "P15-021",
"decision": "STOP",
"full_feature_suite": "NOT_RUN",
"qemu": "NOT_RUN",
"reason": str(error),
"source_modified": False,
"status": "STOP",
}
exit_code = 1
except (InfraBlocked, OSError, KeyError, ValueError, json.JSONDecodeError, subprocess.TimeoutExpired) as error:
result = {
"batch": "B19b",
"candidate": "P15-021",
"decision": "INFRA_BLOCKED",
"full_feature_suite": "NOT_RUN",
"qemu": "INFRA_BLOCKED",
"reason": str(error),
"source_modified": False,
"status": "INFRA_BLOCKED",
}
exit_code = 21
finally:
shutil.rmtree(work, ignore_errors=False)
write_json(OUTPUT / "result.json", result)
cleanup = {
"owned_processes_remaining": 0,
"owned_temp_remaining": [],
"protected_base_image_touched": False,
"protected_pid_touched": False,
"protected_port_touched": False,
"retained_evidence": sorted(path.name for path in OUTPUT.iterdir()),
"source_modified": False,
"status": "PASS",
}
write_json(OUTPUT / "cleanup.json", cleanup)
lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT).as_posix()}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
print(json.dumps(result, sort_keys=True))
raise SystemExit(exit_code)
PY
tests/pre15/gates/P15-022-input.json
+95
View File
@@ -0,0 +1,95 @@
{
"b19a_assets": {
"tests/pre15/fixtures/B19a-cache-model.c": "101b7c407d080b9caa8d86c7ccbef5364ed332cf3fd543879576719bc26efddd",
"tests/pre15/fixtures/B19a-xattr-generate.py": "c0a39c0b6410074ed3727615bb26ea267010221af2441f5caa10b54b5c907d49",
"tests/pre15/fixtures/B19a-xattr-oracle.py": "cc39cea5b1ca1d503166f34c414f173e01e00b9e1117acf06925dd2b707aba05",
"tests/pre15/fixtures/B19a-xattr-spec.json": "9cc4c306432bb2b6d42faaec6b1ca595f725e7d9d0cc52c913d77f951ae2512c"
},
"b17_assets": {
"tests/pre15/fixtures/B17-xattr-generate.py": "a6eacf435912654cc6dd39ce1f27016d2ea5cd80ad7d4c9fbcfc1a973fddf32e",
"tests/pre15/fixtures/B17-xattr-oracle.py": "b2baec99caa1f710a5ffa8503c152ca18aaf049d98cdf4cdce0e1c36a4358f43",
"tests/pre15/fixtures/B17-xattr-seed.erofs": "8137e7966a9dc5fba85dc63750d065bd54f6d08dc25b13f1e8c4f0deea934041",
"tests/pre15/fixtures/B17-xattr-spec.json": "b6b9f187cb475e1bc6f0117eb0de64f184f6b6a3edd48bc0e0108c556fba39b7"
},
"candidate": "P15-022",
"expected_b17_fixture_set_sha256": "d821aeb36de37ae40b817b91f8169e585721c33a3a9e59098cdfbcbfff74e364",
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/amd64/include/atomic.h": "e66ef69410265aad51184b78425f03015e9ddc997bae5c42e8222f6aaa7df6bb",
"sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0",
"sys/kern/vfs_mount.c": "11b1c00bef1cb3a258ce469f62b36179ac391a130c5fed71a6d414e38994560c",
"sys/kern/vfs_subr.c": "f2a71321e567b3e5b4b5decbed130b7aa3de577ebd974e6bd2682cc856ed9a60",
"sys/sys/atomic_common.h": "3917e23c60c362ebb445731ce1020ee127aac437bf48cc29af722371953ae7a5",
"sys/sys/condvar.h": "b1eda9ce615ec9ca8ed5064e977aa7c26b6da8fd3e7d5e4df523adfdf0dc13f2",
"sys/sys/malloc.h": "b65023be3d636e7e84570bd09b1fbf8ac55ca7538cd815e18f868a3ae871aef3",
"sys/sys/mount.h": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738",
"sys/sys/mutex.h": "8136fe1626a04812ecbcef73d8b8cf677cc5220a115db13e83018febb78179ee",
"sys/sys/vnode.h": "244d7c51cb75d6c19d07a8b9dab85183f87d05c221bbd760af05839ceb2e4a0d"
}
},
"gates": [
"G03",
"G05"
],
"linux_source_sha256": {
"src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405",
"src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9",
"src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a"
},
"protected": {
"base_image": "/work/build/vm-freebsd-build.qcow2.bp",
"pid": 26318,
"port": 9222
},
"required_base": "50a4e84d0da33592a81361e0294b7feb5bbd3ffa",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64",
"repo-pre-15/src/inode.c": "811ffa023d0386d46be3b1fc771cf9ca9c088f8d74b8aa8c47f390f7d9172c52",
"repo-pre-15/src/internal.h": "f6b4a8e84352c8d4e005a9aabdf261e4aa8716ec7d9e12eced0b972501609408",
"repo-pre-15/src/xattr.c": "7ce78b7af8838715fed10843f412458bdacb8c9b6811f08f660549f18a0743b0"
},
"tools": {
"cc": {
"path": "/usr/bin/cc",
"sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f"
},
"dump.erofs": {
"path": "/usr/bin/dump.erofs",
"sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c",
"version": "mkfs.erofs (erofs-utils) 1.8.6"
},
"python3": {
"path": "/usr/bin/python3",
"sha256": "4703a3d15898c0b5d81c3f939e93bdd8ca6116342093fb160ab1e01860dd7d8b"
}
},
"write_set": {
"gate": [
"repo-pre-15/docs/pre15-stage0/P15-022.md",
"repo-pre-15/tests/pre15/fixtures/B19a-cache-model.c",
"repo-pre-15/tests/pre15/fixtures/B19a-xattr-generate.py",
"repo-pre-15/tests/pre15/fixtures/B19a-xattr-oracle.py",
"repo-pre-15/tests/pre15/fixtures/B19a-xattr-spec.json",
"repo-pre-15/tests/pre15/gates/P15-022-input.json",
"repo-pre-15/tests/pre15/gates/P15-022.sh"
],
"source": [
"repo-pre-15/src/erofs_vnops.c",
"repo-pre-15/src/inode.c",
"repo-pre-15/src/internal.h",
"repo-pre-15/src/xattr.c",
"repo-pre-15/tests/pre15/cases/B19a-xattr-cache.sh",
"repo-pre-15/tests/pre15/fixtures/B19a-*"
]
}
}
tests/pre15/gates/P15-022.sh
Executable
+464
View File
@@ -0,0 +1,464 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
dut=$root/repo-pre-15
fixture_dir=$dut/tests/pre15/fixtures
input=$gate_dir/P15-022-input.json
freebsd_src=${FREEBSD_SRC:-/work/dev-freebsd-releng}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
work=$output/.work
mkdir "$work"
commands=$output/COMMANDS.txt
: >"$commands"
result_written=0
status=RUNNER_FAIL
reason='gate did not complete'
write_result()
{
python3 -B - "$output/result.json" "$status" "$reason" "$base" <<'PY'
import json
from pathlib import Path
import sys
path = Path(sys.argv[1])
status = sys.argv[2]
result = {
"b19a": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE",
"candidate": "P15-022",
"decision": status,
"full_feature_suite": "NOT_RUN",
"gates": ["G03", "G05"],
"qemu": "NOT_RUN",
"reason": sys.argv[3],
"requested_base": sys.argv[4],
"schema": 1,
"source_modified": False,
"status": status,
}
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
result_written=1
}
finish()
{
rc=$?
trap - EXIT HUP INT TERM
if test -d "$work"; then
find "$work" -depth -delete
fi
if test "$result_written" -eq 0; then
write_result
fi
python3 -B - "$output/cleanup.json" "$status" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(json.dumps({
"owned_processes_remaining": 0,
"owned_temp_remaining": [],
"protected_base_image_touched": False,
"protected_pid_touched": False,
"protected_port_touched": False,
"source_modified": False,
"status": "PASS",
"verdict": sys.argv[2],
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
find "$output" -type f ! -name SHA256SUMS -print0 | sort -z | \
xargs -0 sha256sum >"$output/SHA256SUMS"
exit "$rc"
}
trap finish EXIT HUP INT TERM
record_command()
{
printf '%s' "$1" >>"$commands"
shift
for argument in "$@"; do
printf ' %s' "$(printf '%s' "$argument" | sed "s/'/'\\\\''/g; s/^/'/; s/$/'/")" >>"$commands"
done
printf '\n' >>"$commands"
}
run_step()
{
label=$1
seconds=$2
shift 2
record_command "timeout -k 5 $seconds" "$@"
set +e
timeout -k 5 "$seconds" "$@" >"$output/$label.stdout" \
2>"$output/$label.stderr"
rc=$?
set -e
printf '%s\n' "$rc" >"$output/$label.exit"
if test "$rc" -ne 0; then
status=RUNNER_FAIL
reason="$label failed or timed out with exit $rc"
write_result
exit 20
fi
}
for tool in cc cmp fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
status=INFRA_BLOCKED
reason="missing required host tool: $tool"
write_result
exit 21
}
done
test -d "$freebsd_src/sys" || {
status=INFRA_BLOCKED
reason="missing exact-ABI FreeBSD source: $freebsd_src"
write_result
exit 21
}
run_step identity 30 python3 -B - "$root" "$dut" "$input" "$base" \
"$output/identity.json" "$freebsd_src" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import subprocess
import sys
root = Path(sys.argv[1])
dut = Path(sys.argv[2])
spec = json.loads(Path(sys.argv[3]).read_text(encoding="ascii"))
requested = sys.argv[4]
output = Path(sys.argv[5])
freebsd = Path(sys.argv[6])
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
value.update(block)
return value.hexdigest()
def git(*args: str, cwd: Path = root) -> str:
completed = subprocess.run(
["git", "-C", str(cwd), *args], check=False,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
)
if completed.returncode != 0:
raise SystemExit(completed.stderr.strip())
return completed.stdout.strip()
resolved = git("rev-parse", f"{requested}^{{commit}}")
if resolved != spec["required_base"]:
raise SystemExit(f"required base {spec['required_base']}, got {resolved}")
source_hashes = {}
for relative, expected in {**spec["source_sha256"], **spec["linux_source_sha256"]}.items():
data = subprocess.run(
["git", "-C", str(root), "show", f"{resolved}:{relative}"],
check=True, stdout=subprocess.PIPE,
).stdout
actual = hashlib.sha256(data).hexdigest()
if actual != expected:
raise SystemExit(f"frozen source changed: {relative}")
source_hashes[relative] = actual
asset_hashes = {}
for group in ("b17_assets", "b19a_assets"):
for relative, expected in spec[group].items():
actual = digest(dut / relative)
if actual != expected:
raise SystemExit(f"gate asset changed: {relative}")
asset_hashes[relative] = actual
if git("rev-parse", "HEAD", cwd=freebsd) != spec["freebsd"]["head"]:
raise SystemExit("FreeBSD source HEAD changed")
freebsd_hashes = {}
for relative, expected in spec["freebsd"]["sha256"].items():
actual = digest(freebsd / relative)
if actual != expected:
raise SystemExit(f"FreeBSD source changed: {relative}")
freebsd_hashes[relative] = actual
tool_hashes = {}
for name, item in spec["tools"].items():
actual = digest(Path(item["path"]))
if actual != item["sha256"]:
raise SystemExit(f"host tool changed: {name}")
tool_hashes[name] = actual
version = subprocess.run(
[spec["tools"]["mkfs.erofs"]["path"], "-V"], check=True,
stdout=subprocess.PIPE, text=True,
).stdout.splitlines()[0]
if version != spec["tools"]["mkfs.erofs"]["version"]:
raise SystemExit("mkfs.erofs version changed")
output.write_text(json.dumps({
"assets": asset_hashes,
"base": resolved,
"freebsd_head": spec["freebsd"]["head"],
"freebsd_sha256": freebsd_hashes,
"source_sha256": source_hashes,
"status": "PASS",
"tool_sha256": tool_hashes,
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
spec=$fixture_dir/B19a-xattr-spec.json
generator=$fixture_dir/B19a-xattr-generate.py
oracle=$fixture_dir/B19a-xattr-oracle.py
model=$fixture_dir/B19a-cache-model.c
fixtures=$output/fixtures
repeat_fixtures=$work/fixtures-repeat
run_step generate-first 240 python3 -B "$generator" --spec "$spec" \
--output "$fixtures" --work "$work/generate-first"
run_step generate-repeat 240 python3 -B "$generator" --spec "$spec" \
--output "$repeat_fixtures" --work "$work/generate-repeat"
run_step fixture-reproducibility 30 python3 -B - "$fixtures" \
"$repeat_fixtures" "$output/fixture-reproducibility.json" <<'PY'
import hashlib
import json
from pathlib import Path
import sys
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
first = Path(sys.argv[1])
second = Path(sys.argv[2])
names = sorted(path.name for path in first.glob("*.erofs"))
if names != sorted(path.name for path in second.glob("*.erofs")):
raise SystemExit("fixture file sets differ")
hashes = {name: digest(first / name) for name in names}
repeat = {name: digest(second / name) for name in names}
if hashes != repeat:
raise SystemExit("B19a fixtures are not byte reproducible")
Path(sys.argv[3]).write_text(json.dumps({
"first": hashes,
"repeat": repeat,
"status": "PASS",
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
run_step oracle-selftest 30 python3 -B "$oracle" --spec "$spec" selftest \
--output "$output/oracle-selftest.json"
run_step oracle-first 240 python3 -B "$oracle" --spec "$spec" correctness \
--fixtures "$fixtures" --output "$output/G03-B19a-first.json"
run_step oracle-repeat 240 python3 -B "$oracle" --spec "$spec" correctness \
--fixtures "$repeat_fixtures" --output "$output/G03-B19a-repeat.json"
mkdir "$output/G03-B17"
b17_spec=$fixture_dir/B17-xattr-spec.json
b17_seed=$fixture_dir/B17-xattr-seed.erofs
b17_generator=$fixture_dir/B17-xattr-generate.py
b17_oracle=$fixture_dir/B17-xattr-oracle.py
run_step b17-generate-first 240 python3 -B "$b17_generator" generate \
--spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-first"
run_step b17-generate-repeat 240 python3 -B "$b17_generator" generate \
--spec "$b17_spec" --seed "$b17_seed" --output "$work/b17-repeat"
run_step b17-oracle-first 240 python3 -B "$b17_oracle" --spec "$b17_spec" \
--fixtures "$work/b17-first" --report "$output/G03-B17/oracle-first.json"
run_step b17-oracle-repeat 240 python3 -B "$b17_oracle" --spec "$b17_spec" \
--fixtures "$work/b17-repeat" --report "$output/G03-B17/oracle-repeat.json"
cp "$work/b17-first/fixture-index.json" "$output/G03-B17/fixture-index.json"
run_step b17-summary 30 python3 -B - "$input" \
"$output/G03-B17/fixture-index.json" "$output/G03-B17/oracle-first.json" \
"$output/G03-B17/oracle-repeat.json" "$output/G03-B17/summary.json" <<'PY'
import json
from pathlib import Path
import sys
gate = json.loads(Path(sys.argv[1]).read_text(encoding="ascii"))
index = json.loads(Path(sys.argv[2]).read_text(encoding="ascii"))
reports = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[3:5]]
if index["fixture_count"] != 25 or index["legal_count"] != 10 or index["damaged_count"] != 15:
raise SystemExit("B17 fixture cardinality changed")
if index["fixture_set_sha256"] != gate["expected_b17_fixture_set_sha256"]:
raise SystemExit("B17 fixture set hash changed")
for report in reports:
if report["status"] != "PASS" or report["legal_passed"] != 10 or report["damaged_passed"] != 15:
raise SystemExit("B17 oracle replay is incomplete")
Path(sys.argv[5]).write_text(json.dumps({
"damaged_passed": 15,
"fixture_count": 25,
"fixture_set_sha256": index["fixture_set_sha256"],
"legal_passed": 10,
"replays": 2,
"status": "PASS",
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
run_step cache-model-compile 30 cc -std=c11 -O2 -Wall -Wextra -Werror \
-pthread "$model" -o "$work/B19a-cache-model"
run_step cache-model 120 "$work/B19a-cache-model"
cp "$output/cache-model.stdout" "$output/G05-cache-model.json"
mkdir "$output/G05-samples"
printf 'sample\tfirst\tsecond\n' >"$output/G05-sample-order.tsv"
sample=1
while test "$sample" -le 5; do
if test $((sample % 2)) -eq 1; then
first=baseline
second=candidate
else
first=candidate
second=baseline
fi
printf '%s\t%s\t%s\n' "$sample" "$first" "$second" \
>>"$output/G05-sample-order.tsv"
for variant in "$first" "$second"; do
run_step "G05-sample-$sample-$variant" 30 python3 -B "$oracle" \
--spec "$spec" sample --image "$fixtures/valid.erofs" \
--variant "$variant" --sample "$sample" \
--output "$output/G05-samples/$variant-$sample.json"
done
sample=$((sample + 1))
done
set +e
record_command "timeout -k 5 30" python3 -B - "$spec" "$output/G05-samples" \
"$output/G05-benchmark.json" "$output/G05-raw-samples.tsv"
timeout -k 5 30 python3 -B - "$spec" "$output/G05-samples" \
"$output/G05-benchmark.json" "$output/G05-raw-samples.tsv" \
>"$output/G05-aggregate.stdout" 2>"$output/G05-aggregate.stderr" <<'PY'
import json
from pathlib import Path
import statistics
import sys
spec = json.loads(Path(sys.argv[1]).read_text(encoding="ascii"))
sample_dir = Path(sys.argv[2])
groups = {"baseline": [], "candidate": []}
for variant in groups:
for sample in range(1, spec["benchmark"]["samples"] + 1):
path = sample_dir / f"{variant}-{sample}.json"
value = json.loads(path.read_text(encoding="ascii"))
if value["status"] != "PASS" or value["variant"] != variant or value["sample"] != sample:
raise SystemExit(f"invalid raw sample: {path}")
groups[variant].append(value)
identity_fields = ("fixture_sha256", "host", "loops", "operations", "warmup_loops")
reference = groups["baseline"][0]
for values in groups.values():
for value in values:
if any(value[field] != reference[field] for field in identity_fields):
raise SystemExit("baseline/candidate sample conditions differ")
def series(variant: str, field: str) -> list[int]:
return [int(value["reads"][field]) for value in groups[variant]]
baseline_calls = series("baseline", "calls")
candidate_calls = series("candidate", "calls")
baseline_blocks = series("baseline", "provider_block_reads")
candidate_blocks = series("candidate", "provider_block_reads")
call_reduction = 100.0 * (
statistics.median(baseline_calls) - statistics.median(candidate_calls)
) / statistics.median(baseline_calls)
block_reduction = 100.0 * (
statistics.median(baseline_blocks) - statistics.median(candidate_blocks)
) / statistics.median(baseline_blocks)
threshold = spec["thresholds"]["minimum_provider_metadata_read_reduction_percent"]
report = {
"baseline_calls": baseline_calls,
"baseline_provider_block_reads": baseline_blocks,
"call_reduction_percent": call_reduction,
"candidate_calls": candidate_calls,
"candidate_provider_block_reads": candidate_blocks,
"conditions": {field: reference[field] for field in identity_fields},
"failed_samples_filtered": False,
"provider_block_reduction_percent": block_reduction,
"sample_count_per_variant": len(groups["baseline"]),
"status": "PASS" if call_reduction >= threshold and block_reduction >= threshold else "STOP",
"threshold_percent": threshold,
}
Path(sys.argv[3]).write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="ascii")
lines = ["variant\tsample\tcalls\tprovider_block_reads\tbytes\telapsed_ns\tstatus"]
for variant in ("baseline", "candidate"):
for value in groups[variant]:
lines.append("\t".join(map(str, (
variant, value["sample"], value["reads"]["calls"],
value["reads"]["provider_block_reads"], value["reads"]["bytes"],
value["elapsed_ns"], value["status"],
))))
Path(sys.argv[4]).write_text("\n".join(lines) + "\n", encoding="ascii")
print(json.dumps(report, sort_keys=True))
raise SystemExit(0 if report["status"] == "PASS" else 22)
PY
aggregate_rc=$?
set -e
printf '%s\n' "$aggregate_rc" >"$output/G05-aggregate.exit"
if test "$aggregate_rc" -ne 0; then
if test "$aggregate_rc" -eq 22; then
status=STOP
reason='valid G05 samples did not meet the unchanged 25 percent threshold'
write_result
exit 22
fi
status=RUNNER_FAIL
reason="G05 aggregate failed or timed out with exit $aggregate_rc"
write_result
exit 20
fi
run_step gate-summary 30 python3 -B - "$output/G03-B19a-first.json" \
"$output/G03-B19a-repeat.json" "$output/G03-B17/summary.json" \
"$output/G05-cache-model.json" "$output/G05-benchmark.json" \
"$output/GATE-SUMMARY.json" <<'PY'
import json
from pathlib import Path
import sys
values = [json.loads(Path(path).read_text(encoding="ascii")) for path in sys.argv[1:6]]
if any(value["status"] != "PASS" for value in values):
raise SystemExit("one or more G03/G05 components did not pass")
Path(sys.argv[6]).write_text(json.dumps({
"b19a": "AUTHORIZED",
"g03": "GO",
"g05": "GO",
"source_modified": False,
"status": "GO",
}, indent=2, sort_keys=True) + "\n", encoding="ascii")
PY
status=GO
reason='G03 and G05 both reached GO with valid multi-block oracle and symmetric samples'
write_result
printf '%s\n' 'P15-022 G03/G05 GO; B19a source is authorized'
exit 0
tests/pre15/gates/P15-027-input.json
+180
View File
@@ -0,0 +1,180 @@
{
"architectures": [
{
"compiler_backend": "aarch64",
"emulator_tool": "qemu-system-aarch64",
"expected_uname_m": "arm64",
"expected_uname_p": "aarch64",
"header_aliases": {},
"id": "arm64",
"kernel_cflags": [
"-mgeneral-regs-only",
"-ffixed-x18",
"-mbranch-protection=standard",
"-mno-outline-atomics"
],
"machine_arch": "aarch64",
"machine_header_dir": "arm64",
"native_environment": null,
"param_h_sha256": "234e3066dbc2f1cde6d6f41a73c3fcac7b6e2af632208e950a721b02ca77ebbb",
"pointer_bits": 64,
"target_triple": "aarch64-unknown-freebsd15.0"
},
{
"compiler_backend": "riscv64",
"emulator_tool": "qemu-system-riscv64",
"expected_uname_m": "riscv",
"expected_uname_p": "riscv64",
"header_aliases": {},
"id": "riscv64",
"kernel_cflags": [
"-march=rv64imafdch_zifencei",
"-mabi=lp64",
"-mcmodel=medium",
"-mno-relax"
],
"machine_arch": "riscv64",
"machine_header_dir": "riscv",
"native_environment": null,
"param_h_sha256": "b6596e77b0169cd6fe1a360c6160bead2f60f7145c52f42740ef7bc8691200ce",
"pointer_bits": 64,
"target_triple": "riscv64-unknown-freebsd15.0"
},
{
"compiler_backend": "x86",
"emulator_tool": "qemu-system-i386",
"expected_uname_m": "i386",
"expected_uname_p": "i386",
"header_aliases": {
"x86": "x86/include"
},
"id": "i386",
"kernel_cflags": [
"-mno-aes",
"-mno-avx",
"-mno-mmx",
"-mno-sse",
"-msoft-float"
],
"machine_arch": "i386",
"machine_header_dir": "i386",
"native_environment": null,
"param_h_sha256": "46a648ac4fd5ee4db0f104d61e4916caf8c95b465d44166faf58f1574b03cc5b",
"pointer_bits": 32,
"target_triple": "i386-unknown-freebsd15.0"
}
],
"batch": "B36",
"candidate": "P15-027",
"cross_build_contract": {
"configurations": [
0,
1
],
"module_sources": [
"super.c",
"inode.c",
"data.c",
"namei.c",
"dir.c",
"xattr.c",
"erofs_vnops.c",
"decompressor.c",
"zmap.c",
"zdata.c",
"decompressor_lz4.c",
"decompressor_lzma.c",
"decompressor_deflate.c",
"decompressor_zstd.c"
],
"required_outputs": [
"WITH_ZSTDIO=0 KMOD size and nm -u",
"WITH_ZSTDIO=1 KMOD size and nm -u",
"ondisk layout static assertions",
"unaligned le16dec/le32dec/le64dec target assembly",
"little-endian target assertion"
],
"unaligned_endian_probe_sha256": "b05fc54381177c3cbcd9bd8b2e9f56ba5a986b31fe8930eeb7855fa76620905c",
"unaligned_endian_probe_source": "#include <stdint.h>\n#include <sys/endian.h>\n\n#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__\n#error selected P15-027 targets must be little endian\n#endif\n\nuint64_t\np15_027_unaligned_ledec(const unsigned char *bytes)\n{\n return ((uint64_t)le16dec(bytes + 1) ^\n (uint64_t)le32dec(bytes + 1) ^ le64dec(bytes + 1));\n}\n"
},
"freebsd_source": {
"git_prefix": "build/freebsd-src/",
"git_tree_path": "freebsd-src",
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/arm64/include/param.h": "234e3066dbc2f1cde6d6f41a73c3fcac7b6e2af632208e950a721b02ca77ebbb",
"sys/conf/Makefile.arm64": "facbe05d9154ba68c39296f3cb9d2dd78da68e7c51d88e9008284a04c377cf50",
"sys/conf/Makefile.i386": "15c7164ef3641ffdfba5152d039cca7349859e37c782283e6d158582393df517",
"sys/conf/Makefile.riscv": "0f46b4c62d5c44c74d309cfe51786c48e3aadbcedb33c968e1a9e785c64a97c8",
"sys/conf/kern.mk": "04126b5fd5ab8d78f1bc778e59b89a5e5daf67f3dcf5ae460cc14d1958b830fe",
"sys/i386/include/param.h": "46a648ac4fd5ee4db0f104d61e4916caf8c95b465d44166faf58f1574b03cc5b",
"sys/kern/vnode_if.src": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b",
"sys/riscv/include/param.h": "b6596e77b0169cd6fe1a360c6160bead2f60f7145c52f42740ef7bc8691200ce",
"sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806",
"sys/tools/vnode_if.awk": "a834e22233718f13dc2a4a53177b1439581552145fd24be85c038e7ff2a24441"
},
"source_tree_oid": "eb10834ee8b13558a084d3c624f5691152760a11",
"sys_tree_oid": "c4e8dddaa56277dd6609369152081d6bed80caff"
},
"gate": "G07",
"native_runtime_contract": {
"required_identity": [
"FreeBSD uname -s",
"architecture-specific uname -m and uname -p",
"frozen EROFS BASE",
"FreeBSD source HEAD",
"temporary patch SHA-256",
"exact cross-built module SHA-256 values"
],
"required_operations": [
"zstdio0-kld-load",
"zstdio0-plain-read",
"zstdio0-lz4-read",
"zstdio0-lzma-read",
"zstdio0-zstd-read",
"zstdio0-kld-unload",
"zstdio1-kld-load",
"zstdio1-plain-read",
"zstdio1-lz4-read",
"zstdio1-lzma-read",
"zstdio1-zstd-read",
"zstdio1-kld-unload"
],
"zstd_rules": [
"running kernel options ZSTDIO capability item is PASS and true",
"WITH_ZSTDIO=0 Zstd read is PASS only for exact EOPNOTSUPP",
"WITH_ZSTDIO=1 Zstd read returns the expected content hash"
]
},
"prohibited_paths": [
"introduction.md"
],
"repo_source_tree_oid": "21bbe769bf75cd177116674fedafe12396817171",
"required_base": "4683579dabc57b0c9aaf5762e13ecc2a0ec3f8f9",
"schema": 1,
"source_sha256": {
"planning/pre15/20-final-candidate-ledger.md": "8333131b0e0951ef90bec00587a41f25841a156e5a0f59eaf2264c0004a48140",
"planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8",
"planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20",
"planning/pre15/50-feature-test-matrix.md": "ce39b44ef3989f4fcdbcd8eb677193ef53b630080c74d8539664fb2be3f38408",
"planning/pre15/60-smoke-build-matrix.md": "2f7b4861b004bad82669042959ecd43a6839d4d8b241337af5b85485b02d5c99",
"planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d",
"repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8",
"repo-pre-15/src/erofs_fs.h": "d70c5920c10164a6cc8ade3c32077bffa604d62aa6eb7932d098dcb3e99ea980",
"repo-pre-15/src/internal.h": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a",
"repo-pre-15/tests/pre15/probes/ondisk_layout.c": "8fa700b5076d5f427323a7a98ce1195f5117cd63f9bc6be6b052c2535be9241d"
},
"temporary_makefile_relaxation": {
"patched_makefile_sha256": "05159677a510b1348f8fd0f7a69db02bea00437631c54dc10337e480e39ec467",
"path": "repo-pre-15/src/Makefile",
"sha256": "30da7ce74a6d00a002c9dc9a388c36e206dd2be7d10f7f58d1ca13b79822c76f",
"unified_diff": "--- a/repo-pre-15/src/Makefile\n+++ b/repo-pre-15/src/Makefile\n@@ -1,9 +1,5 @@\n KMOD= erofs\n \n-.if ${MACHINE_ARCH} != \"amd64\"\n-.error erofs supports only MACHINE_ARCH=amd64\n-.endif\n-\n WITH_ZSTDIO?= 0\n .if empty(WITH_ZSTDIO:M0) && empty(WITH_ZSTDIO:M1)\n .error WITH_ZSTDIO must be 0 or 1\n"
},
"toolchain": {
"compiler": {
"realpath": "/usr/lib/llvm-19/bin/clang",
"sha256": "0a3c55936ac43954fbe4914e6e73b577ca9e35e75d89ef13d3e28c187f41ca1a",
"version": "Debian clang version 19.1.7 (3+b1)"
}
}
}
tests/pre15/gates/P15-027.sh
Executable
+845
View File
@@ -0,0 +1,845 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-027-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
deadline=120
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; }
base=$2
shift 2
;;
--freebsd-src)
test "$#" -ge 2 || { printf '%s\n' '--freebsd-src requires a directory' >&2; exit 20; }
freebsd_src=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 20
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; }
for tool in date git mktemp patch python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 21
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
case "$freebsd_src" in
/*) ;;
*) freebsd_src=$PWD/$freebsd_src ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; }
mkdir -p "$output"
work=$(mktemp -d "${TMPDIR:-/tmp}/P15-027.XXXXXX")
cleanup_trap()
{
rm -rf -- "$work"
}
trap cleanup_trap EXIT HUP INT TERM
python3 -B - "$output/command-argv.json" "$0" --base "$base" \
--freebsd-src "$freebsd_src" --output "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n",
encoding="ascii",
)
PY
python3 -B - "$output/ownership.json" "$work" "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(
{
"owned_temporary_paths": [sys.argv[2]],
"persistent_output": sys.argv[3],
"owned_processes": [],
"owned_ports": [],
"qemu": "NOT_RUN",
"protected_resources_addressed": False,
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
PY
set +e
timeout -k 5 "$deadline" python3 -B - \
"$root" "$input" "$base" "$freebsd_src" "$output" "$work" "$deadline" <<'PY' \
>"$output/stdout.log" 2>"$output/stderr.log"
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import shutil
import subprocess
import sys
import tarfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
FREEBSD_SRC = Path(sys.argv[4])
OUTPUT = Path(sys.argv[5])
WORK = Path(sys.argv[6])
DEADLINE = int(sys.argv[7])
SPEC = json.loads(INPUT.read_text(encoding="utf-8"))
class GateFailure(RuntimeError):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
return sha256_bytes(path.read_bytes())
def write_json(path: Path, value: Any) -> None:
path.write_text(
json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
def run(
argv: list[str],
*,
cwd: Path | None = None,
input_bytes: bytes | None = None,
allowed: set[int] | None = None,
timeout_seconds: int = 30,
) -> subprocess.CompletedProcess[bytes]:
try:
completed = subprocess.run(
argv,
cwd=cwd,
check=False,
input=input_bytes,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout_seconds,
)
except subprocess.TimeoutExpired as error:
raise GateFailure(
"INFRA_BLOCKED", f"command timed out: {' '.join(argv)}"
) from error
accepted = {0} if allowed is None else allowed
if completed.returncode not in accepted:
detail = completed.stderr.decode("utf-8", errors="replace").strip()
raise GateFailure(
"RUNNER_FAIL",
f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}",
)
return completed
def git_text(repository: Path, *args: str) -> str:
return run(["git", "-C", str(repository), *args]).stdout.decode(
"utf-8", errors="strict"
).strip()
def source_at(commit: str, relative: str) -> bytes:
return run(
["git", "-C", str(ROOT), "show", f"{commit}:{relative}"]
).stdout
def safe_relative_path(value: str) -> Path:
path = Path(value)
if path.is_absolute() or ".." in path.parts:
raise GateFailure("RUNNER_FAIL", f"unsafe declared path: {value}")
if path.name == "introduction.md":
raise GateFailure("RUNNER_FAIL", "prohibited path declaration")
return path
def validate_hash(value: Any, label: str) -> str:
if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None:
raise GateFailure("RUNNER_FAIL", f"invalid SHA-256 for {label}")
return value
def validate_runtime_item(item: Any, item_id: str) -> None:
if not isinstance(item, dict):
raise GateFailure("RUNNER_FAIL", f"runtime item is not an object: {item_id}")
expected = {
"status": "PASS",
"exit_code": 0,
"target_marker": "reached",
"cleanup": "PASS",
}
for key, value in expected.items():
if item.get(key) != value:
raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} has invalid {key}")
argv = item.get("command_argv")
if not isinstance(argv, list) or not argv or not all(isinstance(v, str) for v in argv):
raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks exact argv")
if not isinstance(item.get("deadline_seconds"), int) or item["deadline_seconds"] <= 0:
raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks deadline")
for stream in ("stdout", "stderr"):
relative = safe_relative_path(item.get(stream, ""))
evidence_path = OUTPUT / relative
if not evidence_path.is_file():
raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} lacks {stream}")
expected_hash = validate_hash(item.get(f"{stream}_sha256"), f"{item_id} {stream}")
if sha256_path(evidence_path) != expected_hash:
raise GateFailure("RUNNER_FAIL", f"runtime item {item_id} {stream} hash mismatch")
def validate_runtime_record(
path: Path,
architecture: dict[str, Any],
modules: dict[str, dict[str, Any]],
resolved_base: str,
patch_sha256: str,
) -> dict[str, Any]:
record = json.loads(path.read_text(encoding="utf-8"))
expected_identity = {
"schema": 1,
"architecture": architecture["id"],
"uname_s": "FreeBSD",
"uname_m": architecture["expected_uname_m"],
"uname_p": architecture["expected_uname_p"],
"dut_base": resolved_base,
"temporary_patch_sha256": patch_sha256,
"freebsd_source_head": SPEC["freebsd_source"]["head"],
}
for key, value in expected_identity.items():
if record.get(key) != value:
raise GateFailure("RUNNER_FAIL", f"native record identity mismatch: {key}")
if record.get("status") not in {"PASS", "DUT_FAIL"}:
raise GateFailure("RUNNER_FAIL", "native record status is not authoritative")
if record.get("cleanup") != "PASS" or record.get("target_marker") != "reached":
raise GateFailure("RUNNER_FAIL", "native record did not reach target with cleanup")
if record.get("module_sha256") != {
name: value["sha256"] for name, value in modules.items()
}:
raise GateFailure("RUNNER_FAIL", "native module hashes differ from cross artifacts")
fixtures = record.get("fixture_sha256")
if not isinstance(fixtures, list) or len(fixtures) < 4:
raise GateFailure("RUNNER_FAIL", "native record lacks four fixture hashes")
for index, value in enumerate(fixtures):
validate_hash(value, f"native fixture {index}")
capability = record.get("kernel_zstdio")
validate_runtime_item(capability, "kernel-zstdio-capability")
if capability.get("options_zstdio") is not True:
raise GateFailure("RUNNER_FAIL", "native ZSTDIO capability is not enabled")
operations = record.get("operations")
if not isinstance(operations, dict):
raise GateFailure("RUNNER_FAIL", "native operations are absent")
required = SPEC["native_runtime_contract"]["required_operations"]
if set(operations) != set(required):
raise GateFailure("RUNNER_FAIL", "native operation set is not exact")
for item_id in required:
validate_runtime_item(operations[item_id], item_id)
disabled = operations["zstdio0-zstd-read"]
if disabled.get("errno") != "EOPNOTSUPP":
raise GateFailure("RUNNER_FAIL", "disabled native Zstd errno is not EOPNOTSUPP")
return record
def extract_repo_source(resolved_base: str, destination: Path) -> None:
archive = run(
[
"git", "-C", str(ROOT), "archive", "--format=tar", resolved_base,
"repo-pre-15/src", "repo-pre-15/tests/pre15/probes/ondisk_layout.c",
],
timeout_seconds=60,
).stdout
destination.mkdir(parents=True)
archive_path = WORK / "repo-source.tar"
archive_path.write_bytes(archive)
with tarfile.open(archive_path, mode="r:") as tar:
tar.extractall(destination, filter="data")
def compile_architecture(
architecture: dict[str, Any],
resolved_base: str,
patch_bytes: bytes,
compiler: Path,
) -> dict[str, Any]:
arch_id = architecture["id"]
arch_root = WORK / "cross" / arch_id
source_root = arch_root / "source"
extract_repo_source(resolved_base, source_root)
run(["patch", "-p1"], cwd=source_root, input_bytes=patch_bytes)
src = source_root / "repo-pre-15/src"
sys_root = FREEBSD_SRC / "sys"
resource_include = Path(
run([str(compiler), "-print-resource-dir"]).stdout.decode("ascii").strip()
) / "include"
common_flags = [
"-O2", "-pipe", "-fno-common", "-fno-strict-aliasing", "-D_KERNEL",
"-DKLD_MODULE", "-nostdinc", "-ffreestanding", "-fwrapv",
"-fno-asynchronous-unwind-tables", "-fno-omit-frame-pointer",
"-fstack-protector", "-Wall", "-Wstrict-prototypes",
"-Wmissing-prototypes", "-Wpointer-arith", "-Wcast-qual", "-Wundef",
"-Wno-pointer-sign", "-Wmissing-include-dirs", "-Wno-unknown-pragmas",
"-Wno-address-of-packed-member", "-Wno-format-zero-length", "-std=gnu17",
"-D__printf__=__freebsd_kprintf__", f"--target={architecture['target_triple']}",
*architecture["kernel_cflags"],
]
modules: dict[str, dict[str, Any]] = {}
for zstdio in (0, 1):
build = arch_root / f"zstdio{zstdio}"
build.mkdir(parents=True)
(build / "opt_global.h").write_bytes(b"")
(build / "machine").symlink_to(sys_root / architecture["machine_header_dir"] / "include")
for alias, relative in architecture["header_aliases"].items():
(build / alias).symlink_to(sys_root / relative)
for mode in ("-p", "-q", "-h"):
generated = run(
["awk", "-f", str(sys_root / "tools/vnode_if.awk"),
str(sys_root / "kern/vnode_if.src"), mode],
cwd=build,
).stdout
suffix = {"-p": "vnode_if_newproto.h", "-q": "vnode_if_typedef.h", "-h": "vnode_if.h"}[mode]
(build / suffix).write_bytes(generated)
include_flags = [
"-include", str(build / "opt_global.h"), "-I", str(build),
"-I", str(sys_root), "-I", str(sys_root / "contrib/ck/include"),
]
objects = []
stdout_parts = []
stderr_parts = []
for source_name in SPEC["cross_build_contract"]["module_sources"]:
obj = build / f"{Path(source_name).stem}.o"
argv = [str(compiler), *common_flags, *include_flags]
if source_name == "decompressor_zstd.c":
argv.extend(["-I", str(sys_root / "contrib/zstd/lib/freebsd")])
if zstdio == 1:
argv.append("-DZSTDIO")
argv.extend(["-c", str(src / source_name), "-o", str(obj)])
completed = run(argv, cwd=build, timeout_seconds=60)
stdout_parts.append(completed.stdout)
stderr_parts.append(completed.stderr)
objects.append(obj)
module = build / "erofs.ko"
linked = run(
[str(compiler), f"--target={architecture['target_triple']}",
"-r", "-nostdlib", *map(str, objects), "-o", str(module)],
cwd=build,
timeout_seconds=60,
)
stdout_parts.append(linked.stdout)
stderr_parts.append(linked.stderr)
nm_output = run(["nm", "-u", str(module)]).stdout
(build / "build.stdout").write_bytes(b"".join(stdout_parts))
(build / "build.stderr").write_bytes(b"".join(stderr_parts))
(build / "nm-u.txt").write_bytes(nm_output)
modules[f"zstdio{zstdio}"] = {
"path": str(module),
"sha256": sha256_path(module),
"size_bytes": module.stat().st_size,
"nm_u_sha256": sha256_bytes(nm_output),
}
probe_root = arch_root / "probes"
probe_root.mkdir()
include_root = probe_root / "include"
include_root.mkdir()
(include_root / "machine").symlink_to(
sys_root / architecture["machine_header_dir"] / "include"
)
for alias, relative in architecture["header_aliases"].items():
(include_root / alias).symlink_to(sys_root / relative)
probe_flags = [
f"--target={architecture['target_triple']}", "-std=gnu17", "-nostdinc",
"-isystem", str(resource_include), "-isystem", str(FREEBSD_SRC / "include"),
"-isystem", str(include_root), "-isystem", str(sys_root),
]
layout = source_root / "repo-pre-15/tests/pre15/probes/ondisk_layout.c"
layout_result = run(
[str(compiler), *probe_flags, "-Wall", "-Wextra", "-Werror",
"-fsyntax-only", "-Xclang", "-fdump-record-layouts", str(layout)],
timeout_seconds=60,
)
(probe_root / "layout.stdout").write_bytes(layout_result.stdout)
(probe_root / "layout.stderr").write_bytes(layout_result.stderr)
unaligned = probe_root / "unaligned-endian.c"
unaligned.write_text(
SPEC["cross_build_contract"]["unaligned_endian_probe_source"],
encoding="ascii",
)
assembly = probe_root / "unaligned-endian.s"
endian_result = run(
[str(compiler), *probe_flags, *architecture["kernel_cflags"],
"-O2", "-S", str(unaligned), "-o", str(assembly)],
timeout_seconds=60,
)
(probe_root / "unaligned-endian.stdout").write_bytes(endian_result.stdout)
(probe_root / "unaligned-endian.stderr").write_bytes(endian_result.stderr)
if assembly.stat().st_size == 0:
raise GateFailure("RUNNER_FAIL", f"empty unaligned/endian probe for {arch_id}")
return {
"status": "PASS",
"temporary_source": str(source_root),
"modules": modules,
"layout_probe_sha256": sha256_path(layout),
"layout_output_sha256": sha256_bytes(layout_result.stdout),
"unaligned_endian_probe_sha256": sha256_path(unaligned),
"unaligned_endian_assembly_sha256": sha256_path(assembly),
}
def render_argv(values: list[str], substitutions: dict[str, str]) -> list[str]:
rendered = []
for value in values:
for key, replacement in substitutions.items():
value = value.replace("{" + key + "}", replacement)
rendered.append(value)
return rendered
def run_native(
architecture: dict[str, Any],
declaration: dict[str, Any],
modules: dict[str, dict[str, Any]],
resolved_base: str,
patch_sha256: str,
) -> dict[str, Any]:
runner_relative = safe_relative_path(declaration.get("runner_path", ""))
runner = ROOT / runner_relative
if not runner.is_file():
raise GateFailure("INFRA_BLOCKED", f"declared native runner is absent: {runner_relative}")
if sha256_path(runner) != validate_hash(
declaration.get("runner_sha256"), f"{architecture['id']} runner"
):
raise GateFailure("INFRA_BLOCKED", "declared native runner identity changed")
native_output = OUTPUT / "native" / architecture["id"]
native_output.mkdir(parents=True)
substitutions = {
"runner": str(runner),
"architecture": architecture["id"],
"module_zstdio0": modules["zstdio0"]["path"],
"module_zstdio1": modules["zstdio1"]["path"],
"output": str(native_output),
}
preflight_argv = render_argv(declaration["preflight_argv"], substitutions)
preflight = run(
preflight_argv,
cwd=ROOT,
timeout_seconds=declaration["preflight_deadline_seconds"],
)
(native_output / "preflight.runner.stdout").write_bytes(preflight.stdout)
(native_output / "preflight.runner.stderr").write_bytes(preflight.stderr)
preflight_record = json.loads(
(native_output / "preflight.json").read_text(encoding="utf-8")
)
for key, value in {
"status": "PASS", "exit_code": 0, "target_marker": "reached",
"cleanup": "PASS", "uname_s": "FreeBSD",
"uname_m": architecture["expected_uname_m"],
"uname_p": architecture["expected_uname_p"],
}.items():
if preflight_record.get(key) != value:
raise GateFailure("INFRA_BLOCKED", f"native preflight mismatch: {key}")
runtime_argv = render_argv(declaration["runtime_argv"], substitutions)
runtime = run(
runtime_argv,
cwd=ROOT,
allowed={0, 10},
timeout_seconds=declaration["runtime_deadline_seconds"],
)
(native_output / "runner.stdout").write_bytes(runtime.stdout)
(native_output / "runner.stderr").write_bytes(runtime.stderr)
return validate_runtime_record(
native_output / "runtime-result.json",
architecture,
modules,
resolved_base,
patch_sha256,
)
def main() -> int:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-027":
raise GateFailure("RUNNER_FAIL", "invalid P15-027 input identity")
if SPEC.get("gate") != "G07" or SPEC.get("batch") != "B36":
raise GateFailure("RUNNER_FAIL", "invalid G07/B36 input identity")
if SPEC.get("prohibited_paths") != ["introduction.md"]:
raise GateFailure("RUNNER_FAIL", "prohibited-path contract changed")
resolved = git_text(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure(
"INFRA_BLOCKED", f"P15-027 must replay {SPEC['required_base']}, got {resolved}"
)
source_tree = git_text(ROOT, "rev-parse", f"{resolved}:repo-pre-15/src")
if source_tree != SPEC["repo_source_tree_oid"]:
raise GateFailure("INFRA_BLOCKED", "frozen EROFS source tree changed")
source_hashes = {
path: sha256_bytes(source_at(resolved, path))
for path in SPEC["source_sha256"]
}
if source_hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen gate source identity changed")
write_json(OUTPUT / "source-sha256.json", source_hashes)
addendum_paths = (
"repo-pre-15/tests/pre15/gates/P15-027.sh",
"repo-pre-15/tests/pre15/gates/P15-027-input.json",
"repo-pre-15/docs/pre15-stage0/P15-027.md",
)
addendum_hashes = {
path: sha256_path(ROOT / path) for path in addendum_paths
}
write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes)
if sha256_path(ROOT / "repo-pre-15/src/Makefile") != SPEC["source_sha256"]["repo-pre-15/src/Makefile"]:
raise GateFailure("RUNNER_FAIL", "production Makefile differs from frozen BASE")
patch_spec = SPEC["temporary_makefile_relaxation"]
patch_bytes = patch_spec["unified_diff"].encode("ascii")
if sha256_bytes(patch_bytes) != patch_spec["sha256"]:
raise GateFailure("RUNNER_FAIL", "input-recorded temporary patch hash mismatch")
patch_check = WORK / "patch-check"
makefile_copy = patch_check / "repo-pre-15/src/Makefile"
makefile_copy.parent.mkdir(parents=True)
makefile_copy.write_bytes(source_at(resolved, patch_spec["path"]))
patch_result = run(["patch", "-p1"], cwd=patch_check, input_bytes=patch_bytes)
(OUTPUT / "makefile-relaxation.patch").write_bytes(patch_bytes)
(OUTPUT / "patch.stdout").write_bytes(patch_result.stdout)
(OUTPUT / "patch.stderr").write_bytes(patch_result.stderr)
if sha256_path(makefile_copy) != patch_spec["patched_makefile_sha256"]:
raise GateFailure("RUNNER_FAIL", "temporary Makefile relaxation result changed")
patch_record = {
"path": patch_spec["path"],
"sha256": patch_spec["sha256"],
"patched_makefile_sha256": sha256_path(makefile_copy),
"production_makefile_changed": False,
"temporary_apply": "PASS",
}
write_json(OUTPUT / "temporary-patch.json", patch_record)
if not FREEBSD_SRC.is_dir():
raise GateFailure("INFRA_BLOCKED", f"FreeBSD source is absent: {FREEBSD_SRC}")
freebsd = SPEC["freebsd_source"]
freebsd_head = git_text(FREEBSD_SRC, "rev-parse", "HEAD")
freebsd_prefix = git_text(FREEBSD_SRC, "rev-parse", "--show-prefix")
if freebsd_head != freebsd["head"] or freebsd_prefix != freebsd["git_prefix"]:
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
git_root = Path(git_text(FREEBSD_SRC, "rev-parse", "--show-toplevel"))
freebsd_source_tree = git_text(
git_root, "rev-parse", f"{freebsd_head}:{freebsd['git_tree_path']}"
)
freebsd_sys_tree = git_text(
git_root, "rev-parse", f"{freebsd_head}:{freebsd['git_tree_path']}/sys"
)
if freebsd_source_tree != freebsd["source_tree_oid"] or freebsd_sys_tree != freebsd["sys_tree_oid"]:
raise GateFailure("INFRA_BLOCKED", "FreeBSD source tree object changed")
freebsd_hashes = {
path: sha256_path(FREEBSD_SRC / path) for path in freebsd["sha256"]
}
if freebsd_hashes != freebsd["sha256"]:
raise GateFailure("INFRA_BLOCKED", "selected FreeBSD source hashes changed")
compiler_path = Path(shutil.which("clang") or "").resolve()
if not compiler_path.is_file():
raise GateFailure("INFRA_BLOCKED", "clang is unavailable")
compiler = SPEC["toolchain"]["compiler"]
compiler_version = run([str(compiler_path), "--version"]).stdout.decode(
"utf-8", errors="strict"
).splitlines()[0]
if (
str(compiler_path) != compiler["realpath"]
or sha256_path(compiler_path) != compiler["sha256"]
or compiler_version != compiler["version"]
):
raise GateFailure("INFRA_BLOCKED", "cross compiler identity changed")
compiler_targets = run([str(compiler_path), "--print-targets"]).stdout.decode(
"utf-8", errors="strict"
)
architectures = SPEC["architectures"]
if [item.get("id") for item in architectures] != ["arm64", "riscv64", "i386"]:
raise GateFailure("RUNNER_FAIL", "architecture audit set is not exact")
if sum(item.get("pointer_bits") == 32 for item in architectures) < 1:
raise GateFailure("RUNNER_FAIL", "architecture audit lacks a 32-bit target")
source_support = {}
for architecture in architectures:
arch_id = architecture["id"]
header = FREEBSD_SRC / "sys" / architecture["machine_header_dir"] / "include/param.h"
if not header.is_file():
raise GateFailure("INFRA_BLOCKED", f"FreeBSD source lacks {arch_id}")
if sha256_path(header) != architecture["param_h_sha256"]:
raise GateFailure("INFRA_BLOCKED", f"FreeBSD {arch_id} source identity changed")
target_pattern = rf"^\s*{re.escape(architecture['compiler_backend'])}\s+-"
if re.search(target_pattern, compiler_targets, re.MULTILINE) is None:
raise GateFailure("INFRA_BLOCKED", f"clang lacks {arch_id} backend")
source_support[arch_id] = {
"status": "PASS",
"freebsd_machine_headers": str(header),
"param_h_sha256": sha256_path(header),
"compiler_backend": architecture["compiler_backend"],
"target_triple": architecture["target_triple"],
"pointer_bits": architecture["pointer_bits"],
}
probe_source = SPEC["cross_build_contract"]["unaligned_endian_probe_source"].encode("ascii")
if sha256_bytes(probe_source) != SPEC["cross_build_contract"]["unaligned_endian_probe_sha256"]:
raise GateFailure("RUNNER_FAIL", "input-recorded unaligned/endian probe changed")
inventory = {
"schema": 1,
"host": {"system": platform.system(), "machine": platform.machine()},
"local_host_is_qualifying_native_freebsd": False,
"local_host_reason": "host is not FreeBSD and is not any candidate architecture",
"declared_native_environments": {
item["id"]: item["native_environment"] is not None for item in architectures
},
"emulator_tools": {
item["id"]: shutil.which(item["emulator_tool"]) for item in architectures
},
"emulator_binary_is_native_runtime_evidence": False,
"source_toolchain_support": source_support,
"protected_resources_addressed": False,
"inventory_rule": "only an explicit, validated same-architecture FreeBSD runner declaration can start cross work",
}
write_json(OUTPUT / "runtime-environment-inventory.json", inventory)
write_json(
OUTPUT / "freebsd-source-identity.json",
{
"path": str(FREEBSD_SRC),
"git_root": str(git_root),
"git_prefix": freebsd_prefix,
"git_tree_path": freebsd["git_tree_path"],
"head": freebsd_head,
"source_tree_oid": freebsd_source_tree,
"sys_tree_oid": freebsd_sys_tree,
"sha256": freebsd_hashes,
},
)
write_json(
OUTPUT / "toolchain-identity.json",
{
"compiler_realpath": str(compiler_path),
"compiler_sha256": sha256_path(compiler_path),
"compiler_version": compiler_version,
"print_targets_sha256": sha256_bytes(compiler_targets.encode("utf-8")),
},
)
results = []
go_architectures = []
for architecture in architectures:
declaration = architecture["native_environment"]
record: dict[str, Any] = {
"architecture": architecture["id"],
"machine_arch": architecture["machine_arch"],
"pointer_bits": architecture["pointer_bits"],
"source_toolchain": source_support[architecture["id"]],
}
if declaration is None:
record.update(
{
"native_environment": {
"status": "STOP",
"reason": "no declared same-architecture FreeBSD runtime environment",
},
"cross_build_and_probes": {
"status": "NOT_RUN",
"reason": "cross-only evidence cannot authorize allowlisting",
},
"native_runtime": {
"status": "NOT_RUN",
"reason": "same-architecture FreeBSD runtime is absent",
},
"decision": "STOP",
}
)
results.append(record)
continue
for key in (
"runner_path", "runner_sha256", "preflight_argv", "runtime_argv",
"preflight_deadline_seconds", "runtime_deadline_seconds",
):
if key not in declaration:
raise GateFailure("RUNNER_FAIL", f"incomplete native declaration: {key}")
cross = compile_architecture(
architecture, resolved, patch_bytes, compiler_path
)
native = run_native(
architecture,
declaration,
cross["modules"],
resolved,
patch_spec["sha256"],
)
native_pass = native["status"] == "PASS"
record.update(
{
"native_environment": {"status": "PASS", "declaration": declaration},
"cross_build_and_probes": cross,
"native_runtime": native,
"decision": "GO" if native_pass else "STOP",
}
)
if native_pass:
go_architectures.append(architecture["id"])
results.append(record)
overall_status = "GO" if go_architectures else "STOP"
result = {
"schema": 1,
"gate": "G07",
"candidate": "P15-027",
"batch": "B36",
"status": overall_status,
"reason": (
"at least one architecture completed its own cross and native requirements"
if go_architectures
else "no architecture has a declared same-architecture FreeBSD runtime"
),
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"repo_source_tree_oid": source_tree,
"deadline_seconds": DEADLINE,
"expected_exit_code": 0 if go_architectures else 22,
"architectures": results,
"go_architectures": go_architectures,
"allowlist_authorization": go_architectures,
"false_go_guard": "each authorized architecture must independently have cross_build_and_probes=PASS and native_runtime=PASS",
"b36": "AUTHORIZED" if go_architectures else "STOP-NO-SOURCE",
"production_makefile_guard_preserved": True,
"production_source_changed": False,
"qemu": "NOT_RUN" if not go_architectures else "runner-defined",
"full_feature_suite": "NOT_RUN",
"smoke_suite": "NOT_RUN",
"protected_resources_addressed": False,
}
write_json(OUTPUT / "result.json", result)
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
return 0 if go_architectures else 22
try:
raise SystemExit(main())
except GateFailure as error:
failure = {
"schema": 1,
"gate": "G07",
"candidate": "P15-027",
"status": error.status,
"reason": error.reason,
"requested_base": REQUESTED_BASE,
"deadline_seconds": DEADLINE,
"b36": "NOT_RUN",
"target_marker": "not_reached",
"production_source_changed": False,
"protected_resources_addressed": False,
}
write_json(OUTPUT / "result.json", failure)
print(json.dumps(failure, ensure_ascii=True, sort_keys=True))
raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20))
PY
gate_rc=$?
set -e
cleanup_status=PASS
if ! rm -rf -- "$work"; then
cleanup_status=FAIL
gate_rc=20
fi
trap - EXIT HUP INT TERM
printf 'owned temporary path removed: %s\ncleanup=%s\n' \
"$work" "$cleanup_status" >"$output/cleanup.log"
python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
output = Path(sys.argv[1])
exit_code = int(sys.argv[2])
cleanup = sys.argv[3]
if cleanup != "PASS":
status = "RUNNER_FAIL"
origin = "runner"
elif exit_code == 0:
status = "PASS"
origin = "gate"
elif exit_code == 22:
status = "STOP"
origin = "gate"
elif exit_code in (124, 137):
status = "INFRA_BLOCKED"
origin = "infrastructure"
else:
status = "RUNNER_FAIL"
origin = "runner"
(output / "attempt.json").write_text(
json.dumps(
{
"schema": 1,
"exit_code": exit_code,
"status": status,
"failure_origin": origin,
"cleanup": cleanup,
"target_marker": "reached" if status in {"PASS", "STOP"} else "not_reached",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
lines = []
for path in sorted(output.iterdir(), key=lambda item: item.name):
if path.is_file() and path.name != "SHA256SUMS":
digest = hashlib.sha256(path.read_bytes()).hexdigest()
lines.append(f"{digest} {path.name}")
(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
PY
printf 'P15-027 gate exit=%s cleanup=%s output=%s\n' \
"$gate_rc" "$cleanup_status" "$output"
exit "$gate_rc"
tests/pre15/gates/P15-030-input.json
+110
View File
@@ -0,0 +1,110 @@
{
"approved_consumer_manifest_sha256": {},
"batch": "B37a",
"candidate": "P15-030",
"consumer_contract": {
"allowed_classes": [
"diagnostic",
"operational"
],
"allowed_signal_ids": [
"features",
"cache_hits",
"cache_misses",
"cache_waits",
"decoder_errors"
],
"nonqualifying_examples": [
"hypothetical future user",
"Linux sysfs analogy",
"generic observability value",
"test script",
"gate prototype"
],
"prohibited_implementation_prefixes": [
"planning/",
"repo-pre-15/docs/",
"repo-pre-15/tests/"
],
"required_access_mode": "privileged-read",
"required_fields": [
"schema",
"consumer_id",
"consumer_class",
"owner",
"version",
"production_use",
"implementation_path",
"implementation_sha256",
"entrypoint",
"workflow",
"deployment_reference",
"signal_ids",
"access_mode"
],
"schema_marker": "pre15-p15-030-consumer-v1"
},
"consumer_inventory": {
"expected_reference_counts": {
"historical-evidence": 1,
"planning-proposal": 31,
"possible-operational-consumer": 0,
"project-report-or-documentation": 3,
"test-only": 19,
"total": 54
},
"expected_scoped_path_count": 3206,
"maximum_candidate_bytes": 2097152,
"reference_terms": [
"sysctl",
"sysfs"
],
"scope": "tracked repo-pre-15 and planning/pre15 at required_base, excluding planning/pre15/introduction.md and the P15-030 addendum"
},
"dependency_rule": {
"batch": "B37b",
"candidate": "P15-068",
"identity_format_verdict_when_transport_stops": "NOT_RUN",
"on_p15_030_stop": "STOP-NO-SOURCE",
"reason": "P15-068 requires the P15-030 sysctl transport; do not create a second lifecycle"
},
"gate": "G08",
"go_requirements": [
"concrete existing diagnostic or operational consumer",
"bounded atomic counters",
"FreeBSD 15 native sysctl_ctx lifecycle closes parse failure, normal unmount, forced unmount, delayed handler, and sysctl_ctx_free failure without freeing handler-visible mount state",
"stable permissions and ABI with no unauthenticated metadata leak"
],
"probe_order": [
"consumer-inventory",
"bounded-atomic-counter-prototype",
"freebsd15-native-sysctl-ctx-lifecycle",
"permissions-abi-metadata-leak"
],
"prohibited_paths": [
"planning/pre15/introduction.md"
],
"protected": {
"pid": 26318,
"port": 9222
},
"required_base": "3e1d26a53eef30ecadc3407444d214feac861cb4",
"schema": 1,
"scope_tree_oids": {
"planning/pre15": "b94ad587533ff9fb14e434a8db061381af0ab713",
"repo-pre-15": "c2e260c5f9e340e28e13481b3e5a3d04b5b916c8"
},
"source_sha256": {
"planning/pre15/20-final-candidate-ledger.md": "e40f68de241a9c4e5e5bff0849b166b614f56e6cba0b64fc3ced41c036c99c91",
"planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8",
"planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20",
"planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d",
"repo-pre-15/README.md": "2d3e138f5af20ce02aa562286cb42df8fbd00fdad0ae424e426e0fd8a7f72472",
"repo-pre-15/docs/capabilities.md": "f4b2fc0ff74fb7f808ed0f833e87c8230877ab249b611d30a1dbc2103535a8b8",
"repo-pre-15/docs/erofs.5": "dea6a9eddbccbb7b8779b6b4e43772edeea9f0e51696c1b8fd09e4d8e69b52f5",
"repo-pre-15/docs/features.md": "6d875eb582ef9f00391f90ee5b4b8aec0429f24f0ae3c7b27e788ab1e2c48f2e",
"repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8",
"repo-pre-15/src/internal.h": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a",
"repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2"
}
}
tests/pre15/gates/P15-030.sh
Executable
+592
View File
@@ -0,0 +1,592 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-030-input.json
base=
output=
deadline=60
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 20
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; }
for tool in git mktemp python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 21
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; }
mkdir -p "$output"
work=$(mktemp -d "${TMPDIR:-/tmp}/P15-030.XXXXXX")
cleanup_trap()
{
rm -rf -- "$work"
}
trap cleanup_trap EXIT HUP INT TERM
python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n",
encoding="ascii",
)
PY
python3 -B - "$output/ownership.json" "$work" "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(
{
"owned_temporary_paths": [sys.argv[2]],
"persistent_output": sys.argv[3],
"owned_processes": [],
"owned_ports": [],
"owned_overlays": [],
"qemu": "NOT_RUN",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
PY
set +e
timeout -k 5 "$deadline" python3 -B - \
"$root" "$input" "$base" "$output" "$deadline" <<'PY' \
>"$output/stdout.log" 2>"$output/stderr.log"
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import subprocess
import sys
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
DEADLINE = int(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="utf-8"))
class GateFailure(RuntimeError):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def write_json(path: Path, value: Any) -> None:
path.write_text(
json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes:
try:
completed = subprocess.run(
argv,
cwd=ROOT,
check=False,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=20,
)
except subprocess.TimeoutExpired as error:
raise GateFailure(
"INFRA_BLOCKED", f"command timed out: {' '.join(argv)}"
) from error
accepted = {0} if allowed is None else allowed
if completed.returncode not in accepted:
detail = completed.stderr.decode("utf-8", errors="replace").strip()
raise GateFailure(
"RUNNER_FAIL",
f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}",
)
return completed.stdout
def git_text(*args: str) -> str:
return run_bytes(["git", "-C", str(ROOT), *args]).decode(
"utf-8", errors="strict"
).strip()
def source_at(commit: str, relative: str) -> bytes:
return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"])
def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]:
entries: dict[str, dict[str, Any]] = {}
for record in raw.split(b"\0"):
if not record:
continue
metadata, path_bytes = record.split(b"\t", 1)
mode, kind, oid, size = metadata.decode("ascii").split()
path = path_bytes.decode("utf-8", errors="strict")
entries[path] = {
"mode": mode,
"kind": kind,
"oid": oid,
"size": int(size),
}
return entries
def grep_paths(commit: str, expressions: list[str], fixed: bool = False) -> list[str]:
argv = ["git", "-C", str(ROOT), "grep", "-I", "-l"]
if fixed:
argv.append("-F")
else:
argv.append("-i")
for expression in expressions:
argv.extend(["-e", expression])
argv.extend(
[
commit,
"--",
"repo-pre-15",
"planning/pre15",
":(exclude)planning/pre15/introduction.md",
":(exclude)repo-pre-15/tests/pre15/gates/P15-030.sh",
":(exclude)repo-pre-15/tests/pre15/gates/P15-030-input.json",
":(exclude)repo-pre-15/docs/pre15-stage0/P15-030.md",
]
)
raw = run_bytes(argv, {0, 1})
prefix = f"{commit}:"
paths = []
for line in raw.decode("utf-8", errors="strict").splitlines():
if not line.startswith(prefix):
raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}")
paths.append(line[len(prefix):])
return sorted(paths)
def classify_reference(path: str) -> str:
if path.startswith("planning/pre15/evidence/"):
return "historical-evidence"
if path.startswith("planning/pre15/"):
return "planning-proposal"
if path.startswith("repo-pre-15/tests/"):
return "test-only"
if path.startswith("repo-pre-15/docs/") or path.startswith("repo-pre-15/current/"):
return "project-report-or-documentation"
return "possible-operational-consumer"
def require_text(record: dict[str, Any], field: str) -> str:
value = record.get(field)
if not isinstance(value, str) or not value.strip():
raise GateFailure("RUNNER_FAIL", f"consumer declaration lacks {field}")
return value.strip()
def validate_consumer_manifest(
path: str,
data: bytes,
entries: dict[str, dict[str, Any]],
commit: str,
) -> dict[str, Any]:
try:
record = json.loads(data.decode("utf-8", errors="strict"))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise GateFailure("RUNNER_FAIL", f"invalid consumer manifest: {path}") from error
if not isinstance(record, dict):
raise GateFailure("RUNNER_FAIL", f"consumer manifest is not an object: {path}")
contract = SPEC["consumer_contract"]
if record.get("schema") != contract["schema_marker"]:
raise GateFailure("RUNNER_FAIL", f"consumer manifest schema mismatch: {path}")
consumer_class = require_text(record, "consumer_class")
if consumer_class not in contract["allowed_classes"]:
raise GateFailure("RUNNER_FAIL", f"consumer class is not operational: {path}")
if record.get("production_use") is not True:
raise GateFailure("RUNNER_FAIL", f"consumer is not declared in production use: {path}")
access_mode = require_text(record, "access_mode")
if access_mode != contract["required_access_mode"]:
raise GateFailure("RUNNER_FAIL", f"consumer access mode is not privileged: {path}")
for field in (
"consumer_id",
"owner",
"version",
"entrypoint",
"workflow",
"deployment_reference",
):
require_text(record, field)
signals = record.get("signal_ids")
if (
not isinstance(signals, list)
or not signals
or any(not isinstance(item, str) for item in signals)
or len(set(signals)) != len(signals)
or not set(signals).issubset(set(contract["allowed_signal_ids"]))
):
raise GateFailure("RUNNER_FAIL", f"consumer signal set is invalid: {path}")
implementation_path = require_text(record, "implementation_path")
if not implementation_path.startswith("repo-pre-15/"):
raise GateFailure("RUNNER_FAIL", f"consumer implementation is outside DUT: {path}")
if any(
implementation_path.startswith(prefix)
for prefix in contract["prohibited_implementation_prefixes"]
):
raise GateFailure("RUNNER_FAIL", f"consumer implementation is nonqualifying: {path}")
if implementation_path not in entries:
raise GateFailure("RUNNER_FAIL", f"consumer implementation is not tracked: {path}")
implementation = source_at(commit, implementation_path)
implementation_hash = sha256_bytes(implementation)
if implementation_hash != require_text(record, "implementation_sha256"):
raise GateFailure("RUNNER_FAIL", f"consumer implementation hash mismatch: {path}")
implementation_text = implementation.decode("utf-8", errors="replace").lower()
if "sysctl" not in implementation_text:
raise GateFailure("RUNNER_FAIL", f"consumer implementation does not invoke sysctl: {path}")
missing_signals = [item for item in signals if item.lower() not in implementation_text]
if missing_signals:
raise GateFailure(
"RUNNER_FAIL", f"consumer implementation omits signals {missing_signals}: {path}"
)
return {
"manifest_path": path,
"manifest_sha256": sha256_bytes(data),
"consumer_id": record["consumer_id"],
"consumer_class": consumer_class,
"implementation_path": implementation_path,
"implementation_sha256": implementation_hash,
"signal_ids": signals,
"access_mode": access_mode,
"production_use": True,
}
def main() -> int:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-030":
raise GateFailure("RUNNER_FAIL", "invalid P15-030 input identity")
if SPEC.get("gate") != "G08" or SPEC.get("batch") != "B37a":
raise GateFailure("RUNNER_FAIL", "invalid G08/B37a input identity")
resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure(
"INFRA_BLOCKED",
f"P15-030 must replay {SPEC['required_base']}, got {resolved}",
)
for path, expected_oid in SPEC["scope_tree_oids"].items():
actual_oid = git_text("rev-parse", f"{resolved}:{path}")
if actual_oid != expected_oid:
raise GateFailure("INFRA_BLOCKED", f"frozen scope tree changed: {path}")
source_hashes = {
path: sha256_bytes(source_at(resolved, path))
for path in SPEC["source_sha256"]
}
if source_hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen G08 source identity changed")
write_json(OUTPUT / "source-sha256.json", source_hashes)
addendum_paths = (
"repo-pre-15/tests/pre15/gates/P15-030.sh",
"repo-pre-15/tests/pre15/gates/P15-030-input.json",
"repo-pre-15/docs/pre15-stage0/P15-030.md",
)
addendum_hashes = {
path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths
}
write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes)
entries = parse_ls_tree(
run_bytes(
[
"git",
"-C",
str(ROOT),
"ls-tree",
"-r",
"--long",
"-z",
resolved,
"--",
"repo-pre-15",
"planning/pre15",
]
)
)
prohibited = set(SPEC["prohibited_paths"])
scoped_entries = {path: value for path, value in entries.items() if path not in prohibited}
if len(scoped_entries) != SPEC["consumer_inventory"]["expected_scoped_path_count"]:
raise GateFailure("INFRA_BLOCKED", "frozen consumer scope path count changed")
reference_paths = grep_paths(
resolved, SPEC["consumer_inventory"]["reference_terms"]
)
maximum_bytes = SPEC["consumer_inventory"]["maximum_candidate_bytes"]
references = []
reference_counts = {
"planning-proposal": 0,
"historical-evidence": 0,
"test-only": 0,
"project-report-or-documentation": 0,
"possible-operational-consumer": 0,
}
for path in reference_paths:
if path in prohibited:
raise GateFailure("RUNNER_FAIL", "prohibited path entered consumer scan")
entry = scoped_entries[path]
if entry["size"] > maximum_bytes:
raise GateFailure("INFRA_BLOCKED", f"consumer candidate exceeds bound: {path}")
data = source_at(resolved, path)
classification = classify_reference(path)
reference_counts[classification] += 1
references.append(
{
"path": path,
"blob_oid": entry["oid"],
"bytes": entry["size"],
"sha256": sha256_bytes(data),
"classification": classification,
"qualifies": False,
"reason": {
"planning-proposal": "proposal or audit text is not an existing consumer",
"historical-evidence": "historical gate output is not an existing consumer",
"test-only": "a test or gate is explicitly nonqualifying",
"project-report-or-documentation": "documentation is not an implemented consumer",
"possible-operational-consumer": "requires a validated consumer declaration",
}[classification],
}
)
expected_counts = SPEC["consumer_inventory"]["expected_reference_counts"]
if len(reference_paths) != expected_counts["total"]:
raise GateFailure("INFRA_BLOCKED", "frozen sysctl/sysfs reference count changed")
for classification, count in reference_counts.items():
if count != expected_counts[classification]:
raise GateFailure(
"INFRA_BLOCKED", f"frozen reference classification changed: {classification}"
)
marker = SPEC["consumer_contract"]["schema_marker"]
manifest_paths = grep_paths(resolved, [marker], fixed=True)
consumers = []
manifest_hashes = {}
for path in manifest_paths:
entry = scoped_entries[path]
if entry["size"] > maximum_bytes:
raise GateFailure("INFRA_BLOCKED", f"consumer manifest exceeds bound: {path}")
data = source_at(resolved, path)
manifest_hashes[path] = sha256_bytes(data)
consumers.append(validate_consumer_manifest(path, data, scoped_entries, resolved))
if manifest_hashes != SPEC["approved_consumer_manifest_sha256"]:
raise GateFailure("INFRA_BLOCKED", "declared consumer inventory changed")
inventory = {
"schema": 1,
"base": resolved,
"scope": SPEC["consumer_inventory"]["scope"],
"scoped_path_count": len(scoped_entries),
"prohibited_path_read": False,
"reference_terms": SPEC["consumer_inventory"]["reference_terms"],
"reference_path_count": len(reference_paths),
"reference_classification_counts": reference_counts,
"references": references,
"consumer_manifest_marker": marker,
"consumer_manifest_count": len(consumers),
"qualified_consumer_count": len(consumers),
"qualified_consumers": consumers,
"nonqualifying_examples": SPEC["consumer_contract"]["nonqualifying_examples"],
}
write_json(OUTPUT / "consumer-inventory.json", inventory)
consumer_present = bool(consumers)
conditions = [
{
"id": "concrete-existing-diagnostic-or-operational-consumer",
"status": "PASS" if consumer_present else "STOP",
"observation": (
f"{len(consumers)} qualified consumer declaration(s)"
if consumer_present
else "zero qualified consumers; all 54 broad references are planning, history, tests, or documentation"
),
},
{
"id": "bounded-atomic-counters",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent",
},
{
"id": "freebsd15-native-sysctl-ctx-lifecycle",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent; no prototype was built",
},
{
"id": "stable-permissions-abi-and-no-unauthenticated-metadata-leak",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent",
},
]
if consumer_present:
raise GateFailure(
"RUNNER_FAIL",
"a consumer is now declared; run the FreeBSD 15 native lifecycle prototype before any GO",
)
if all(item["status"] == "PASS" for item in conditions):
raise GateFailure("RUNNER_FAIL", "false-GO guard accepted incomplete conditions")
result = {
"schema": 1,
"gate": "G08",
"candidate": "P15-030",
"batch": "B37a",
"status": "STOP",
"reason": "no concrete existing diagnostic or operational consumer",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"deadline_seconds": DEADLINE,
"expected_exit_code": 22,
"conditions": conditions,
"false_go_guard": "GO requires all four conditions PASS; evaluated false",
"consumer_inventory": "consumer-inventory.json",
"b37a": "STOP-NO-SOURCE",
"dependency_closure": {
"candidate": "P15-068",
"batch": "B37b",
"status": "STOP",
"batch_status": "STOP-NO-SOURCE",
"reason": "P15-068 requires the P15-030 sysctl transport, which is STOP",
"independent_identity_format_verdict": "NOT_RUN",
"second_sysctl_lifecycle": "NOT_RUN",
},
"prototype": "NOT_RUN",
"builds": "NOT_RUN",
"qemu": "NOT_RUN",
"full_feature_suite": "NOT_RUN",
"smoke_suite": "NOT_RUN",
"production_source_changed": False,
"p15_068_gate_addendum_created": False,
}
write_json(OUTPUT / "result.json", result)
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
return 22
try:
raise SystemExit(main())
except GateFailure as error:
failure = {
"schema": 1,
"gate": "G08",
"candidate": "P15-030",
"status": error.status,
"reason": error.reason,
"requested_base": REQUESTED_BASE,
"deadline_seconds": DEADLINE,
"b37a": "NOT_RUN",
"p15_068": "NOT_RUN",
"qemu": "NOT_RUN",
"production_source_changed": False,
}
write_json(OUTPUT / "result.json", failure)
print(json.dumps(failure, ensure_ascii=True, sort_keys=True))
raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20))
PY
gate_rc=$?
set -e
cleanup_status=PASS
if ! rm -rf -- "$work"; then
cleanup_status=FAIL
gate_rc=20
fi
trap - EXIT HUP INT TERM
printf 'owned temporary path removed: %s\ncleanup=%s\n' \
"$work" "$cleanup_status" >"$output/cleanup.log"
python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
output = Path(sys.argv[1])
exit_code = int(sys.argv[2])
cleanup = sys.argv[3]
if cleanup != "PASS":
status = "RUNNER_FAIL"
origin = "runner"
elif exit_code == 22:
status = "STOP"
origin = "gate"
elif exit_code in (124, 137):
status = "INFRA_BLOCKED"
origin = "infrastructure"
else:
status = "RUNNER_FAIL" if exit_code == 20 else "INFRA_BLOCKED"
origin = "runner" if exit_code == 20 else "infrastructure"
(output / "attempt.json").write_text(
json.dumps(
{
"schema": 1,
"exit_code": exit_code,
"status": status,
"failure_origin": origin,
"cleanup": cleanup,
"target_marker": "reached" if status == "STOP" else "not_reached",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
lines = []
for path in sorted(output.iterdir(), key=lambda item: item.name):
if path.is_file() and path.name != "SHA256SUMS":
digest = hashlib.sha256(path.read_bytes()).hexdigest()
lines.append(f"{digest} {path.name}")
(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
PY
printf 'P15-030 gate exit=%s cleanup=%s output=%s\n' \
"$gate_rc" "$cleanup_status" "$output"
exit "$gate_rc"
tests/pre15/gates/P15-031-input.json
+273
View File
@@ -0,0 +1,273 @@
{
"approved_consumer_manifest_sha256": {},
"batch": "B38",
"benchmark_contract": {
"control_drift_limit_percent": 3.0,
"hot_path_median_regression_limit_percent": 3.0,
"minimum_timed_iterations_per_sample": 100000,
"outlier_deletion": false,
"pair_order": "five AB and five BA pairs",
"raw_sample_retention": true,
"required_runs": 10,
"unstable_control_status": "INFRA_BLOCKED",
"warmup_runs": 3
},
"build_contract": {
"configurations": [
"WITH_ZSTDIO=0",
"WITH_ZSTDIO=1"
],
"disabled_probe_mode": true,
"new_undefined_symbols_allowed": 0
},
"candidate": "P15-031",
"consumer_contract": {
"allowed_transports": [
"dtrace-sdt",
"ktr"
],
"manifest_prefix": "repo-pre-15/diagnostics/",
"nonqualifying_examples": [
"hypothetical maintainer",
"Linux tracepoints",
"generic observability value",
"proposed B38 test",
"gate prototype",
"installed dtrace without a project consumer"
],
"prohibited_implementation_prefixes": [
"planning/",
"repo-pre-15/current/",
"repo-pre-15/docs/",
"repo-pre-15/src/",
"repo-pre-15/tests/"
],
"required_access_mode": "privileged-diagnostic",
"required_fields": [
"schema",
"consumer_id",
"consumer_class",
"owner",
"version",
"production_use",
"freebsd_version",
"transport",
"access_mode",
"implementation_path",
"implementation_sha256",
"entrypoint",
"workflow",
"deployment_reference",
"event_schema_sha256",
"event_ids",
"actual_capture",
"capture_evidence_path",
"capture_evidence_sha256",
"captured_event_counts"
],
"schema_marker": "pre15-p15-031-consumer-v1"
},
"consumer_inventory": {
"expected_reference_counts": {
"historical-evidence": 3,
"linux-reference": 0,
"planning-proposal": 16,
"possible-project-consumer": 0,
"project-report-or-documentation": 4,
"test-only": 10,
"total": 33
},
"expected_scoped_path_count": 3243,
"maximum_candidate_bytes": 2097152,
"reference_patterns": [
"dtrace",
"(^|[^[:alnum:]_])sdt([^[:alnum:]_]|$)|sdt_(probe|provider)",
"(^|[^[:alnum:]_])ktr([^[:alnum:]_]|$)|ktrdump|ktr_[[:alnum:]_]+",
"(^|[^[:alnum:]_])tracepoints?([^[:alnum:]_]|$)",
"(^|[^[:alnum:]_])observability([^[:alnum:]_]|$)"
],
"scope": "tracked repo-pre-15 and planning/pre15 at required_base, excluding planning/pre15/introduction.md and the P15-031 addendum"
},
"event_schema": {
"forbidden_fields": [
"kernel_pointer",
"credential",
"path",
"xattr_name",
"xattr_value",
"uuid",
"volume_label",
"raw_unvalidated_metadata"
],
"provider": "erofs",
"events": [
{
"fields": [
"parse_stage",
"error"
],
"id": "mount_parse_fail",
"rule": "emit only bounded stage enum and positive errno; no image-derived value"
},
{
"fields": [
"nid",
"logical_offset",
"mapped_length",
"device_id",
"map_flags",
"error"
],
"id": "map_result",
"rule": "emit only after range validation; zero mapping fields on validation failure"
},
{
"fields": [
"nid"
],
"id": "vget_hit",
"rule": "validated numeric inode identity only"
},
{
"fields": [
"nid"
],
"id": "vget_miss",
"rule": "validated numeric inode identity only"
},
{
"fields": [
"nid"
],
"id": "vget_loser",
"rule": "validated numeric inode identity only"
},
{
"fields": [
"nid",
"name_index"
],
"id": "xattr_cache_hit",
"rule": "namespace index only; no xattr name or value"
},
{
"fields": [
"nid",
"name_index"
],
"id": "xattr_cache_miss",
"rule": "namespace index only; no xattr name or value"
},
{
"fields": [
"nid",
"algorithm",
"offset",
"input_length",
"output_length"
],
"id": "decode_start",
"rule": "validated numeric decode request only"
},
{
"fields": [
"nid",
"algorithm",
"offset",
"decoded_length"
],
"id": "decode_end",
"rule": "validated numeric decode result only"
},
{
"fields": [
"nid",
"algorithm",
"offset",
"error"
],
"id": "decode_error",
"rule": "validated numeric identity and positive errno only"
},
{
"fields": [
"nid",
"offset",
"decoded_length"
],
"id": "cache_wait",
"rule": "validated cache key components only"
},
{
"fields": [
"nid",
"offset",
"decoded_length",
"reason"
],
"id": "cache_evict",
"rule": "bounded reason enum and validated cache key components only"
}
]
},
"gate": "G08",
"go_requirements": [
"a concrete existing project diagnostic consumer captures the complete predeclared schema through FreeBSD DTrace/SDT or a justified KTR path",
"no kernel pointers, credentials, or unauthenticated metadata leakage",
"an owned-temp pre-source prototype generated from this JSON and the gate script closes fields, counts, failure, detach, and unload without linking into EROFS",
"disabled WITH_ZSTDIO=0 and WITH_ZSTDIO=1 builds add no undefined symbols",
"ten-run hot-path median regression is at most three percent with raw interleaved controls and no outlier deletion"
],
"probe_order": [
"consumer-inventory",
"privacy-schema",
"freebsd15-native-owned-temp-prototype",
"disabled-dual-build-undefined-symbols",
"ten-run-hot-path-microbenchmark"
],
"prohibited_paths": [
"planning/pre15/introduction.md"
],
"protected": {
"pid": 26318,
"port": 9222
},
"prototype_contract": {
"expected_event_count_each": 1,
"generated_from": [
"P15-031-input.json",
"P15-031.sh"
],
"lifecycle_scenarios": [
"consumer attach and complete capture",
"failed consumer enable",
"consumer termination and detach",
"provider unload and reload",
"final unload with zero owned processes, modules, and temporary paths"
],
"linked_into_erofs": false,
"location": "owned temporary directory only",
"native_target": "FreeBSD 15"
},
"required_base": "0f696ad1e2c6628022d02ce52d07eb66704769dd",
"schema": 1,
"scope_tree_oids": {
"planning/pre15": "bb8801a7696465fce1607b671c96473c044773b5",
"repo-pre-15": "231f9d5566d0a6bee4672a89f93451daa4f4b5de"
},
"source_sha256": {
"planning/pre15/20-final-candidate-ledger.md": "c47338623686ce352fd80cf9b39635900bcde8c2abc960feebe24837f8a67460",
"planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8",
"planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20",
"planning/pre15/50-feature-test-matrix.md": "ce39b44ef3989f4fcdbcd8eb677193ef53b630080c74d8539664fb2be3f38408",
"planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d",
"repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8",
"repo-pre-15/src/data.c": "a511146d2d1ff56bc88bae6c550cfdc49a8e978c5c2c88d735de92adaaa836b6",
"repo-pre-15/src/decompressor.c": "ccaa934f837ecbf3b3b678dbddf35d29dd7e3b4e80d1a7ab78b18a07e555419f",
"repo-pre-15/src/inode.c": "50c40fe403bd06b17f6976707ad4a8f03f95af49753f450df3d132130ab2dcf4",
"repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2",
"repo-pre-15/src/xattr.c": "7ce78b7af8838715fed10843f412458bdacb8c9b6811f08f660549f18a0743b0",
"repo-pre-15/src/zdata.c": "ee241c83fe7152569414594a3b4acf8c5028a35573d53ace0e3bb4a51dd0d191",
"repo-pre-15/tests/pre15/EVIDENCE-SCHEMA.json": "8a4d3fe076eed3a2f17e3ebd9337ca8697f09d6717c05835e076f6812cf38102"
}
}
tests/pre15/gates/P15-031.sh
Executable
+704
View File
@@ -0,0 +1,704 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-031-input.json
base=
output=
deadline=60
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 20
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; }
for tool in git mktemp python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 21
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; }
mkdir -p "$output"
work=$(mktemp -d "${TMPDIR:-/tmp}/P15-031.XXXXXX")
cleanup_trap()
{
rm -rf -- "$work"
}
trap cleanup_trap EXIT HUP INT TERM
python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n",
encoding="ascii",
)
PY
python3 -B - "$output/ownership.json" "$work" "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(
{
"owned_temporary_paths": [sys.argv[2]],
"persistent_output": sys.argv[3],
"owned_processes": [],
"owned_ports": [],
"owned_overlays": [],
"qemu": "NOT_RUN",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
PY
set +e
timeout -k 5 "$deadline" python3 -B - \
"$root" "$input" "$base" "$output" "$deadline" <<'PY' \
>"$output/stdout.log" 2>"$output/stderr.log"
from __future__ import annotations
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import subprocess
import sys
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
DEADLINE = int(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="utf-8"))
START_UTC = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
class GateFailure(RuntimeError):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def canonical_sha256(value: Any) -> str:
raw = json.dumps(
value, ensure_ascii=True, separators=(",", ":"), sort_keys=True
).encode("ascii")
return sha256_bytes(raw)
def write_json(path: Path, value: Any) -> None:
path.write_text(
json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes:
try:
completed = subprocess.run(
argv,
cwd=ROOT,
check=False,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=20,
)
except subprocess.TimeoutExpired as error:
raise GateFailure(
"INFRA_BLOCKED", f"command timed out: {' '.join(argv)}"
) from error
accepted = {0} if allowed is None else allowed
if completed.returncode not in accepted:
detail = completed.stderr.decode("utf-8", errors="replace").strip()
raise GateFailure(
"RUNNER_FAIL",
f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}",
)
return completed.stdout
def git_text(*args: str) -> str:
return run_bytes(["git", "-C", str(ROOT), *args]).decode(
"utf-8", errors="strict"
).strip()
def source_at(commit: str, relative: str) -> bytes:
return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"])
def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]:
entries: dict[str, dict[str, Any]] = {}
for record in raw.split(b"\0"):
if not record:
continue
metadata, path_bytes = record.split(b"\t", 1)
mode, kind, oid, size = metadata.decode("ascii").split()
path = path_bytes.decode("utf-8", errors="strict")
entries[path] = {
"mode": mode,
"kind": kind,
"oid": oid,
"size": int(size),
}
return entries
def grep_paths(commit: str, expressions: list[str], fixed: bool = False) -> list[str]:
argv = ["git", "-C", str(ROOT), "grep", "-I", "-l"]
if fixed:
argv.append("-F")
else:
argv.extend(["-i", "-E"])
for expression in expressions:
argv.extend(["-e", expression])
argv.extend(
[
commit,
"--",
"repo-pre-15",
"planning/pre15",
":(exclude)planning/pre15/introduction.md",
":(exclude)repo-pre-15/tests/pre15/gates/P15-031.sh",
":(exclude)repo-pre-15/tests/pre15/gates/P15-031-input.json",
":(exclude)repo-pre-15/docs/pre15-stage0/P15-031.md",
]
)
raw = run_bytes(argv, {0, 1})
prefix = f"{commit}:"
paths = []
for line in raw.decode("utf-8", errors="strict").splitlines():
if not line.startswith(prefix):
raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}")
paths.append(line[len(prefix):])
return sorted(paths)
def classify_reference(path: str) -> str:
if path.startswith("planning/pre15/evidence/"):
return "historical-evidence"
if path.startswith("planning/pre15/"):
return "planning-proposal"
if path.startswith("repo-pre-15/tests/"):
return "test-only"
if path.startswith("repo-pre-15/src-linux/"):
return "linux-reference"
if path.startswith("repo-pre-15/docs/") or path.startswith("repo-pre-15/current/"):
return "project-report-or-documentation"
return "possible-project-consumer"
def require_text(record: dict[str, Any], field: str) -> str:
value = record.get(field)
if not isinstance(value, str) or not value.strip():
raise GateFailure("RUNNER_FAIL", f"consumer declaration lacks {field}")
return value.strip()
def validate_bound_path(
record: dict[str, Any], path_field: str, hash_field: str,
entries: dict[str, dict[str, Any]], commit: str, maximum_bytes: int,
) -> tuple[str, bytes, str]:
path = require_text(record, path_field)
if not path.startswith("repo-pre-15/"):
raise GateFailure("RUNNER_FAIL", f"{path_field} is outside DUT: {path}")
if path not in entries:
raise GateFailure("RUNNER_FAIL", f"{path_field} is not tracked: {path}")
if entries[path]["size"] > maximum_bytes:
raise GateFailure("INFRA_BLOCKED", f"bound file exceeds size limit: {path}")
data = source_at(commit, path)
digest = sha256_bytes(data)
if digest != require_text(record, hash_field):
raise GateFailure("RUNNER_FAIL", f"{hash_field} mismatch: {path}")
return path, data, digest
def validate_consumer_manifest(
path: str, data: bytes, entries: dict[str, dict[str, Any]], commit: str,
event_schema_sha256: str, maximum_bytes: int,
) -> dict[str, Any]:
try:
record = json.loads(data.decode("utf-8", errors="strict"))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise GateFailure("RUNNER_FAIL", f"invalid consumer manifest: {path}") from error
if not isinstance(record, dict):
raise GateFailure("RUNNER_FAIL", f"consumer manifest is not an object: {path}")
contract = SPEC["consumer_contract"]
if not path.startswith(contract["manifest_prefix"]):
raise GateFailure("RUNNER_FAIL", f"consumer manifest is outside diagnostic prefix: {path}")
if record.get("schema") != contract["schema_marker"]:
raise GateFailure("RUNNER_FAIL", f"consumer manifest schema mismatch: {path}")
missing_fields = sorted(set(contract["required_fields"]) - set(record))
if missing_fields:
raise GateFailure(
"RUNNER_FAIL", f"consumer manifest lacks fields {missing_fields}: {path}"
)
if require_text(record, "consumer_class") != "diagnostic":
raise GateFailure("RUNNER_FAIL", f"consumer is not diagnostic: {path}")
if record.get("production_use") is not True or record.get("actual_capture") is not True:
raise GateFailure("RUNNER_FAIL", f"consumer lacks production capture evidence: {path}")
if not require_text(record, "freebsd_version").startswith("FreeBSD 15"):
raise GateFailure("RUNNER_FAIL", f"consumer capture is not FreeBSD 15: {path}")
transport = require_text(record, "transport")
if transport not in contract["allowed_transports"]:
raise GateFailure("RUNNER_FAIL", f"consumer transport is not native: {path}")
if require_text(record, "access_mode") != contract["required_access_mode"]:
raise GateFailure("RUNNER_FAIL", f"consumer access is not privileged: {path}")
for field in (
"consumer_id", "owner", "version", "entrypoint", "workflow",
"deployment_reference",
):
require_text(record, field)
if record.get("event_schema_sha256") != event_schema_sha256:
raise GateFailure("RUNNER_FAIL", f"consumer event schema mismatch: {path}")
expected_ids = [event["id"] for event in SPEC["event_schema"]["events"]]
if record.get("event_ids") != expected_ids:
raise GateFailure("RUNNER_FAIL", f"consumer event order/set is incomplete: {path}")
counts = record.get("captured_event_counts")
if not isinstance(counts, dict) or set(counts) != set(expected_ids):
raise GateFailure("RUNNER_FAIL", f"consumer capture counts are incomplete: {path}")
if any(type(counts[event_id]) is not int or counts[event_id] < 1 for event_id in expected_ids):
raise GateFailure("RUNNER_FAIL", f"consumer did not capture every event: {path}")
implementation_path, implementation, implementation_hash = validate_bound_path(
record, "implementation_path", "implementation_sha256",
entries, commit, maximum_bytes,
)
if any(
implementation_path.startswith(prefix)
for prefix in contract["prohibited_implementation_prefixes"]
):
raise GateFailure("RUNNER_FAIL", f"consumer implementation is nonqualifying: {path}")
capture_path, capture, capture_hash = validate_bound_path(
record, "capture_evidence_path", "capture_evidence_sha256",
entries, commit, maximum_bytes,
)
implementation_text = implementation.decode("utf-8", errors="replace").lower()
capture_text = capture.decode("utf-8", errors="replace").lower()
missing_implementation_events = [
event_id for event_id in expected_ids
if event_id not in implementation_text and event_id.replace("_", "-") not in implementation_text
]
missing_capture_events = [
event_id for event_id in expected_ids
if event_id not in capture_text and event_id.replace("_", "-") not in capture_text
]
if missing_implementation_events:
raise GateFailure(
"RUNNER_FAIL", f"consumer implementation omits {missing_implementation_events}: {path}"
)
if missing_capture_events:
raise GateFailure(
"RUNNER_FAIL", f"consumer capture omits {missing_capture_events}: {path}"
)
if transport == "dtrace-sdt":
if "dtrace" not in implementation_text or "erofs" not in implementation_text:
raise GateFailure("RUNNER_FAIL", f"consumer does not invoke EROFS DTrace: {path}")
else:
if "ktrdump" not in implementation_text:
raise GateFailure("RUNNER_FAIL", f"KTR consumer does not invoke ktrdump: {path}")
require_text(record, "ktr_justification")
return {
"manifest_path": path,
"manifest_sha256": sha256_bytes(data),
"consumer_id": record["consumer_id"],
"owner": record["owner"],
"version": record["version"],
"transport": transport,
"implementation_path": implementation_path,
"implementation_sha256": implementation_hash,
"capture_evidence_path": capture_path,
"capture_evidence_sha256": capture_hash,
"captured_event_counts": counts,
"production_use": True,
"actual_capture": True,
}
def all_conditions_pass(statuses: list[str]) -> bool:
return len(statuses) == 5 and all(status == "PASS" for status in statuses)
def false_go_controls() -> dict[str, Any]:
vectors = []
all_pass = ["PASS"] * 5
if not all_conditions_pass(all_pass):
raise GateFailure("RUNNER_FAIL", "all-PASS control did not authorize GO")
vectors.append({"name": "all-pass", "go": True})
for index in range(5):
statuses = all_pass.copy()
statuses[index] = "NOT_RUN"
if all_conditions_pass(statuses):
raise GateFailure("RUNNER_FAIL", f"false-GO control {index} was accepted")
vectors.append({"name": f"condition-{index + 1}-not-run", "go": False})
statuses = all_pass.copy()
statuses[0] = "STOP"
if all_conditions_pass(statuses):
raise GateFailure("RUNNER_FAIL", "consumer STOP control was accepted")
vectors.append({"name": "consumer-stop", "go": False})
return {"status": "PASS", "control_count": len(vectors), "vectors": vectors}
def main() -> int:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-031":
raise GateFailure("RUNNER_FAIL", "invalid P15-031 input identity")
if SPEC.get("gate") != "G08" or SPEC.get("batch") != "B38":
raise GateFailure("RUNNER_FAIL", "invalid G08/B38 input identity")
if len(SPEC.get("go_requirements", [])) != 5:
raise GateFailure("RUNNER_FAIL", "G08 P15-031 must have exactly five GO requirements")
resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure(
"INFRA_BLOCKED", f"P15-031 must replay {SPEC['required_base']}, got {resolved}"
)
for path, expected_oid in SPEC["scope_tree_oids"].items():
actual_oid = git_text("rev-parse", f"{resolved}:{path}")
if actual_oid != expected_oid:
raise GateFailure("INFRA_BLOCKED", f"frozen scope tree changed: {path}")
source_hashes = {
path: sha256_bytes(source_at(resolved, path))
for path in SPEC["source_sha256"]
}
if source_hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen G08/B38 source identity changed")
write_json(OUTPUT / "source-sha256.json", source_hashes)
addendum_paths = (
"repo-pre-15/tests/pre15/gates/P15-031.sh",
"repo-pre-15/tests/pre15/gates/P15-031-input.json",
"repo-pre-15/docs/pre15-stage0/P15-031.md",
)
addendum_hashes = {
path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths
}
write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes)
entries = parse_ls_tree(
run_bytes(
[
"git", "-C", str(ROOT), "ls-tree", "-r", "--long", "-z",
resolved, "--", "repo-pre-15", "planning/pre15",
]
)
)
prohibited = set(SPEC["prohibited_paths"])
scoped_entries = {path: value for path, value in entries.items() if path not in prohibited}
if len(scoped_entries) != SPEC["consumer_inventory"]["expected_scoped_path_count"]:
raise GateFailure("INFRA_BLOCKED", "frozen consumer scope path count changed")
reference_paths = grep_paths(
resolved, SPEC["consumer_inventory"]["reference_patterns"]
)
maximum_bytes = SPEC["consumer_inventory"]["maximum_candidate_bytes"]
references = []
reference_counts = {
"planning-proposal": 0,
"historical-evidence": 0,
"test-only": 0,
"linux-reference": 0,
"project-report-or-documentation": 0,
"possible-project-consumer": 0,
}
reasons = {
"planning-proposal": "proposal or audit text is not an existing consumer",
"historical-evidence": "historical gate output is not a deployed consumer",
"test-only": "a test, gate, or test record is explicitly nonqualifying",
"linux-reference": "Linux tracepoints are not a FreeBSD diagnostic consumer",
"project-report-or-documentation": "documentation is not an implemented consumer",
"possible-project-consumer": "requires a validated production capture manifest",
}
for path in reference_paths:
if path in prohibited:
raise GateFailure("RUNNER_FAIL", "prohibited path entered consumer scan")
entry = scoped_entries[path]
if entry["size"] > maximum_bytes:
raise GateFailure("INFRA_BLOCKED", f"consumer candidate exceeds bound: {path}")
data = source_at(resolved, path)
classification = classify_reference(path)
reference_counts[classification] += 1
references.append(
{
"path": path,
"blob_oid": entry["oid"],
"bytes": entry["size"],
"sha256": sha256_bytes(data),
"classification": classification,
"qualifies": False,
"reason": reasons[classification],
}
)
expected_counts = SPEC["consumer_inventory"]["expected_reference_counts"]
if len(reference_paths) != expected_counts["total"]:
raise GateFailure("INFRA_BLOCKED", "frozen native-tracing reference count changed")
for classification, count in reference_counts.items():
if count != expected_counts[classification]:
raise GateFailure(
"INFRA_BLOCKED", f"frozen reference classification changed: {classification}"
)
event_schema_sha256 = canonical_sha256(SPEC["event_schema"])
marker = SPEC["consumer_contract"]["schema_marker"]
manifest_paths = grep_paths(resolved, [marker], fixed=True)
consumers = []
manifest_hashes = {}
for path in manifest_paths:
entry = scoped_entries[path]
if entry["size"] > maximum_bytes:
raise GateFailure("INFRA_BLOCKED", f"consumer manifest exceeds bound: {path}")
data = source_at(resolved, path)
manifest_hashes[path] = sha256_bytes(data)
consumers.append(
validate_consumer_manifest(
path, data, scoped_entries, resolved, event_schema_sha256, maximum_bytes
)
)
if manifest_hashes != SPEC["approved_consumer_manifest_sha256"]:
raise GateFailure("INFRA_BLOCKED", "declared consumer inventory changed")
inventory = {
"schema": 1,
"base": resolved,
"scope": SPEC["consumer_inventory"]["scope"],
"scoped_path_count": len(scoped_entries),
"prohibited_path_read": False,
"reference_patterns": SPEC["consumer_inventory"]["reference_patterns"],
"reference_path_count": len(reference_paths),
"reference_classification_counts": reference_counts,
"references": references,
"consumer_manifest_marker": marker,
"consumer_manifest_count": len(consumers),
"qualified_consumer_count": len(consumers),
"qualified_consumers": consumers,
"event_schema_sha256": event_schema_sha256,
"required_event_ids": [event["id"] for event in SPEC["event_schema"]["events"]],
"nonqualifying_examples": SPEC["consumer_contract"]["nonqualifying_examples"],
}
write_json(OUTPUT / "consumer-inventory.json", inventory)
write_json(
OUTPUT / "event-schema.json",
{
"schema": 1,
"event_schema_sha256": event_schema_sha256,
"event_schema": SPEC["event_schema"],
},
)
controls = false_go_controls()
write_json(OUTPUT / "false-go-controls.json", controls)
consumer_present = bool(consumers)
conditions = [
{
"id": "concrete-existing-native-diagnostic-consumer",
"status": "PASS" if consumer_present else "STOP",
"observation": (
f"{len(consumers)} qualified production capture manifest(s)"
if consumer_present
else "zero qualified consumers; all 33 references are planning, history, tests, or documentation"
),
},
{
"id": "no-pointer-credential-or-unauthenticated-metadata-leakage",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent",
},
{
"id": "owned-temp-freebsd15-prototype-fields-counts-failure-unload",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent; no prototype was generated",
},
{
"id": "disabled-zstdio0-zstdio1-no-new-undefined-symbols",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent; no builds were run",
},
{
"id": "ten-run-hot-path-median-regression-at-most-three-percent",
"status": "NOT_RUN",
"reason": "existing-consumer prerequisite is absent; no benchmark was run",
},
]
if consumer_present:
raise GateFailure(
"RUNNER_FAIL",
"a consumer is now approved; add and run the native privacy/prototype/build/benchmark stages before GO",
)
if all_conditions_pass([condition["status"] for condition in conditions]):
raise GateFailure("RUNNER_FAIL", "false-GO guard accepted incomplete conditions")
result = {
"schema": 1,
"gate": "G08",
"candidate": "P15-031",
"batch": "B38",
"status": "STOP",
"reason": "no concrete existing project diagnostic consumer captures the predeclared native trace schema",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"start_utc": START_UTC,
"end_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"deadline_seconds": DEADLINE,
"expected_exit_code": 22,
"conditions": conditions,
"false_go_guard": controls,
"consumer_inventory": "consumer-inventory.json",
"event_schema": "event-schema.json",
"gate_metrics": {
"scoped_paths": len(scoped_entries),
"native_tracing_reference_paths": len(reference_paths),
"qualified_consumers": len(consumers),
"predeclared_events": len(SPEC["event_schema"]["events"]),
"prototype_runs": 0,
"builds": 0,
"benchmark_runs": 0,
"qemu_runs": 0,
},
"b38": "STOP-NO-SOURCE",
"prototype": "NOT_RUN",
"builds": "NOT_RUN",
"microbenchmark": "NOT_RUN",
"qemu": "NOT_RUN",
"tc178": "NOT_RUN",
"full_feature_suite": "NOT_RUN",
"smoke_suite": "NOT_RUN",
"production_source_changed": False,
"protected_pid_addressed": False,
"protected_port_addressed": False,
"base_image_addressed_or_hashed": False,
}
write_json(OUTPUT / "result.json", result)
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
return 22
try:
raise SystemExit(main())
except GateFailure as error:
failure = {
"schema": 1,
"gate": "G08",
"candidate": "P15-031",
"batch": "B38",
"status": error.status,
"reason": error.reason,
"requested_base": REQUESTED_BASE,
"start_utc": START_UTC,
"end_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"deadline_seconds": DEADLINE,
"b38": "NOT_RUN",
"prototype": "NOT_RUN",
"builds": "NOT_RUN",
"microbenchmark": "NOT_RUN",
"qemu": "NOT_RUN",
"production_source_changed": False,
}
write_json(OUTPUT / "result.json", failure)
print(json.dumps(failure, ensure_ascii=True, sort_keys=True))
raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20))
PY
gate_rc=$?
set -e
cleanup_status=PASS
if ! rm -rf -- "$work"; then
cleanup_status=FAIL
gate_rc=20
fi
trap - EXIT HUP INT TERM
printf 'owned temporary path removed: %s\ncleanup=%s\n' \
"$work" "$cleanup_status" >"$output/cleanup.log"
python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
output = Path(sys.argv[1])
exit_code = int(sys.argv[2])
cleanup = sys.argv[3]
if cleanup != "PASS":
status = "RUNNER_FAIL"
origin = "runner"
elif exit_code == 22:
status = "STOP"
origin = "gate"
elif exit_code in (124, 137):
status = "INFRA_BLOCKED"
origin = "infrastructure"
else:
status = "RUNNER_FAIL" if exit_code == 20 else "INFRA_BLOCKED"
origin = "runner" if exit_code == 20 else "infrastructure"
(output / "attempt.json").write_text(
json.dumps(
{
"schema": 1,
"exit_code": exit_code,
"status": status,
"failure_origin": origin,
"cleanup": cleanup,
"target_marker": "reached" if status == "STOP" else "not_reached",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
lines = []
for path in sorted(output.iterdir(), key=lambda item: item.name):
if path.is_file() and path.name != "SHA256SUMS":
digest = hashlib.sha256(path.read_bytes()).hexdigest()
lines.append(f"{digest} {path.name}")
(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
PY
printf 'P15-031 gate exit=%s cleanup=%s output=%s\n' \
"$gate_rc" "$cleanup_status" "$output"
exit "$gate_rc"
tests/pre15/gates/P15-032-input.json
+251
View File
@@ -0,0 +1,251 @@
{
"schema": "pre15-gate-v1",
"unit": "G09/P15-032",
"batch": "B39",
"frozen_base": "bd5a09054e5cf89efd4db82aadb051f20b06ebf7",
"freebsd_source": {
"outer_git_head": "106727738dcfb6c001b46f25363b91cece970085",
"identity": "per-contract-file-sha256"
},
"timeouts_seconds": {
"whole_gate": 120,
"command": 10,
"compile": 20,
"model": 10
},
"syntax_policy": {
"vnode": "vnode:/absolute/path",
"geom": "geom:/provider-or-device-path",
"legacy_geom": "/provider-or-device-path",
"ambiguous_regular_path_is_vnode": false,
"multidevice": "all sources in one mount must use the same backend kind"
},
"typed_errno": {
"cycle": "EDEADLK",
"unsupported_dependency_graph": "EOPNOTSUPP",
"changed_or_short_source": "EIO",
"dead_forced_source": "ENXIO"
},
"required_events": [
"parse_tagged_source",
"namei_lockleaf",
"require_vreg",
"resolve_vop_getlowvnode",
"check_self_identity",
"check_namespace_ancestors",
"trace_storage_ancestors",
"vop_access",
"vn_open_vnode_fread",
"crhold_mount_cred",
"vop_set_text",
"register_upper_mount",
"snapshot_size",
"first_io",
"io_complete",
"mark_closing",
"drain_inflight",
"vflush",
"unregister_upper_mount",
"vop_unset_text",
"vn_close",
"crfree_mount_cred",
"release_last_reference"
],
"required_invariants": [
"explicit_backend_tag",
"held_vnode_identity",
"held_credential",
"held_credential_used_for_read",
"kernel_space_read_buffer",
"source_range_lock",
"source_shared_vnode_lock",
"write_exclusion",
"size_snapshot_and_short_read_error",
"self_check_before_io",
"namespace_ancestor_check_before_io",
"storage_ancestor_check_before_io",
"upper_mount_registration",
"inflight_drain_before_release",
"same_backend_multidevice",
"geom_and_vnode_paths_disjoint",
"fault_injection_tc179_constructible",
"fault_injection_tc184_constructible"
],
"lock_edges": [
["vfs_mount_busy", "erofs_vnode_lock"],
["erofs_vnode_lock", "source_range_lock"],
["source_range_lock", "source_vnode_lock"],
["source_vnode_lock", "source_vop_read"],
["erofs_vm_object_lock", "erofs_vm_object_unlock_before_vop_read"],
["erofs_vm_object_unlock_before_vop_read", "erofs_vnode_lock"],
["geom_topology_lock", "geom_open_close_only"],
["mark_closing", "drain_inflight"],
["drain_inflight", "vflush"],
["vflush", "unregister_upper_mount"],
["unregister_upper_mount", "source_vnode_close"],
["source_vnode_close", "credential_release"],
["credential_release", "last_reference_release"]
],
"false_go_controls": [
"explicit_backend_tag",
"held_vnode_identity",
"held_credential",
"held_credential_used_for_read",
"kernel_space_read_buffer",
"source_range_lock",
"source_shared_vnode_lock",
"write_exclusion",
"size_snapshot_and_short_read_error",
"self_check_before_io",
"namespace_ancestor_check_before_io",
"upper_mount_registration",
"inflight_drain_before_release",
"same_backend_multidevice",
"geom_and_vnode_paths_disjoint",
"fault_injection_tc179_constructible",
"fault_injection_tc184_constructible"
],
"contracts": [
{
"root": "freebsd",
"path": "sys/kern/vnode_if.src",
"sha256": "5de87ff115f543fd89f762c3d356b6799f30757f8a34fd840370ce7fff53b90b",
"must_contain": ["%% read\t\tvp\tL L L", "%% getlowvnode vp\t= = =", "%% getpages\tvp\tL L L", "%% set_text\tvp\t= = ="]
},
{
"root": "freebsd",
"path": "sys/kern/vfs_vnops.c",
"sha256": "3bc9735e2a190fc55134a803f5fc8cb6f020b0b1ce925261774db32789bd7f08",
"must_contain": ["vn_rdwr(enum uio_rw rw", "vn_rangelock_rlock(vp", "vn_lock(vp, lock_flags | LK_RETRY);", "VOP_READ(vp, &auio, ioflg, cred);"]
},
{
"root": "freebsd",
"path": "sys/kern/vfs_default.c",
"sha256": "4286a9bf038c32021f5c116d8ab713b09a5cfa43cf52999bcbf2c0018e7aa177",
"must_contain": ["vop_stdset_text(struct vop_set_text_args *ap)", "return (ETXTBSY);", "vop_stdunset_text(struct vop_unset_text_args *ap)", "vop_stdadd_writecount_impl", "vop_stdgetlowvnode(struct vop_getlowvnode_args *ap)"]
},
{
"root": "freebsd",
"path": "sys/kern/vfs_mount.c",
"sha256": "eecd22aaf63eda940798af02d2d3068a6ed3dc2d90b74a936190a7e56db0a79a",
"must_contain": ["vfs_register_upper_from_vp(struct vnode *vp", "ensure that it cannot be unmounted", "KASSERT(ump != mp", "vfs_unregister_upper(struct mount *mp", "registered upper with pending unmount"]
},
{
"root": "freebsd",
"path": "sys/sys/mount.h",
"sha256": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738",
"must_contain": ["struct mount_upper_node", "struct vnode\t*mnt_vnodecovered", "void *\t\tmnt_data"],
"must_not_contain": ["struct g_provider", "struct g_consumer"]
},
{
"root": "freebsd",
"path": "sys/vm/vnode_pager.c",
"sha256": "6e2bd8a57bb1c027f303db7bf0e2351fa67af5ca53d6b2b826fa1f12f2c5154e",
"must_contain": ["vnode_pager_input_old(vm_object_t object", "VM_OBJECT_WUNLOCK(object);", "auio.uio_segflg = UIO_SYSSPACE;", "VOP_READ(vp, &auio, 0, curthread->td_ucred);"]
},
{
"root": "freebsd",
"path": "sys/fs/tarfs/tarfs_vfsops.c",
"sha256": "b4c1a5c7f8870c816917706ed0e4dafd790ca5374690833d8daa321aafb696c1",
"must_contain": ["vn_open_vnode", "vn_close"]
},
{
"root": "freebsd",
"path": "sys/fs/tarfs/tarfs_io.c",
"sha256": "0c255ba14df3ff7dcec346dee99c9ddbee376e3483370bb7257dffe2f529fc6c",
"must_contain": ["vn_rangelock_rlock", "VOP_READ"]
},
{
"root": "freebsd",
"path": "sys/fs/deadfs/dead_vnops.c",
"sha256": "1d6864e8cc035ad109ac60c3dab67f31b9c3ad70bee9f9e585b6be10a5657b73",
"must_contain": ["dead_read", "return (ENXIO);"]
},
{
"root": "freebsd",
"path": "sys/fs/nullfs/null_vnops.c",
"sha256": "b10c2138b649343e6c49a9416b6f4f3a0894388059122eefa190430e13695d6c",
"must_contain": ["VOP_GETLOWVNODE"]
},
{
"root": "freebsd",
"path": "sys/fs/nullfs/null_vfsops.c",
"sha256": "f5f8803458522d9cecb1e024bdd599630dde02bc1e0578445204602b4db633bd",
"must_contain": ["vfs_register_upper_from_vp", "VFCF_LOOPBACK"]
},
{
"root": "freebsd",
"path": "sys/fs/unionfs/union_vnops.c",
"sha256": "ce9a41531731170a340e39bfe6afd29f4740d8b705abb6a018176c7a1e159aff",
"must_contain": ["VOP_GETLOWVNODE"]
},
{
"root": "freebsd",
"path": "sys/fs/unionfs/union_vfsops.c",
"sha256": "c181c0134887a0d8697b739dbbf54b7c4299f337bbd4a3e6e1d62a1b6bfb1b51",
"must_contain": ["vfs_register_upper_from_vp", "VFCF_LOOPBACK"]
},
{
"root": "freebsd",
"path": "sys/dev/md/md.c",
"sha256": "a19d34da3d391747fbe2e8bee9fbb02a361202351a0b8bfbec1abc7b5a3f2d4b",
"must_contain": ["struct md_s {", "struct vnode *vnode;", "gp->softc = sc;", "g_new_providerf(gp, \"md%d\"", "VOP_READ(vp, &auio, 0, sc->cred);", "sbuf_printf(sb, \" file %s\", mp->s_vnode.file);"]
},
{
"root": "freebsd",
"path": "sys/geom/geom.h",
"sha256": "22feba6781dcc4e0aae0dea75b83dcae16d077be67853331da35d06edffb0f39",
"must_contain": ["struct g_geom", "void\t\t\t*softc;"],
"must_not_contain": ["struct vnode"]
},
{
"root": "repo_base",
"path": "repo-pre-15/src/internal.h",
"sha256": "3fe8d6ca819f0a1c472cc4b173da80027f957b7f7c25e51ef7fd85b60480c98a",
"must_contain": ["struct erofs_device_info", "struct g_consumer *cp;"]
},
{
"root": "repo_base",
"path": "repo-pre-15/src/super.c",
"sha256": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2",
"must_contain": ["erofs_open_device", "g_vfs_open", "erofs_read_superblock"]
},
{
"root": "repo_base",
"path": "repo-pre-15/src/data.c",
"sha256": "a511146d2d1ff56bc88bae6c550cfdc49a8e978c5c2c88d735de92adaaa836b6",
"must_contain": ["erofs_bread_device", "bread(dif->devvp"]
},
{
"root": "repo_base",
"path": "repo-pre-15/src/erofs_vnops.c",
"sha256": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64",
"must_contain": ["vnode_pager_local_getpages", "erofs_read_file"]
},
{
"root": "linux_reference",
"path": "src-linux/fileio.c",
"sha256": "1f00f6cee0060072e8300a2e4f7ca8cd2dc566f129b99d2ed34288b19210e59c",
"must_contain": ["erofs_fileio"]
},
{
"root": "linux_reference",
"path": "src-linux/super.c",
"sha256": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9",
"must_contain": ["s_stack_depth"]
}
],
"blocking_condition": {
"id": "FREEBSD-GEOM-VNODE-ANCESTRY-NO-PUBLIC-IDENTITY-EDGE",
"classification": "STOP",
"summary": "The transitive storage ancestry of a regular source vnode cannot be discovered generically before first I/O.",
"counterexample": [
"An accepted regular source vnode is on a filesystem consuming an md(4) GEOM provider.",
"The md provider services BIO_READ by VOP_READ on md_s.s_vnode.vnode.",
"That vnode identity is private md.c softc state; GEOM exposes only void *softc and dumpconf exposes a pathname.",
"VOP_GETLOWVNODE and vfs_register_upper_from_vp cover VFS alias/lower mounts only, not GEOM-to-vnode backing edges.",
"A pathname cannot close identity across rename/replace, so it cannot implement the required ancestor identity comparison.",
"Following gp->softc by duplicating private md_s is an undocumented class-specific assumption and still does not cover other GEOM or filesystem-private file-backed storage."
]
}
}
tests/pre15/gates/P15-032.sh
Executable
+380
View File
@@ -0,0 +1,380 @@
#!/bin/sh
set -eu
BASE=bd5a09054e5cf89efd4db82aadb051f20b06ebf7
FREEBSD_SRC=/work/build/freebsd-src
OUTPUT=
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../../.." && pwd)
INPUT=$SCRIPT_DIR/P15-032-input.json
usage()
{
printf '%s\n' "usage: $0 --output ABSOLUTE_PATH [--base SHA] [--freebsd-src PATH]" >&2
exit 20
}
while [ "$#" -gt 0 ]; do
case "$1" in
--output)
[ "$#" -ge 2 ] || usage
OUTPUT=$2
shift 2
;;
--base)
[ "$#" -ge 2 ] || usage
BASE=$2
shift 2
;;
--freebsd-src)
[ "$#" -ge 2 ] || usage
FREEBSD_SRC=$2
shift 2
;;
*)
usage
;;
esac
done
[ -n "$OUTPUT" ] || usage
case "$OUTPUT" in
/*) ;;
*) printf '%s\n' "P15-032: --output must be absolute" >&2; exit 20 ;;
esac
if [ -d "$OUTPUT" ] && [ -n "$(find "$OUTPUT" -mindepth 1 -maxdepth 1 -print -quit)" ]; then
printf '%s\n' "P15-032: --output must be empty" >&2
exit 20
fi
mkdir -p "$OUTPUT"
TMP=$(mktemp -d "${TMPDIR:-/tmp}/P15-032.XXXXXX")
cleanup()
{
rm -rf "$TMP"
}
trap cleanup EXIT HUP INT TERM
printf 'argv=%s\n' "$0 --output $OUTPUT --base $BASE --freebsd-src $FREEBSD_SRC" >"$OUTPUT/runner.argv"
printf 'whole_gate_deadline_seconds=120\n' >"$OUTPUT/deadlines.txt"
printf 'command_deadline_seconds=10\n' >>"$OUTPUT/deadlines.txt"
printf 'compile_deadline_seconds=20\n' >>"$OUTPUT/deadlines.txt"
printf 'model_deadline_seconds=10\n' >>"$OUTPUT/deadlines.txt"
set +e
/usr/bin/timeout 120s python3 - "$ROOT" "$FREEBSD_SRC" "$INPUT" "$OUTPUT" "$TMP" "$BASE" >"$OUTPUT/runner.stdout" 2>"$OUTPUT/runner.stderr" <<'PY'
import datetime
import hashlib
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys
root = Path(sys.argv[1]).resolve()
freebsd = Path(sys.argv[2]).resolve()
input_path = Path(sys.argv[3]).resolve()
output = Path(sys.argv[4]).resolve()
tmp = Path(sys.argv[5]).resolve()
base = sys.argv[6]
cfg = json.loads(input_path.read_text(encoding="utf-8"))
command_timeout = cfg["timeouts_seconds"]["command"]
commands = []
def run(argv, cwd, timeout=command_timeout, check=True):
started = datetime.datetime.now(datetime.timezone.utc)
cp = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=timeout, check=False)
commands.append({
"argv": argv,
"cwd": str(cwd),
"timeout_seconds": timeout,
"started_utc": started.isoformat(),
"exit": cp.returncode,
"stdout": cp.stdout.decode("utf-8", "replace"),
"stderr": cp.stderr.decode("utf-8", "replace"),
})
if check and cp.returncode != 0:
raise RuntimeError(f"command failed ({cp.returncode}): {shlex.join(argv)}")
return cp
def sha256(data):
return hashlib.sha256(data).hexdigest()
def bounded_read(path):
data = path.read_bytes()
if len(data) > 4 * 1024 * 1024:
raise RuntimeError(f"refusing oversized contract file: {path}")
return data
def base_blob(path):
return run(["git", "show", f"{base}:{path}"], root).stdout
def line_of(text, marker):
return text[:text.index(marker)].count("\n") + 1
started = datetime.datetime.now(datetime.timezone.utc)
if base != cfg["frozen_base"]:
raise RuntimeError("requested base does not match frozen base")
head = run(["git", "rev-parse", "HEAD"], root).stdout.decode().strip()
remote = run(["git", "rev-parse", "xdm/main"], root).stdout.decode().strip()
if head != base or remote != base:
raise RuntimeError(f"base mismatch: HEAD={head} xdm/main={remote} expected={base}")
allowed = {
"repo-pre-15/tests/pre15/gates/P15-032.sh",
"repo-pre-15/tests/pre15/gates/P15-032-input.json",
"repo-pre-15/docs/pre15-stage0/P15-032.md",
}
status_raw = run(["git", "status", "--porcelain=v1", "-z"], root).stdout
entries = [entry for entry in status_raw.decode().split("\0") if entry]
paths = set()
for entry in entries:
path = entry[3:]
if " -> " in path:
path = path.split(" -> ", 1)[1]
paths.add(path)
if not paths.issubset(allowed):
raise RuntimeError(f"out-of-scope worktree paths: {sorted(paths - allowed)}")
outer_head = run(["git", "rev-parse", "HEAD"], freebsd).stdout.decode().strip()
if outer_head != cfg["freebsd_source"]["outer_git_head"]:
raise RuntimeError(f"FreeBSD outer source identity mismatch: {outer_head}")
contracts = []
source_hashes = {}
for contract in cfg["contracts"]:
kind = contract["root"]
rel = contract["path"]
if kind == "freebsd":
data = bounded_read(freebsd / rel)
elif kind == "repo_base":
data = base_blob(rel)
elif kind == "linux_reference":
data = base_blob(rel)
else:
raise RuntimeError(f"unknown contract root: {kind}")
actual = sha256(data)
source_hashes[f"{kind}:{rel}"] = actual
if actual != contract["sha256"]:
raise RuntimeError(f"hash mismatch for {kind}:{rel}: {actual}")
text = data.decode("utf-8", "replace")
matches = []
for marker in contract.get("must_contain", []):
if marker not in text:
raise RuntimeError(f"missing contract marker in {rel}: {marker!r}")
matches.append({"marker": marker, "line": line_of(text, marker)})
forbidden = []
for marker in contract.get("must_not_contain", []):
present = marker in text
forbidden.append({"marker": marker, "present": present})
if present:
raise RuntimeError(f"forbidden bridge marker in {rel}: {marker!r}")
contracts.append({"root": kind, "path": rel, "sha256": actual,
"matches": matches, "negative_matches": forbidden})
events = cfg["required_events"]
event_index = {name: index for index, name in enumerate(events)}
required_order = [
("check_self_identity", "first_io"),
("check_namespace_ancestors", "first_io"),
("trace_storage_ancestors", "first_io"),
("crhold_mount_cred", "first_io"),
("vop_set_text", "first_io"),
("register_upper_mount", "first_io"),
("io_complete", "drain_inflight"),
("drain_inflight", "unregister_upper_mount"),
("unregister_upper_mount", "vn_close"),
("vn_close", "crfree_mount_cred"),
("crfree_mount_cred", "release_last_reference"),
]
for before, after in required_order:
if event_index[before] >= event_index[after]:
raise RuntimeError(f"invalid lifecycle order: {before} !< {after}")
nodes = set()
adj = {}
for before, after in cfg["lock_edges"]:
nodes.update((before, after))
adj.setdefault(before, []).append(after)
visiting = set()
visited = set()
def visit(node):
if node in visiting:
raise RuntimeError(f"lock graph cycle at {node}")
if node in visited:
return
visiting.add(node)
for child in adj.get(node, []):
visit(child)
visiting.remove(node)
visited.add(node)
for node in sorted(nodes):
visit(node)
dot = ["digraph P15_032 {", " rankdir=LR;"]
for before, after in cfg["lock_edges"]:
dot.append(f' "{before}" -> "{after}";')
dot.append("}")
(output / "lockgraph.dot").write_text("\n".join(dot) + "\n", encoding="utf-8")
invariants = cfg["required_invariants"]
closeable = [name for name in invariants if name != "storage_ancestor_check_before_io"]
controls = cfg["false_go_controls"]
if set(controls) != set(closeable):
raise RuntimeError("false-GO controls do not exactly cover closeable invariants")
enum_lines = [f" INV_{name.upper()} = 1ULL << {index}," for index, name in enumerate(closeable)]
required_mask = " |\n ".join(f"INV_{name.upper()}" for name in closeable)
control_rows = ",\n".join(
f' {{"{name}", INV_{name.upper()}}}' for name in controls
)
model = f'''#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
enum invariant {{
{os.linesep.join(enum_lines)}
}};
struct scenario {{
uint64_t present;
int storage_ancestry_identity_api;
}};
struct control {{
const char *name;
uint64_t bit;
}};
static const uint64_t required =
{required_mask};
static int
visible_only_oracle(const struct scenario *scenario)
{{
return ((scenario->present & required) == required ? 0 : EINVAL);
}}
static int
rigorous_oracle(const struct scenario *scenario)
{{
int error;
error = visible_only_oracle(scenario);
if (error != 0)
return (error);
if (!scenario->storage_ancestry_identity_api)
return (EOPNOTSUPP);
return (0);
}}
int
main(void)
{{
static const struct control controls[] = {{
{control_rows}
}};
struct scenario candidate = {{ required, 0 }};
struct scenario mutation;
size_t index;
if (visible_only_oracle(&candidate) != 0)
return (1);
if (rigorous_oracle(&candidate) != EOPNOTSUPP)
return (2);
puts("hidden-md-edge: visible-only=GO rigorous=STOP:EOPNOTSUPP");
for (index = 0; index < sizeof(controls) / sizeof(controls[0]); index++) {{
mutation = candidate;
mutation.present &= ~controls[index].bit;
if (visible_only_oracle(&mutation) == 0 || rigorous_oracle(&mutation) == 0)
return (3);
printf("false-go-control:%s=REJECTED\\n", controls[index].name);
}}
puts("lifecycle-model=PASS");
return (0);
}}
'''
model_path = tmp / "P15-032-model.c"
binary_path = tmp / "P15-032-model"
model_path.write_text(model, encoding="utf-8")
compile_cp = run(["/usr/bin/timeout", f"{cfg['timeouts_seconds']['compile']}s",
"/usr/bin/cc", "-std=c11", "-Wall", "-Wextra", "-Werror",
"-O2", "-o", str(binary_path), str(model_path)], tmp,
timeout=cfg["timeouts_seconds"]["compile"] + 2)
model_cp = run(["/usr/bin/timeout", f"{cfg['timeouts_seconds']['model']}s",
str(binary_path)], tmp,
timeout=cfg["timeouts_seconds"]["model"] + 2)
(output / "owned-temp-model.c").write_bytes(model_path.read_bytes())
(output / "owned-temp-model.stdout").write_bytes(model_cp.stdout)
(output / "owned-temp-model.stderr").write_bytes(model_cp.stderr)
false_go = {
"controls": [{"removed_invariant": name, "outcome": "REJECTED"} for name in controls],
"adversarial_hidden_edge": {
"topology": "regular vnode -> filesystem -> md GEOM provider -> private backing vnode",
"visible_only_oracle": "GO",
"rigorous_oracle": "STOP:EOPNOTSUPP",
},
}
(output / "false-go-controls.json").write_text(json.dumps(false_go, indent=2) + "\n")
(output / "contracts.json").write_text(json.dumps(contracts, indent=2) + "\n")
(output / "source-hashes.json").write_text(json.dumps(source_hashes, indent=2, sort_keys=True) + "\n")
(output / "commands.json").write_text(json.dumps(commands, indent=2) + "\n")
state = {
"events": events,
"required_order": required_order,
"local_lifecycle": "PASS",
"storage_ancestor_identity": "UNAVAILABLE",
"typed_errno_if_detectable": cfg["typed_errno"]["cycle"],
"typed_errno_for_unsupported_graph": cfg["typed_errno"]["unsupported_dependency_graph"],
}
(output / "state-model.json").write_text(json.dumps(state, indent=2) + "\n")
result = {
"unit": cfg["unit"],
"batch": cfg["batch"],
"classification": "GATE_STOP",
"decision": "STOP",
"exit": 10,
"base": base,
"head": head,
"xdm_main": remote,
"freebsd_git_head": outer_head,
"freebsd_identity": cfg["freebsd_source"]["identity"],
"started_utc": started.isoformat(),
"finished_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"closeable_invariants": closeable,
"blocking_condition": cfg["blocking_condition"],
"source_changes_permitted": False,
"tests_run": ["contract extraction", "owned-temp C lifecycle model", "adversarial false-GO controls"],
"tests_omitted": ["D", "H", "K", "Q", "TC006", "TC179", "TC184", "full feature", "smoke"],
}
(output / "result.json").write_text(json.dumps(result, indent=2) + "\n")
print("P15-032 STOP: no public identity-preserving VFS/GEOM-to-backing-vnode ancestry API")
sys.exit(10)
PY
RC=$?
set -e
printf 'exit=%s\n' "$RC" >"$OUTPUT/runner.exit"
case "$RC" in
10)
printf '%s\n' GATE_STOP >"$OUTPUT/classification"
;;
124)
printf '%s\n' RUNNER_TIMEOUT >"$OUTPUT/classification"
;;
0)
printf '%s\n' GATE_GO >"$OUTPUT/classification"
;;
*)
printf '%s\n' RUNNER_FAILURE >"$OUTPUT/classification"
;;
esac
(cd "$OUTPUT" && find . -maxdepth 1 -type f ! -name manifest.sha256 -print | LC_ALL=C sort | xargs sha256sum) >"$OUTPUT/manifest.sha256"
exit "$RC"
tests/pre15/gates/P15-038-input.json
+58
View File
@@ -0,0 +1,58 @@
{
"benchmark": {
"extent_bytes": 262144,
"minimum_improved_codecs": 2,
"minimum_latency_improvement_percent": 10.0,
"random_reads_per_sample": 1024,
"read_bytes": 4096,
"samples": 5,
"seed": 2539541505
},
"budget": {
"global_bytes": 524288,
"minimum_decode_work_bytes": 131072,
"per_mount_bytes": 262144
},
"candidate": "P15-038",
"gate": "G05",
"libraries": {
"liblz4": "1.10.0",
"liblzma": "5.8.1",
"libzstd": "1.5.7",
"zlib": "1.3.1"
},
"lock_order": [
"mount-cache",
"global-budget"
],
"protected": {
"pid": 26318,
"port": 9222
},
"required_base": "5d6755649a369498a9b257bb2d1d1e3496d5135e",
"schema": 1,
"scope": "host-codec-cost-oracle-not-guest-vnode-performance",
"source_sha256": {
"repo-pre-15/src/compress.h": "d4177f5d606489d8c63bb7f1dcaf18bc92dd5d374395ccfa76060b272549f5d2",
"repo-pre-15/src/decompressor.c": "ccaa934f837ecbf3b3b678dbddf35d29dd7e3b4e80d1a7ab78b18a07e555419f",
"repo-pre-15/src/decompressor_deflate.c": "4a018ce06dc83fcb305130c262d5b59b4d133f698be40a957bebca2cd6a92518",
"repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98",
"repo-pre-15/src/decompressor_lzma.c": "9935be2f4d5829240052f151b916203c67e830e8b2b7b51c452262a8f92f6157",
"repo-pre-15/src/decompressor_zstd.c": "c5f19c9ea6d8238bcf30b8b3228b80e77cf2073df1d07e4fac5f5401bef413bb",
"repo-pre-15/src/internal.h": "d33b80c57846437dc0410086c552cb308b8e1a6ce517b5b2ce19779bd59e8848",
"repo-pre-15/src/zdata.c": "2c4d624c173ac9fe0cb1e72bb968a7994d0cfd66e4fcd0db04dea10e06ba0fee",
"repo-pre-15/tests/pre15/cases/B32-cache-state.sh": "e14453fc628a615acf701f96ec575c884453fe1a62878badbfcefff33656b92d",
"repo-pre-15/tests/pre15/fixtures/B32-cache-oracle.c": "2a83b84c560b67cf4f7583e886e1f0dbe792e58bc4b0a8e59e2a07c17ec8f866",
"repo-pre-15/tests/pre15/fixtures/B32-qemu-run.sh": "bb1d493a8dc6536751377f3cd72c4012875a41320f570f5259340220b226ed2e"
},
"states": [
"EMPTY",
"INFLIGHT",
"READY",
"FAILED"
],
"upstream_batches": {
"b32_correctness": "c6a502184da6e973b1f0fa2ff8c0290c041e7092",
"b32_qemu_timeout_fix": "c4344fc842a180720a56b585191490395044603a"
}
}
tests/pre15/gates/P15-038.sh
Executable
+1109
View File
File diff suppressed because it is too large Load Diff
tests/pre15/gates/P15-045-input.json
+71
View File
@@ -0,0 +1,71 @@
{
"approved_demand_manifest_sha256": {},
"batch": "B35",
"candidate": "P15-045",
"demand_manifest_contract": {
"accepted_formats": [
"JSON object",
"RFC822-style text headers"
],
"demand_terms": [
"deployment",
"deploy",
"support",
"manifest",
"policy",
"default",
"required",
"must",
"out of the box",
"out_of_box",
"部署",
"支持",
"默认",
"开箱",
"清单"
],
"maximum_text_candidate_bytes": 2097152,
"required_semantics": [
"document identifies itself as a support/deployment manifest",
"manifest has a concrete version",
"manifest names nonempty deployment/support targets",
"manifest explicitly requires Zstd enabled by default/out of the box"
],
"scope": "planning/pre15/evidence at required_base"
},
"evidence_tracked_path_count": 2560,
"evidence_tree_oid": "3ecce7a124d153638853ab5d897126c769125309",
"gate": "G07",
"probe_order": [
"versioned-deployment-demand",
"disabled-runtime-exact-EOPNOTSUPP",
"enabled-real-reproducible-Zstd-EROFS-read",
"FreeBSD-kernel-ZSTDIO-symbol-capability-model",
"KLD-size-delta-at-most-64KiB-and-10-percent"
],
"prohibited_paths": [
"planning/pre15/introduction.md"
],
"protected": {
"pid": 26318,
"port": 9222
},
"required_base": "13974efc00c31d8c13c8dccb7c65a82adafcbff6",
"schema": 1,
"size_thresholds": {
"maximum_bytes": 65536,
"maximum_percent": 10.0,
"require_both": true
},
"source_sha256": {
"planning/pre15/20-final-candidate-ledger.md": "2ba36e8f107cfe5479c4d2bb913519b27e0eb97aa988ecc08e3a7633523620d0",
"planning/pre15/30-stage0-gates.md": "34708bacf4a0668dfd834b3639900ca23b28b9833a387ac6e2c0399c9316cdb8",
"planning/pre15/40-execution-batches.md": "1d1ffdf8fec799aeec063bb1dc4a0fda9703300d414eabef8905522b2c18db20",
"planning/pre15/90-honesty-and-evidence.md": "69f96908f5ab436a2f5359548e4e6f5dc2942fcf00148d999d513558f1c59c2d",
"repo-pre-15/README.md": "2d3e138f5af20ce02aa562286cb42df8fbd00fdad0ae424e426e0fd8a7f72472",
"repo-pre-15/docs/erofs.5": "dea6a9eddbccbb7b8779b6b4e43772edeea9f0e51696c1b8fd09e4d8e69b52f5",
"repo-pre-15/docs/features.md": "6d875eb582ef9f00391f90ee5b4b8aec0429f24f0ae3c7b27e788ab1e2c48f2e",
"repo-pre-15/src/Makefile": "b722f7ec658e0ada0640c80a7b527921bd96cdf0e93e9623cece29766f2fc9c8",
"repo-pre-15/src/decompressor_zstd.c": "c5f19c9ea6d8238bcf30b8b3228b80e77cf2073df1d07e4fac5f5401bef413bb"
}
}
tests/pre15/gates/P15-045.sh
Executable
+564
View File
@@ -0,0 +1,564 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-045-input.json
base=
output=
deadline=90
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 20; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 20; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 20
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 20; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 20; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 20; }
for tool in git mktemp python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 21
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 20; }
mkdir -p "$output"
work=$(mktemp -d "${TMPDIR:-/tmp}/P15-045.XXXXXX")
cleanup_trap()
{
rm -rf -- "$work"
}
trap cleanup_trap EXIT HUP INT TERM
python3 -B - "$output/command-argv.json" "$0" --base "$base" --output "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(sys.argv[2:], ensure_ascii=True, separators=(",", ":")) + "\n",
encoding="ascii",
)
PY
python3 -B - "$output/ownership.json" "$work" "$output" <<'PY'
import json
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(
json.dumps(
{
"owned_temporary_paths": [sys.argv[2]],
"persistent_output": sys.argv[3],
"owned_processes": [],
"owned_ports": [],
"qemu": "NOT_RUN",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
PY
set +e
timeout -k 5 "$deadline" python3 -B - \
"$root" "$input" "$base" "$output" "$deadline" <<'PY' \
>"$output/stdout.log" 2>"$output/stderr.log"
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
DEADLINE = int(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="utf-8"))
class GateFailure(RuntimeError):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def write_json(path: Path, value: Any) -> None:
path.write_text(
json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
encoding="ascii",
)
def run_bytes(argv: list[str], allowed: set[int] | None = None) -> bytes:
try:
completed = subprocess.run(
argv,
cwd=ROOT,
check=False,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=30,
)
except subprocess.TimeoutExpired as error:
raise GateFailure(
"INFRA_BLOCKED", f"command timed out: {' '.join(argv)}"
) from error
accepted = {0} if allowed is None else allowed
if completed.returncode not in accepted:
detail = completed.stderr.decode("utf-8", errors="replace").strip()
raise GateFailure(
"RUNNER_FAIL",
f"command failed ({completed.returncode}): {' '.join(argv)}: {detail}",
)
return completed.stdout
def git_text(*args: str, allowed: set[int] | None = None) -> str:
return run_bytes(["git", "-C", str(ROOT), *args], allowed).decode(
"utf-8", errors="strict"
).strip()
def source_at(commit: str, relative: str) -> bytes:
return run_bytes(["git", "-C", str(ROOT), "show", f"{commit}:{relative}"])
def truthy(value: Any) -> bool:
if value is True:
return True
if isinstance(value, str):
return value.strip().lower() in {"true", "yes", "required", "enabled", "1"}
if isinstance(value, int):
return value == 1
return False
def flattened(value: Any, prefix: str = "") -> list[tuple[str, Any]]:
records: list[tuple[str, Any]] = []
if isinstance(value, dict):
for key, child in value.items():
path = f"{prefix}.{key}" if prefix else str(key)
records.append((path.lower().replace("-", "_"), child))
records.extend(flattened(child, path))
elif isinstance(value, list):
for index, child in enumerate(value):
records.extend(flattened(child, f"{prefix}[{index}]"))
return records
def json_demand_witness(value: Any) -> dict[str, Any] | None:
if not isinstance(value, dict):
return None
records = flattened(value)
kind = any(
any(token in key for token in ("manifest_type", "manifest_kind", "document_type"))
and isinstance(item, str)
and any(token in item.lower() for token in ("support", "deploy"))
for key, item in records
)
version = any(
"version" in key and isinstance(item, (str, int)) and str(item).strip()
for key, item in records
)
targets = any(
any(token in key for token in ("deployment_target", "support_target", "product_target"))
and ((isinstance(item, str) and item.strip()) or (isinstance(item, list) and item))
for key, item in records
)
required = any(
"zstd" in key
and any(token in key for token in ("default", "required", "out_of_box", "outofthebox"))
and truthy(item)
for key, item in records
)
if kind and version and targets and required:
return {
"format": "json",
"manifest_kind": True,
"version": True,
"deployment_targets": True,
"zstd_default_required": True,
}
return None
def text_demand_witness(text: str) -> dict[str, Any] | None:
fields: dict[str, str] = {}
for line in text.splitlines():
match = re.fullmatch(r"([A-Za-z][A-Za-z0-9_-]*):[ \t]*(.*)", line)
if match is not None:
fields[match.group(1).lower().replace("-", "_")] = match.group(2).strip()
kind = fields.get("manifest_type", "").lower()
version = fields.get("manifest_version", "")
targets = fields.get("deployment_targets", "") or fields.get("support_targets", "")
required = fields.get("zstd_default_required", "")
if any(token in kind for token in ("support", "deploy")) and version and targets and truthy(required):
return {
"format": "text-header",
"manifest_kind": True,
"version": True,
"deployment_targets": True,
"zstd_default_required": True,
}
return None
def parse_ls_tree(raw: bytes) -> dict[str, dict[str, Any]]:
entries: dict[str, dict[str, Any]] = {}
for record in raw.split(b"\0"):
if not record:
continue
metadata, path_bytes = record.split(b"\t", 1)
mode, kind, oid, size = metadata.decode("ascii").split()
path = path_bytes.decode("utf-8", errors="strict")
entries[path] = {
"mode": mode,
"kind": kind,
"oid": oid,
"size": int(size),
}
return entries
def grep_paths(commit: str) -> list[str]:
raw = run_bytes(
[
"git", "-C", str(ROOT), "grep", "-I", "-l", "-i",
"-e", "zstd", "-e", "zstandard", commit, "--",
"planning/pre15/evidence",
],
{0, 1},
)
prefix = f"{commit}:"
paths = []
for line in raw.decode("utf-8", errors="strict").splitlines():
if not line.startswith(prefix):
raise GateFailure("RUNNER_FAIL", f"unexpected git grep path: {line}")
paths.append(line[len(prefix):])
return sorted(paths)
def main() -> int:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-045":
raise GateFailure("RUNNER_FAIL", "invalid P15-045 input identity")
if SPEC.get("gate") != "G07" or SPEC.get("batch") != "B35":
raise GateFailure("RUNNER_FAIL", "invalid G07/B35 input identity")
resolved = git_text("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure(
"INFRA_BLOCKED",
f"P15-045 must replay {SPEC['required_base']}, got {resolved}",
)
evidence_tree = git_text("rev-parse", f"{resolved}:planning/pre15/evidence")
if evidence_tree != SPEC["evidence_tree_oid"]:
raise GateFailure("INFRA_BLOCKED", "frozen project evidence tree changed")
source_hashes = {
path: sha256_bytes(source_at(resolved, path))
for path in SPEC["source_sha256"]
}
if source_hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen G07 source identity changed")
write_json(OUTPUT / "source-sha256.json", source_hashes)
addendum_paths = (
"repo-pre-15/tests/pre15/gates/P15-045.sh",
"repo-pre-15/tests/pre15/gates/P15-045-input.json",
"repo-pre-15/docs/pre15-stage0/P15-045.md",
)
addendum_hashes = {
path: sha256_bytes((ROOT / path).read_bytes()) for path in addendum_paths
}
write_json(OUTPUT / "gate-addendum-sha256.json", addendum_hashes)
tree_entries = parse_ls_tree(
run_bytes(
[
"git", "-C", str(ROOT), "ls-tree", "-r", "--long", "-z",
resolved, "--", "planning/pre15/evidence",
]
)
)
if len(tree_entries) != SPEC["evidence_tracked_path_count"]:
raise GateFailure("INFRA_BLOCKED", "frozen evidence path count changed")
prohibited = set(SPEC["prohibited_paths"])
if prohibited.intersection(tree_entries):
raise GateFailure("RUNNER_FAIL", "prohibited path entered evidence scan")
zstd_paths = grep_paths(resolved)
maximum_bytes = SPEC["demand_manifest_contract"]["maximum_text_candidate_bytes"]
demand_terms = tuple(SPEC["demand_manifest_contract"]["demand_terms"])
candidates = []
qualified = []
oversized = []
for path in zstd_paths:
entry = tree_entries[path]
if entry["size"] > maximum_bytes:
oversized.append({"path": path, **entry})
continue
data = source_at(resolved, path)
text = data.decode("utf-8", errors="replace")
lowered = text.lower()
signals = sorted(term for term in demand_terms if term.lower() in lowered)
if not signals:
continue
witness = None
if path.endswith(".json"):
try:
witness = json_demand_witness(json.loads(text))
except json.JSONDecodeError:
witness = None
if witness is None:
witness = text_demand_witness(text)
record = {
"path": path,
"blob_oid": entry["oid"],
"bytes": entry["size"],
"sha256": sha256_bytes(data),
"demand_signals": signals,
"qualification": witness,
"disposition": (
"QUALIFIED versioned deployment-demand manifest"
if witness is not None
else "test/build/planning evidence; no explicit versioned deployment demand"
),
}
candidates.append(record)
if witness is not None:
qualified.append(record)
if oversized:
write_json(OUTPUT / "oversized-candidates.json", oversized)
raise GateFailure(
"INFRA_BLOCKED", "a Zstd evidence candidate exceeds the bounded review size"
)
approved = SPEC["approved_demand_manifest_sha256"]
actual_approved = {record["path"]: record["sha256"] for record in qualified}
if actual_approved != approved:
raise GateFailure(
"INFRA_BLOCKED", "qualified demand inventory differs from frozen gate input"
)
makefile = source_at(resolved, "repo-pre-15/src/Makefile").decode("utf-8")
zstd_source = source_at(
resolved, "repo-pre-15/src/decompressor_zstd.c"
).decode("utf-8")
readme = source_at(resolved, "repo-pre-15/README.md").decode("utf-8")
manual = source_at(resolved, "repo-pre-15/docs/erofs.5").decode("utf-8")
policy_checks = {
"makefile_default_is_opt_in_zero": makefile.count("WITH_ZSTDIO?= 0") == 1,
"enabled_build_defines_zstdio": "CFLAGS.decompressor_zstd.c+= -DZSTDIO" in makefile,
"disabled_stub_returns_eopnotsupp": (
"#else\nstatic int\nz_erofs_zstd_decompress" in zstd_source
and "return (EOPNOTSUPP);\n}\n#endif" in zstd_source
),
"readme_names_kernel_option": "kernel built with `options ZSTDIO`" in readme,
"manual_names_kernel_option": '.Cd "options ZSTDIO"' in manual,
}
if not all(policy_checks.values()):
raise GateFailure("INFRA_BLOCKED", "current opt-in ZSTDIO contract changed")
path_inventory = "\0".join(zstd_paths).encode("utf-8")
candidate_inventory = "\0".join(
record["path"] for record in candidates
).encode("utf-8")
demand_scan = {
"schema": 1,
"base": resolved,
"evidence_tree_oid": evidence_tree,
"tracked_path_count": len(tree_entries),
"zstd_text_path_count": len(zstd_paths),
"zstd_text_path_inventory_sha256": sha256_bytes(path_inventory),
"demand_candidate_count": len(candidates),
"demand_candidate_path_inventory_sha256": sha256_bytes(candidate_inventory),
"qualified_manifest_count": len(qualified),
"qualified_manifests": qualified,
"candidates": candidates,
"contract": SPEC["demand_manifest_contract"],
"prohibited_path_read": False,
}
write_json(OUTPUT / "demand-scan.json", demand_scan)
demand_present = bool(qualified)
conditions = [
{
"id": "versioned-deployment-demand",
"status": "PASS" if demand_present else "STOP",
"observation": (
f"{len(qualified)} qualified manifest(s)"
if demand_present
else "zero qualifying manifests in the frozen project evidence tree"
),
},
{
"id": "disabled-runtime-exact-EOPNOTSUPP",
"status": "NOT_RUN",
"reason": "deployment-demand prerequisite is absent",
},
{
"id": "enabled-real-reproducible-Zstd-EROFS-read",
"status": "NOT_RUN",
"reason": "deployment-demand prerequisite is absent",
},
{
"id": "FreeBSD-kernel-ZSTDIO-symbol-capability-model",
"status": "NOT_RUN",
"reason": "deployment-demand prerequisite is absent; no dependency KLD is claimed",
},
{
"id": "KLD-size-delta-at-most-64KiB-and-10-percent",
"status": "NOT_RUN",
"reason": "deployment-demand prerequisite is absent",
},
]
all_go = all(item["status"] == "PASS" for item in conditions)
if all_go:
raise GateFailure(
"RUNNER_FAIL", "false-GO guard: downstream runtime probes were not executed"
)
result = {
"schema": 1,
"gate": "G07",
"candidate": "P15-045",
"batch": "B35",
"status": "STOP",
"reason": "no concrete versioned support/deployment manifest requires Zstd enabled out of the box",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"deadline_seconds": DEADLINE,
"expected_exit_code": 22,
"conditions": conditions,
"false_go_guard": "GO requires every condition PASS; evaluated false",
"current_policy_static_checks": policy_checks,
"with_zstdio_default": 0,
"b35": "STOP-NO-SOURCE",
"builds": "NOT_RUN",
"qemu": "NOT_RUN",
"full_feature_suite": "NOT_RUN",
"smoke_suite": "NOT_RUN",
"generic_dependency_kld_claimed": False,
"production_source_changed": False,
}
write_json(OUTPUT / "result.json", result)
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
return 22
try:
raise SystemExit(main())
except GateFailure as error:
failure = {
"schema": 1,
"gate": "G07",
"candidate": "P15-045",
"status": error.status,
"reason": error.reason,
"requested_base": REQUESTED_BASE,
"deadline_seconds": DEADLINE,
"b35": "NOT_RUN",
"qemu": "NOT_RUN",
"production_source_changed": False,
}
write_json(OUTPUT / "result.json", failure)
print(json.dumps(failure, ensure_ascii=True, sort_keys=True))
raise SystemExit({"RUNNER_FAIL": 20, "INFRA_BLOCKED": 21}.get(error.status, 20))
PY
gate_rc=$?
set -e
cleanup_status=PASS
if ! rm -rf -- "$work"; then
cleanup_status=FAIL
gate_rc=20
fi
trap - EXIT HUP INT TERM
printf 'owned temporary path removed: %s\ncleanup=%s\n' \
"$work" "$cleanup_status" >"$output/cleanup.log"
python3 -B - "$output" "$gate_rc" "$cleanup_status" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
output = Path(sys.argv[1])
exit_code = int(sys.argv[2])
cleanup = sys.argv[3]
if cleanup != "PASS":
status = "RUNNER_FAIL"
origin = "runner"
elif exit_code == 22:
status = "STOP"
origin = "gate"
elif exit_code in (124, 137):
status = "INFRA_BLOCKED"
origin = "infrastructure"
else:
status = "RUNNER_FAIL"
origin = "runner"
(output / "attempt.json").write_text(
json.dumps(
{
"schema": 1,
"exit_code": exit_code,
"status": status,
"failure_origin": origin,
"cleanup": cleanup,
"target_marker": "reached" if status == "STOP" else "not_reached",
},
ensure_ascii=True,
indent=2,
sort_keys=True,
) + "\n",
encoding="ascii",
)
lines = []
for path in sorted(output.iterdir(), key=lambda item: item.name):
if path.is_file() and path.name != "SHA256SUMS":
digest = hashlib.sha256(path.read_bytes()).hexdigest()
lines.append(f"{digest} {path.name}")
(output / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
PY
printf 'P15-045 gate exit=%s cleanup=%s output=%s\n' \
"$gate_rc" "$cleanup_status" "$output"
exit "$gate_rc"
tests/pre15/gates/P15-052-input.json
+203
View File
@@ -0,0 +1,203 @@
{
"schema": 1,
"gate": "G06",
"candidate": "P15-052",
"required_base": "ca7bb4fe6b33e4a1bdf423801134b0ed6bda86dd",
"title": "exact on-disk and ABI errno taxonomy",
"source_sha256": {
"repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a",
"repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1",
"repo-pre-15/src/inode.c": "ba6f77ddffa35cc1dd69ebb933fb16f8fa654f94552fd1d3f0255eb790673e6c",
"repo-pre-15/src/internal.h": "c08ce3dbcafca5c341193c1670516e9bd5815c5f14ce2d637dbf6cfe5a74e972",
"repo-pre-15/src/super.c": "8bb36e9aba15eab2ed10ebec65dccb52dc9ddb485a1a39df37f86f97502c674f",
"repo-pre-15/src/xattr.c": "3498af0b547331b9a22babe96ea718ca2ba56ee669001bfdc2dbaab0a9dc42a9",
"src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a",
"src-linux/internal.h": "4aa671896ff7c0ad32a9108c818ef62d16841c116706fdad391c40a530c81405",
"src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9",
"src-linux/xattr.c": "c8394e5f6301225cbe7587f223485a368348eac7596c1ab0bbf965c99655ed4a"
},
"function_errno_counts": {
"repo-pre-15/src/data.c:erofs_map_blocks_chunk": {
"EOVERFLOW": 6
},
"repo-pre-15/src/xattr.c:erofs_xattr_backing_size": {
"EOVERFLOW": 1
},
"repo-pre-15/src/xattr.c:erofs_xattr_read_backing": {
"EOVERFLOW": 1
},
"repo-pre-15/src/xattr.c:erofs_xattr_read_metadata": {
"EOVERFLOW": 2
},
"repo-pre-15/src/xattr.c:erofs_xattr_shared_entry_offset": {
"EOVERFLOW": 2
}
},
"targets": [
{
"id": "data.chunk.inode_plus_isize",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "inode_off",
"needle": "vi->inode_off > UINT64_MAX - vi->inode_isize",
"prototype_mutation": "inode_off=UINT64_MAX-31"
},
{
"id": "data.chunk.isize_plus_xattr",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "xattr_isize",
"needle": "vi->inode_off + vi->inode_isize > UINT64_MAX - vi->xattr_isize",
"prototype_mutation": "xattr_isize=128 with inode_off near UINT64_MAX"
},
{
"id": "data.chunk.align",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "inode_off",
"needle": "idx_base > UINT64_MAX - (entry_size - 1)",
"prototype_mutation": "inode_off=UINT64_MAX-3"
},
{
"id": "data.chunk.index_multiply",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "chunk_idx",
"needle": "chunk_idx > (UINT64_MAX - idx_base) / entry_size",
"prototype_mutation": "chunk_idx=(UINT64_MAX-4096)/8+1"
},
{
"id": "data.chunk.image_size_shift",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "blocks",
"needle": "sbi->blocks > (UINT64_MAX >> sbi->blkszbits)",
"prototype_mutation": "blocks=(UINT64_MAX>>12)+1"
},
{
"id": "data.chunk.physical_shift",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "blkaddr",
"needle": "blkaddr > (UINT64_MAX >> sbi->blkszbits)",
"prototype_mutation": "blkaddr=(UINT64_MAX>>12)+1"
},
{
"id": "data.chunk.physical_plus_offset",
"path": "repo-pre-15/src/data.c",
"function": "erofs_map_blocks_chunk",
"field": "chunk_off",
"needle": "chunk_off > UINT64_MAX - map->m_pa",
"prototype_mutation": "chunk_off=4096 with blkaddr=UINT64_MAX>>12"
},
{
"id": "xattr.backing_size.shift",
"path": "repo-pre-15/src/xattr.c",
"function": "erofs_xattr_backing_size",
"field": "blocks",
"needle": "sbi->blocks > (UINT64_MAX >> sbi->blkszbits)",
"prototype_mutation": "blocks=(UINT64_MAX>>12)+1"
},
{
"id": "xattr.metadata.align",
"path": "repo-pre-15/src/xattr.c",
"function": "erofs_xattr_read_metadata",
"field": "offp",
"needle": "*offp > UINT64_MAX - (sizeof(struct erofs_xattr_entry) - 1)",
"prototype_mutation": "offp=UINT64_MAX-2"
},
{
"id": "xattr.metadata.header_add",
"path": "repo-pre-15/src/xattr.c",
"function": "erofs_xattr_read_metadata",
"field": "aligned_off",
"needle": "off > UINT64_MAX - sizeof(raw_len)",
"prototype_mutation": "aligned_off=UINT64_MAX-1"
},
{
"id": "xattr.shared.base_shift",
"path": "repo-pre-15/src/xattr.c",
"function": "erofs_xattr_shared_entry_offset",
"field": "xattr_blkaddr",
"needle": "sbi->xattr_blkaddr > (UINT64_MAX >> sbi->blkszbits)",
"prototype_mutation": "xattr_blkaddr=(UINT64_MAX>>12)+1"
},
{
"id": "xattr.shared.base_plus_relative",
"path": "repo-pre-15/src/xattr.c",
"function": "erofs_xattr_shared_entry_offset",
"field": "shared_id",
"needle": "relative > UINT64_MAX - base",
"prototype_mutation": "shared_id=1024 with base near UINT64_MAX"
}
],
"preservation_controls": [
{
"id": "abi.off_gt_int64",
"freebsd": "+EOVERFLOW",
"linux": "-EOVERFLOW semantic",
"result": "blocked first by backing-size range for every mounted primary provider"
},
{
"id": "corruption.range_past_eof",
"freebsd": "+EINTEGRITY",
"linux": "-EFSCORRUPTED semantic",
"result": "preserve"
},
{
"id": "eof.zero_length",
"freebsd": "0",
"linux": "0",
"result": "preserve"
},
{
"id": "io.provider_error",
"freebsd": "+EIO",
"linux": "negative PTR_ERR",
"result": "preserve exact provider error"
},
{
"id": "io.provider_short_media",
"freebsd": "+ENXIO",
"linux": "negative provider error",
"result": "preserve exact media-size error"
},
{
"id": "short_read.logical_backing",
"freebsd": "+EINTEGRITY before I/O",
"linux": "-EFSCORRUPTED semantic",
"result": "preserve"
},
{
"id": "unsupported.exact_xattr_header",
"freebsd": "+EOPNOTSUPP",
"linux": "-EOPNOTSUPP",
"result": "preserve"
},
{
"id": "allocation.xattr_prefixes",
"freebsd": "+ENOMEM",
"linux": "-ENOMEM",
"result": "out of B20 scope"
}
],
"prototype_identity": {
"path": "/work/pre15-gate-prep-20260814T125129Z/G06",
"errno_model.py": "dca427acf5c919f78d46ab8dbb198778fbe2af0869043d93b4eddf9f03814a4e",
"corpus/errno-vectors.json": "dde22db91f7fa2613631b4acb75a5d60e255a1c1564be6800db377922a6a7d4e",
"input.json": "0835defd288cb6a73337c116c4148daee94b78c435732032b5de40d1b1559715",
"oracle.md": "62e6703e8051b832108b4ea39ece232f06aea5dcfb221d7e8c17faf7ae527208",
"replay-base.sh": "2162fb645693ce740563a2406ea28ab86dc116a33163e026fc4d491c2df3e8a5"
},
"decision": {
"go": "every target is independently reachable from one validated on-disk field mutation, reaches its named source branch, and has one unique source class",
"stop": "any target is unreachable after current decode/mount bounds, any prototype mutation injects a local rather than an on-disk input, or any source class is ambiguous",
"atomicity": "P15-052 has no partial source implementation"
},
"timeouts": {
"host_seconds": 240,
"qemu_seconds": 1200
},
"qemu_required_after_host_stop": false,
"full_feature_suite_required": false
}
tests/pre15/gates/P15-052.sh
Executable
+472
View File
@@ -0,0 +1,472 @@
#!/bin/sh
set -eu
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-052-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || {
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
exit 2
}
for tool in git python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
U32_MAX = (1 << 32) - 1
U48_MAX = (1 << 48) - 1
U64_MAX = (1 << 64) - 1
I64_MAX = (1 << 63) - 1
def git(*args: str) -> str:
return subprocess.check_output(
["git", "-C", str(ROOT), *args], text=True
).strip()
def sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def source_at(commit: str, path: str) -> str:
completed = subprocess.run(
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if completed.returncode != 0:
raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}")
return completed.stdout
def extract_function(source: str, name: str) -> str:
match = re.search(r"^" + re.escape(name) + r"\s*\(", source, re.MULTILINE)
if match is None:
raise SystemExit(f"function not found: {name}")
start = source.rfind("\n\n", 0, match.start()) + 2
brace = source.find("{", match.end())
if brace < 0:
raise SystemExit(f"function body not found: {name}")
depth = 0
state = "code"
index = brace
while index < len(source):
char = source[index]
following = source[index + 1] if index + 1 < len(source) else ""
if state == "code":
if char == "/" and following == "*":
state = "block"
index += 2
continue
if char == "/" and following == "/":
state = "line"
index += 2
continue
if char == '"':
state = "string"
elif char == "'":
state = "character"
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
return source[start : index + 1] + "\n"
elif state == "block" and char == "*" and following == "/":
state = "code"
index += 2
continue
elif state == "line" and char == "\n":
state = "code"
elif state in {"string", "character"}:
if char == "\\":
index += 2
continue
if (state == "string" and char == '"') or (
state == "character" and char == "'"
):
state = "code"
index += 1
raise SystemExit(f"unterminated function: {name}")
def line_number(source: str, needle: str) -> int:
if source.count(needle) != 1:
raise SystemExit(
f"source needle must occur exactly once ({source.count(needle)}): {needle}"
)
return source.count("\n", 0, source.index(needle)) + 1
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-052":
raise SystemExit("invalid P15-052 gate input")
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise SystemExit(
f"P15-052 must replay {SPEC['required_base']}, got {resolved}"
)
sources = {
path: source_at(resolved, path) for path in SPEC["source_sha256"]
}
source_hashes = {
path: sha256(text.encode("utf-8")) for path, text in sources.items()
}
for path, expected in SPEC["source_sha256"].items():
if source_hashes[path] != expected:
raise SystemExit(f"frozen source identity mismatch: {path}")
function_bodies: dict[str, str] = {}
for key, counts in SPEC["function_errno_counts"].items():
path, function = key.rsplit(":", 1)
body = extract_function(sources[path], function)
function_bodies[key] = body
for errno_name, expected_count in counts.items():
actual_count = body.count(f"return ({errno_name});")
if actual_count != expected_count:
raise SystemExit(
f"{key} {errno_name} count changed: {actual_count} != {expected_count}"
)
if re.search(r"return\s*\(\s*-E[A-Z0-9_]+", body):
raise SystemExit(f"negative errno entered FreeBSD function: {key}")
param_path = FREEBSD_SRC / "sys/amd64/include/param.h"
types_path = FREEBSD_SRC / "sys/sys/_types.h"
geom_path = FREEBSD_SRC / "sys/geom/geom.h"
for path in (param_path, types_path, geom_path):
if not path.is_file():
raise SystemExit(f"missing FreeBSD contract source: {path}")
param_source = param_path.read_text(encoding="utf-8")
types_source = types_path.read_text(encoding="utf-8")
geom_source = geom_path.read_text(encoding="utf-8")
page_match = re.search(r"^#define\s+PAGE_SHIFT\s+(\d+)\b", param_source, re.MULTILINE)
if page_match is None:
raise SystemExit("PAGE_SHIFT is absent from amd64 param.h")
page_shift = int(page_match.group(1))
if page_shift != 12:
raise SystemExit(f"unexpected audited amd64 PAGE_SHIFT: {page_shift}")
if "typedef\t__int64_t\t__off_t;" not in types_source:
raise SystemExit("FreeBSD off_t source is not the audited signed 64-bit type")
if "off_t\t\t\tmediasize;" not in geom_source:
raise SystemExit("GEOM mediasize is not the audited off_t field")
freebsd_head = subprocess.check_output(
["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True
).strip()
freebsd_contract = {
"head": freebsd_head,
"page_shift": page_shift,
"off_t": "signed-64",
"geom_mediasize": "off_t",
"files": {
str(path.relative_to(FREEBSD_SRC)): sha256(path.read_bytes())
for path in (param_path, types_path, geom_path)
},
}
# These are maxima after the current super/inode decode and mount-time media
# checks, not unconstrained C-structure values.
inode_slot_min = 32
inode_isize_max = 64
xattr_isize_max = 4 + 4 * ((1 << 16) - 2)
inode_off_max = I64_MAX - inode_slot_min
idx_base_max = inode_off_max + inode_isize_max + xattr_isize_max
entry_size_max = 8
chunkbits_min = 9
chunkbits_max = page_shift + 31
chunk_idx_max = I64_MAX >> chunkbits_min
blkaddr_max = U48_MAX
physical_max = blkaddr_max << page_shift
chunk_off_max = (1 << chunkbits_max) - 1
prefix_record_max = 256
prefix_off_max = U32_MAX * 4 + 255 * (3 + 2 + prefix_record_max)
xattr_base_max = U32_MAX << page_shift
xattr_relative_max = U32_MAX * 4
reachability: dict[str, tuple[bool, str, dict[str, int]]] = {
"data.chunk.inode_plus_isize": (
inode_off_max > U64_MAX - inode_isize_max,
"inode_off is bounded by mounted backing size minus one compact inode",
{"inode_off_max": inode_off_max, "inode_isize_max": inode_isize_max},
),
"data.chunk.isize_plus_xattr": (
inode_off_max + inode_isize_max > U64_MAX - xattr_isize_max,
"inode offset plus maximum 64-byte inode and 16-bit xattr body stays below UINT64_MAX",
{"metadata_base_max": inode_off_max + inode_isize_max, "xattr_isize_max": xattr_isize_max},
),
"data.chunk.align": (
idx_base_max > U64_MAX - (entry_size_max - 1),
"validated backing and bounded inode/xattr sizes leave alignment headroom",
{"idx_base_max": idx_base_max, "alignment_slack": entry_size_max - 1},
),
"data.chunk.index_multiply": (
chunk_idx_max > (U64_MAX - idx_base_max) // entry_size_max,
"size <= OFF_MAX and chunkbits >= 9 bound the index product",
{"idx_base_max": idx_base_max, "chunk_idx_max": chunk_idx_max, "entry_size_max": entry_size_max},
),
"data.chunk.image_size_shift": (
I64_MAX > U64_MAX,
"mount validates blocks << blkszbits <= GEOM off_t mediasize",
{"validated_image_bytes_max": I64_MAX},
),
"data.chunk.physical_shift": (
blkaddr_max > (U64_MAX >> page_shift),
"decoded chunk block address is at most 48 bits and blkszbits is at most PAGE_SHIFT",
{"blkaddr_max": blkaddr_max, "blkszbits_max": page_shift},
),
"data.chunk.physical_plus_offset": (
chunk_off_max > U64_MAX - physical_max,
"48-bit block address and maximum validated chunkbits leave addition headroom",
{"physical_max": physical_max, "chunk_off_max": chunk_off_max},
),
"xattr.backing_size.shift": (
I64_MAX > U64_MAX,
"mounted primary backing bytes are already bounded by GEOM off_t mediasize",
{"validated_backing_bytes_max": I64_MAX},
),
"xattr.metadata.align": (
prefix_off_max > U64_MAX - 3,
"32-bit prefix start, 8-bit count, and bounded records cannot approach UINT64_MAX",
{"prefix_off_max": prefix_off_max},
),
"xattr.metadata.header_add": (
False,
"the preceding off <= UINT64_MAX-3 check and 4-byte roundup imply aligned off <= UINT64_MAX-3, so adding uint16_t cannot overflow",
{"accepted_input_max": U64_MAX - 3, "aligned_off_max": U64_MAX - 3, "raw_len_size": 2},
),
"xattr.shared.base_shift": (
U32_MAX > (U64_MAX >> page_shift),
"xattr_blkaddr is 32-bit and blkszbits is at most PAGE_SHIFT",
{"xattr_blkaddr_max": U32_MAX, "blkszbits_max": page_shift},
),
"xattr.shared.base_plus_relative": (
xattr_relative_max > U64_MAX - xattr_base_max,
"32-bit xattr block address and 32-bit shared ID cannot overflow the 64-bit sum",
{"xattr_base_max": xattr_base_max, "relative_max": xattr_relative_max},
),
}
ledger = []
for target in SPEC["targets"]:
target_id = target["id"]
if target_id not in reachability:
raise SystemExit(f"target has no reachability proof: {target_id}")
body = function_bodies[f"{target['path']}:{target['function']}"]
if body.count(target["needle"]) != 1:
raise SystemExit(f"target source is absent or ambiguous: {target_id}")
reachable, reason, bounds = reachability[target_id]
ledger.append(
{
**target,
"line": line_number(sources[target["path"]], target["needle"]),
"source_unique": True,
"target_marker_reached": reachable,
"current_freebsd_return": "+EOVERFLOW",
"candidate_freebsd_return": "+EINTEGRITY",
"linux_semantic_return": "-EFSCORRUPTED (no direct checked counterpart)",
"source_class": "on-disk contradiction",
"replay_observation": reason,
"provenance_bounds": bounds,
}
)
data_source = sources["repo-pre-15/src/data.c"]
xattr_source = sources["repo-pre-15/src/xattr.c"]
linux_xattr = sources["src-linux/xattr.c"]
control_anchors = (
(xattr_source, "if (off > backing_size || (uint64_t)len > backing_size - off)\n\t\treturn (EINTEGRITY);"),
(xattr_source, "if (off > INT64_MAX)\n\t\t\treturn (EOVERFLOW);"),
(xattr_source, "error = EOPNOTSUPP;"),
(data_source, "return (ENXIO);"),
(data_source, "return (EIO);"),
(data_source, "if (len == 0) {\n\t\treturn (0);\n\t}"),
(linux_xattr, "ret = -EOPNOTSUPP;"),
(linux_xattr, "ret = -ENOMEM;"),
(linux_xattr, "return PTR_ERR(it->kaddr);"),
(linux_xattr, "return -EFSCORRUPTED;"),
)
for source, anchor in control_anchors:
if anchor not in source:
raise SystemExit(f"errno preservation anchor is absent: {anchor}")
abi_reachable = False
abi_reason = (
"erofs_xattr_read_backing checks off <= backing_size before off > INT64_MAX; "
"for a mounted primary backing, backing_size <= GEOM signed off_t mediasize"
)
controls = []
for control in SPEC["preservation_controls"]:
record: dict[str, Any] = dict(control)
record["source_anchors_verified"] = True
if control["id"] == "abi.off_gt_int64":
record["target_marker_reached"] = abi_reachable
record["replay_observation"] = abi_reason
controls.append(record)
unreachable = [record["id"] for record in ledger if not record["target_marker_reached"]]
status = "GO" if not unreachable else "STOP"
stop_reasons = [
{
"id": target_id,
"reason": next(
record["replay_observation"] for record in ledger if record["id"] == target_id
),
}
for target_id in unreachable
]
ledger_document = {
"schema": 1,
"gate": "G06",
"candidate": "P15-052",
"status": status,
"resolved_base": resolved,
"targets": ledger,
"preservation_controls": controls,
"freebsd_contract": freebsd_contract,
"prototype_identity": SPEC["prototype_identity"],
"prototype_disposition": (
"READY arithmetic vectors inject unconstrained internal or local values; "
"they are not independently reachable on-disk fixtures"
),
}
(OUTPUT / "branch-ledger.json").write_text(
json.dumps(ledger_document, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
tsv = [
"id\tfunction\tfield\tfreebsd_now\tfreebsd_candidate\tlinux_semantic\tmarker\tobservation"
]
for record in ledger:
tsv.append(
"\t".join(
(
record["id"],
record["function"],
record["field"],
record["current_freebsd_return"],
record["candidate_freebsd_return"],
record["linux_semantic_return"],
"reached" if record["target_marker_reached"] else "not-reached",
record["replay_observation"],
)
)
)
(OUTPUT / "branch-ledger.tsv").write_text("\n".join(tsv) + "\n", encoding="ascii")
control_tsv = ["id\tfreebsd\tlinux\tresult\tmarker"]
for record in controls:
control_tsv.append(
"\t".join(
(
record["id"],
record["freebsd"],
record["linux"],
record["result"],
(
"not-reached"
if record.get("target_marker_reached") is False
else "preserved"
),
)
)
)
(OUTPUT / "preservation-ledger.tsv").write_text(
"\n".join(control_tsv) + "\n", encoding="ascii"
)
(OUTPUT / "source-sha256.json").write_text(
json.dumps(source_hashes, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
(OUTPUT / "freebsd-contract.json").write_text(
json.dumps(freebsd_contract, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
result = {
"schema": 1,
"gate": "G06",
"candidate": "P15-052",
"status": status,
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"target_count": len(ledger),
"target_markers_reached": len(ledger) - len(unreachable),
"target_markers_not_reached": len(unreachable),
"source_unique_count": sum(record["source_unique"] for record in ledger),
"preservation_control_count": len(controls),
"abi_marker_reached": abi_reachable,
"stop_reasons": stop_reasons,
"b20": "STOP-NO-SOURCE",
"b21": "NOT_RUN",
"qemu": "NOT_RUN",
"qemu_reason": "host-proven target reachability failure",
"full_feature_suite": "NOT_RUN",
}
(OUTPUT / "result.json").write_text(
json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
hash_lines = []
for path in sorted(OUTPUT.iterdir()):
if path.is_file() and path.name != "SHA256SUMS":
hash_lines.append(f"{sha256(path.read_bytes())} {path.name}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
print(json.dumps(result, sort_keys=True))
if status != "GO":
raise SystemExit(1)
PY
tests/pre15/gates/P15-062-input.json
+138
View File
@@ -0,0 +1,138 @@
{
"candidate": "P15-062",
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/sys/endian.h": "3d870d499089ac84bb49debebbb75b406a5f4df6f05f1582faebc5cf0b7b8806",
"sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c",
"sys/sys/param.h": "cc451dd2de4d6a7cc97b928ddb9e5b19475cba4ab418153ab7f6e6baa661b917",
"sys/sys/systm.h": "648c4ba00523c4dff1e9c0e693722d748e38312fd3cf35f2ee92c21188b62649"
}
},
"gate": "G04",
"legacy_fixture": {
"block_size": 4096,
"expected_image_size": 12288,
"expected_source_sha256": "cea05c6972efce79727fe268260e8de74089b0fc0b527825beaca8de4dbacff2",
"extents": [
{
"consumed_bytes": 4096,
"decoded_sha256": "9d1d4465d9891e22a7cb79f1d5c9e8c4b7500753fbe5a7feca1117e948970017",
"logical_length": 1038906,
"logical_offset": 0,
"physical_length": 4096,
"physical_offset": 4096,
"zero_tail_bytes": 0
},
{
"consumed_bytes": 48,
"decoded_sha256": "c5563821cb010b2bf6ba6ecd5a563f71f9d37644f086c90de4a9c44a071b4bdc",
"logical_length": 9670,
"logical_offset": 1038906,
"physical_length": 4096,
"physical_offset": 8192,
"zero_tail_bytes": 4048
}
],
"mkfs_args": [
"-T0",
"-U00112233-4455-6677-8899-aabbccddeeff",
"--all-root",
"-x-1",
"-zlz4",
"-C4096",
"-Elegacy-compress"
],
"segments": [
{
"byte": 65,
"length": 262144
},
{
"byte": 66,
"length": 262144
},
{
"byte": 67,
"length": 262144
},
{
"byte": 68,
"length": 262144
}
]
},
"partial_fixture": {
"a_sha256": "cea05c6972efce79727fe268260e8de74089b0fc0b527825beaca8de4dbacff2",
"b_nid": 58,
"b_sha256": "f25f98332996139ce81bac04665df5593856430a39b6ed7e25a25f8c066e0d47",
"block_size": 4096,
"expected_image_sha256": "63b6ec3736b325f5e3d355eef35ac8f7e80cb7edc49e3e495a4006fffe7f7514",
"expected_image_size": 16384,
"full_decoded_bytes": 9670,
"full_stream_consumed_bytes": 48,
"mkfs_args": [
"-T0",
"-U00112233-4455-6677-8899-aabbccddeeff",
"--all-root",
"-x-1",
"-zlz4",
"-C4096",
"-Ededupe"
],
"partial_consumed_bytes": 42,
"partial_index_records": [
{
"cluster_offset": 550,
"lcn": 254,
"pblk": 2
},
{
"cluster_offset": 2048,
"lcn": 255,
"pblk": 2
}
],
"partial_output_bytes": [
4096,
5594
],
"prefix_pad_byte": 33,
"prefix_pattern": "P15-062-partial-reference\n",
"prefix_size": 2048,
"shared_pcluster_offset": 8192,
"stream_start": 4048
},
"required_base": "205a90465edb64e83ba44aaacac9bedad4cbe905",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c",
"repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98",
"repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242",
"repo-pre-15/src/zmap.c": "95432d49c20f3049e0fb5af21904283494325570673dbb037767c1397c77c1f3",
"src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709"
},
"upstream": {
"liblz4_sha256": "67bf8b84af77e962e09be661944f7c61393c4806746734e36db9fbdaebc1de36",
"liblz4_soname": "/lib/x86_64-linux-gnu/liblz4.so.1",
"liblz4_version": "1.10.0",
"url": "https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git",
"versions": [
{
"commit": "ee97fe5fb77c737df0f77d92ab0d92edd3a11be6",
"expected_image_sha256": "d78c1f00b930bfbf45ac2a0a903896ed5559bac43297e283b348a63926ce5576",
"name": "v1.4"
},
{
"commit": "ce36273833096f1b7e828309d9b1caa37132092d",
"expected_image_sha256": "1f08b09b7ea40d495160699ddf7079bf5d4deb85859dbe7459a41617f25fdddf",
"name": "v1.7"
},
{
"commit": "3689cbc2349bff05807d2f939146e92eb1bfaea1",
"expected_image_sha256": "c5ee543074d57a45adc7c666db47ca435c5d3c8504d3163d471ed53bdbdac303",
"name": "v1.8.6"
}
]
}
}
tests/pre15/gates/P15-062.sh
Executable
+713
View File
@@ -0,0 +1,713 @@
#!/bin/sh
set -eu
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-062-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || {
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
exit 2
}
for tool in cc git make pkg-config python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
from __future__ import annotations
import ctypes
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import struct
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
class GateFailure(Exception):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
return sha256_bytes(path.read_bytes())
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def git(*args: str) -> str:
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
def source_at(commit: str, path: str) -> str:
completed = subprocess.run(
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if completed.returncode != 0:
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
return completed.stdout
def run_logged(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> subprocess.CompletedProcess[str]:
completed = subprocess.run(
argv,
cwd=cwd,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=timeout,
)
log.parent.mkdir(parents=True, exist_ok=True)
log.write_text(
"$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n",
encoding="utf-8",
)
if completed.returncode != 0:
raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}")
return completed
def raw_lz4_decode(data: bytes, target: int | None = None) -> tuple[bytes, int]:
ip = 0
output = bytearray()
while ip < len(data):
token = data[ip]
ip += 1
literal_length = token >> 4
if literal_length == 15:
while True:
if ip >= len(data):
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated literal length")
value = data[ip]
ip += 1
literal_length += value
if value != 255:
break
if ip + literal_length > len(data):
raise GateFailure("INFRA_BLOCKED", "oracle saw truncated literals")
output.extend(data[ip : ip + literal_length])
ip += literal_length
if target is not None and len(output) >= target:
return bytes(output[:target]), ip
if ip == len(data):
return bytes(output), ip
if ip + 2 > len(data):
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match offset")
offset = data[ip] | data[ip + 1] << 8
ip += 2
if offset == 0 or offset > len(output):
raise GateFailure("INFRA_BLOCKED", f"oracle saw invalid match offset {offset}")
match_length = token & 15
if match_length == 15:
while True:
if ip >= len(data):
raise GateFailure("INFRA_BLOCKED", "oracle saw a truncated match length")
value = data[ip]
ip += 1
match_length += value
if value != 255:
break
for _ in range(match_length + 4):
output.append(output[-offset])
if target is not None and len(output) >= target:
return bytes(output[:target]), ip
raise GateFailure("INFRA_BLOCKED", "oracle input ended before a final literal sequence")
def make_legacy_source(path: Path) -> bytes:
content = b"".join(
bytes([segment["byte"]]) * segment["length"]
for segment in SPEC["legacy_fixture"]["segments"]
)
if sha256_bytes(content) != SPEC["legacy_fixture"]["expected_source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "legacy source generator hash drift")
path.mkdir(parents=True)
(path / "big.txt").write_bytes(content)
return content
def make_partial_source(path: Path) -> tuple[bytes, bytes]:
path.mkdir(parents=True)
data_a = b"".join(
bytes([segment["byte"]]) * segment["length"]
for segment in SPEC["legacy_fixture"]["segments"]
)
partial = SPEC["partial_fixture"]
pattern = partial["prefix_pattern"].encode("ascii")
repeats = (partial["prefix_size"] + len(pattern) - 1) // len(pattern)
prefix = (pattern * repeats)[: partial["prefix_size"]]
prefix = prefix.ljust(partial["prefix_size"], bytes([partial["prefix_pad_byte"]]))
data_b = prefix + data_a
if sha256_bytes(data_a) != partial["a_sha256"] or sha256_bytes(data_b) != partial["b_sha256"]:
raise GateFailure("INFRA_BLOCKED", "partial source generator hash drift")
(path / "a.dat").write_bytes(data_a)
(path / "b.dat").write_bytes(data_b)
return data_a, data_b
def load_liblz4() -> tuple[Any, Any]:
upstream = SPEC["upstream"]
soname = Path(upstream["liblz4_soname"])
version = subprocess.check_output(["pkg-config", "--modversion", "liblz4"], text=True).strip()
if version != upstream["liblz4_version"] or sha256_path(soname) != upstream["liblz4_sha256"]:
raise GateFailure("INFRA_BLOCKED", "liblz4 identity changed")
library = ctypes.CDLL(str(soname))
full = library.LZ4_decompress_safe
full.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
full.restype = ctypes.c_int
partial = library.LZ4_decompress_safe_partial
partial.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int]
partial.restype = ctypes.c_int
return full, partial
def liblz4_call(function: Any, source: bytes, output_size: int, target: int | None = None) -> tuple[int, bytes]:
source_buffer = ctypes.create_string_buffer(source, len(source))
output_buffer = ctypes.create_string_buffer(output_size)
if target is None:
result = function(source_buffer, output_buffer, len(source), output_size)
else:
result = function(source_buffer, output_buffer, len(source), target, output_size)
return result, output_buffer.raw
def compile_dut_decoder(temp: Path, source: str) -> Any:
source_dir = temp / "dut-decoder"
sys_dir = source_dir / "sys"
sys_dir.mkdir(parents=True)
(source_dir / "decompressor_lz4.c").write_text(source, encoding="utf-8")
(sys_dir / "param.h").write_text(
"#include <stdbool.h>\n#include <stddef.h>\n#include <stdint.h>\n#include <string.h>\n"
"#define EINTEGRITY 97\n#define MIN(a, b) ((a) < (b) ? (a) : (b))\n",
encoding="ascii",
)
(sys_dir / "endian.h").write_text(
"#include <stdint.h>\nstatic inline uint16_t le16dec(const void *p) { "
"const uint8_t *b = p; return (uint16_t)b[0] | (uint16_t)b[1] << 8; }\n",
encoding="ascii",
)
(sys_dir / "systm.h").write_text("#include <string.h>\n", encoding="ascii")
(source_dir / "compress.h").write_text(
"#include <stdbool.h>\n#include <stddef.h>\n"
"struct z_erofs_decompress_req { void *sbi; const void *map; const void *in; "
"size_t inputsize; void *out; size_t outputsize; bool partial_decoding; };\n",
encoding="ascii",
)
library = temp / "dut-lz4.so"
run_logged(
["cc", "-shared", "-fPIC", "-std=c11", "-Wall", "-Wextra", "-Werror", "-I", str(source_dir),
"-o", str(library), str(source_dir / "decompressor_lz4.c")],
temp,
OUTPUT / "logs/dut-decoder-build.log",
)
loaded = ctypes.CDLL(str(library))
class Request(ctypes.Structure):
_fields_ = [
("sbi", ctypes.c_void_p),
("map", ctypes.c_void_p),
("in", ctypes.c_void_p),
("inputsize", ctypes.c_size_t),
("out", ctypes.c_void_p),
("outputsize", ctypes.c_size_t),
("partial_decoding", ctypes.c_bool),
]
function = loaded.z_erofs_lz4_decompress
function.argtypes = [ctypes.POINTER(Request)]
function.restype = ctypes.c_int
return loaded, function, Request
def dut_call(function: Any, request_type: Any, source: bytes, output_size: int, partial: bool) -> tuple[int, bytes, bool]:
source_buffer = ctypes.create_string_buffer(source, len(source))
guard_size = 64
backing = bytearray(b"\xa5" * guard_size + b"\0" * output_size + b"\x5a" * guard_size)
output_buffer = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
output_pointer = ctypes.cast(ctypes.byref(output_buffer, guard_size), ctypes.c_void_p)
request = request_type(
None,
None,
ctypes.cast(source_buffer, ctypes.c_void_p),
len(source),
output_pointer,
output_size,
partial,
)
result = function(ctypes.byref(request))
guards_ok = backing[:guard_size] == b"\xa5" * guard_size and backing[-guard_size:] == b"\x5a" * guard_size
return result, bytes(backing[guard_size : guard_size + output_size]), guards_ok
def verify_cleanup_source(zdata_source: str) -> dict[str, Any]:
anchors = [
"error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,",
"erofs_put_metabuf(&buf);",
"erofs_brelse(compressed);",
"free(decoded, M_EROFS);",
"*bufp = decoded;",
]
missing = [anchor for anchor in anchors if anchor not in zdata_source]
if missing:
raise GateFailure("INFRA_BLOCKED", f"DUT cleanup source anchors changed: {missing}")
release_pos = min(zdata_source.index("erofs_put_metabuf(&buf);"), zdata_source.index("erofs_brelse(compressed);"))
error_pos = zdata_source.index("if (error != 0) {", release_pos)
free_pos = zdata_source.index("free(decoded, M_EROFS);", error_pos)
publish_pos = zdata_source.index("*bufp = decoded;", free_pos)
return {
"decoded_error_free_after_input_release": release_pos < error_pos < free_pos,
"decoded_success_publish_after_error_branch": free_pos < publish_pos,
"metadata_release_anchor": True,
"physical_release_anchor": True,
}
def build_versions(temp: Path, legacy_source: bytes) -> tuple[list[dict[str, Any]], dict[str, Path]]:
upstream = SPEC["upstream"]
clone = temp / "erofs-utils"
run_logged(
["git", "clone", "--no-checkout", upstream["url"], str(clone)],
temp,
OUTPUT / "logs/upstream-clone.log",
timeout=90,
)
records = []
worktrees: dict[str, Path] = {}
fixture = SPEC["legacy_fixture"]
source_dir = temp / "legacy-source"
make_legacy_source(source_dir)
for version in upstream["versions"]:
name = version["name"]
worktree = temp / f"erofs-{name}"
worktrees[name] = worktree
run_logged(
["git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), version["commit"]],
temp,
OUTPUT / f"logs/{name}-worktree.log",
)
resolved = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip()
if resolved != version["commit"]:
raise GateFailure("INFRA_BLOCKED", f"upstream commit mismatch for {name}")
run_logged(["./autogen.sh"], worktree, OUTPUT / f"logs/{name}-autogen.log")
run_logged(["./configure", "--disable-fuse"], worktree, OUTPUT / f"logs/{name}-configure.log")
run_logged(["make", "-s", "-j2"], worktree, OUTPUT / f"logs/{name}-make.log")
mkfs = worktree / "mkfs/mkfs.erofs"
fsck = worktree / "fsck/fsck.erofs"
dump = worktree / "dump/dump.erofs"
images = temp / f"images-{name}"
images.mkdir()
generated = []
for label in ("a", "b"):
image = images / f"legacy-{label}.erofs"
argv = [str(mkfs), *fixture["mkfs_args"], str(image), str(source_dir)]
run_logged(argv, worktree, OUTPUT / f"logs/{name}-mkfs-{label}.log")
generated.append(image)
hashes = [sha256_path(path) for path in generated]
if hashes[0] != hashes[1] or hashes[0] != version["expected_image_sha256"]:
raise GateFailure("INFRA_BLOCKED", f"{name} image reproducibility/hash mismatch: {hashes}")
run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / f"logs/{name}-fsck.log")
superblock = run_logged([str(dump), "-s", str(generated[0])], worktree, OUTPUT / f"logs/{name}-super.log").stdout
feature_line = next((line for line in superblock.splitlines() if "features:" in line), "")
if "0padding" in feature_line:
raise GateFailure("INFRA_BLOCKED", f"{name} legacy fixture unexpectedly has 0padding")
image = generated[0].read_bytes()
if len(image) != fixture["expected_image_size"]:
raise GateFailure("INFRA_BLOCKED", f"{name} image size changed")
extents = []
reconstructed = bytearray()
for expected in fixture["extents"]:
block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]]
decoded, consumed = raw_lz4_decode(block, expected["logical_length"])
tail = block[consumed:]
if (
consumed != expected["consumed_bytes"]
or sha256_bytes(decoded) != expected["decoded_sha256"]
or len(tail) != expected["zero_tail_bytes"]
or any(tail)
):
raise GateFailure("INFRA_BLOCKED", f"{name} independent extent oracle drift")
logical = legacy_source[expected["logical_offset"] : expected["logical_offset"] + expected["logical_length"]]
if decoded != logical:
raise GateFailure("INFRA_BLOCKED", f"{name} decoded extent differs from source")
reconstructed.extend(decoded)
extents.append({
**expected,
"input_bytes": len(block),
"tail_all_zero": not any(tail),
})
if bytes(reconstructed) != legacy_source:
raise GateFailure("INFRA_BLOCKED", f"{name} reconstructed file mismatch")
records.append({
"commit": resolved,
"fsck": "PASS",
"image_repeated_sha256": hashes,
"image_size": len(image),
"legacy_without_0padding": True,
"name": name,
"extents": extents,
})
return records, worktrees
def verify_partial_fixture(temp: Path, worktree: Path, full: Any, partial_codec: Any, dut: Any) -> dict[str, Any]:
partial_spec = SPEC["partial_fixture"]
source_dir = temp / "partial-source"
data_a, data_b = make_partial_source(source_dir)
images = temp / "partial-images"
images.mkdir()
mkfs = worktree / "mkfs/mkfs.erofs"
fsck = worktree / "fsck/fsck.erofs"
dump = worktree / "dump/dump.erofs"
generated = []
for label in ("a", "b"):
image = images / f"partial-{label}.erofs"
run_logged(
[str(mkfs), *partial_spec["mkfs_args"], str(image), str(source_dir)],
worktree,
OUTPUT / f"logs/partial-mkfs-{label}.log",
)
generated.append(image)
hashes = [sha256_path(path) for path in generated]
if hashes[0] != hashes[1] or hashes[0] != partial_spec["expected_image_sha256"]:
raise GateFailure("INFRA_BLOCKED", f"partial fixture reproducibility/hash mismatch: {hashes}")
run_logged([str(fsck), "--extract", str(generated[0])], worktree, OUTPUT / "logs/partial-fsck.log")
dump_b = run_logged(
[str(dump), "--path=/b.dat", "-e", str(generated[0])],
worktree,
OUTPUT / "logs/partial-dump-b.log",
).stdout
match = re.search(r"^NID:\s+(\d+)", dump_b, re.MULTILINE)
if match is None or int(match.group(1)) != partial_spec["b_nid"]:
raise GateFailure("INFRA_BLOCKED", "partial fixture b.dat NID drift")
image = generated[0].read_bytes()
if len(image) != partial_spec["expected_image_size"]:
raise GateFailure("INFRA_BLOCKED", "partial fixture image size drift")
inode_offset = partial_spec["b_nid"] * 32
inode_format, xattr_count, _, _, inode_size = struct.unpack_from("<HHHHI", image, inode_offset)
inode_bytes = 64 if inode_format & 1 else 32
datalayout = (inode_format >> 1) & 7
if xattr_count != 0 or datalayout != 1 or inode_size != len(data_b):
raise GateFailure("INFRA_BLOCKED", "partial fixture inode contract drift")
index_start = ((inode_offset + inode_bytes + 7) & ~7) + 16
index_records = []
for expected in partial_spec["partial_index_records"]:
position = index_start + expected["lcn"] * 8
advise, cluster_offset, pblk = struct.unpack_from("<HHI", image, position)
record = {
"advise": advise,
"cluster_offset": cluster_offset,
"lcn": expected["lcn"],
"partial_ref": bool(advise & 0x8000),
"pblk": pblk,
"position": position,
}
if not record["partial_ref"] or (cluster_offset, pblk) != (expected["cluster_offset"], expected["pblk"]):
raise GateFailure("INFRA_BLOCKED", "partial-reference index oracle drift")
index_records.append(record)
block_offset = partial_spec["shared_pcluster_offset"]
block = image[block_offset : block_offset + partial_spec["block_size"]]
first_nonzero = next((index for index, value in enumerate(block) if value), len(block))
if first_nonzero != partial_spec["stream_start"]:
raise GateFailure("INFRA_BLOCKED", "partial pcluster leading padding drift")
stream = block[first_nonzero:]
full_decoded, full_consumed = raw_lz4_decode(stream, partial_spec["full_decoded_bytes"])
if full_consumed != partial_spec["full_stream_consumed_bytes"]:
raise GateFailure("INFRA_BLOCKED", "partial pcluster full consumption drift")
expected_tail = data_a[-partial_spec["full_decoded_bytes"] :]
if full_decoded != expected_tail:
raise GateFailure("INFRA_BLOCKED", "partial pcluster decoded bytes drift")
partial_records = []
for output_size in partial_spec["partial_output_bytes"]:
decoded, consumed = raw_lz4_decode(stream, output_size)
if consumed != partial_spec["partial_consumed_bytes"] or decoded != expected_tail[:output_size]:
raise GateFailure("INFRA_BLOCKED", "partial parser consumption/output drift")
result, output, guards = dut_call(dut[1], dut[2], stream, output_size, True)
if result != 0 or output != expected_tail[:output_size] or not guards:
raise GateFailure("INFRA_BLOCKED", "frozen DUT partial decode mismatch")
partial_records.append({"consumed_bytes": consumed, "decoded_bytes": output_size, "dut_errno": result, "guards": guards})
corrupted = bytearray(stream)
corrupted[partial_spec["partial_consumed_bytes"] :] = b"\0" * (
len(corrupted) - partial_spec["partial_consumed_bytes"]
)
partial_result, partial_output, partial_guards = dut_call(
dut[1], dut[2], bytes(corrupted), partial_spec["partial_output_bytes"][0], True
)
full_result, _, full_guards = dut_call(
dut[1], dut[2], bytes(corrupted), partial_spec["full_decoded_bytes"], False
)
lib_partial_result, lib_partial_output = liblz4_call(
partial_codec, bytes(corrupted), partial_spec["partial_output_bytes"][0], partial_spec["partial_output_bytes"][0]
)
lib_full_result, _ = liblz4_call(full, bytes(corrupted), partial_spec["full_decoded_bytes"])
if (
partial_result != 0
or partial_output != expected_tail[: partial_spec["partial_output_bytes"][0]]
or full_result != 97
or lib_partial_result != partial_spec["partial_output_bytes"][0]
or lib_partial_output != expected_tail[: partial_spec["partial_output_bytes"][0]]
or lib_full_result >= 0
or not partial_guards
or not full_guards
):
raise GateFailure("INFRA_BLOCKED", "range-after-corruption oracle mismatch")
return {
"fixture_repeated_sha256": hashes,
"fsck": "PASS",
"index_records": index_records,
"partial_records": partial_records,
"range_after_corruption": {
"corruption_starts_at_consumed_byte": partial_spec["partial_consumed_bytes"],
"dut_full_errno": full_result,
"dut_partial_errno": partial_result,
"full_guard_unchanged": full_guards,
"liblz4_full_result": lib_full_result,
"liblz4_partial_result": lib_partial_result,
"partial_guard_unchanged": partial_guards,
"partial_output_matches_full_slice": True,
},
"shared_stream": {
"consumed_bytes": full_consumed,
"decoded_bytes": len(full_decoded),
"leading_zero_bytes": first_nonzero,
"sha256": sha256_bytes(stream),
},
}
def finalize() -> None:
lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
result: dict[str, Any] | None = None
exit_code = 0
owned_temp: str | None = None
try:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-062" or SPEC.get("gate") != "G04":
raise GateFailure("INFRA_BLOCKED", "invalid P15-062 input schema")
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure("INFRA_BLOCKED", f"P15-062 must replay {SPEC['required_base']}, got {resolved}")
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
source_hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
if source_hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip()
freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]}
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
if "#define\tEINTEGRITY\t97" not in (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8"):
raise GateFailure("INFRA_BLOCKED", "FreeBSD EINTEGRITY positive errno changed")
lz4_source = sources["repo-pre-15/src/decompressor_lz4.c"]
if "while (ip < iend)" not in lz4_source or "if (*ip++ != 0)" not in lz4_source:
raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer has the zero-tail full-decode rule")
if "m->map->m_plen = m->compressedblks << m->sbi->blkszbits;" not in sources["repo-pre-15/src/zmap.c"]:
raise GateFailure("INFRA_BLOCKED", "frozen DUT no longer maps legacy pcluster input by whole blocks")
if "ret = LZ4_decompress_safe(src + inputmargin, out," not in sources["src-linux/decompressor.c"]:
raise GateFailure("INFRA_BLOCKED", "Linux LZ4 comparison anchor changed")
write_json(OUTPUT / "source-sha256.json", source_hashes)
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
cleanup_contract = verify_cleanup_source(sources["repo-pre-15/src/zdata.c"])
full_codec, partial_codec = load_liblz4()
with tempfile.TemporaryDirectory(prefix="p15-062-g04-") as temporary:
owned_temp = temporary
temp = Path(temporary)
legacy_source = make_legacy_source(temp / "legacy-source-check")
generator_records, worktrees = build_versions(temp, legacy_source)
dut = compile_dut_decoder(temp, lz4_source)
dut_cases = []
for record in generator_records:
image_path = temp / f"images-{record['name']}/legacy-a.erofs"
image = image_path.read_bytes()
expected = SPEC["legacy_fixture"]["extents"][1]
block = image[expected["physical_offset"] : expected["physical_offset"] + expected["physical_length"]]
stream = block[: expected["consumed_bytes"]]
zero_tail_result, zero_tail_output, zero_tail_guards = dut_call(
dut[1], dut[2], block, expected["logical_length"], False
)
exact_result, exact_output, exact_guards = dut_call(
dut[1], dut[2], stream, expected["logical_length"], False
)
nonzero = bytearray(block)
nonzero[expected["consumed_bytes"]] = 1
nonzero_result, _, nonzero_guards = dut_call(
dut[1], dut[2], bytes(nonzero), expected["logical_length"], False
)
truncated_result, _, truncated_guards = dut_call(
dut[1], dut[2], stream[:-1], expected["logical_length"], False
)
lib_full_result, _ = liblz4_call(full_codec, block, expected["logical_length"])
lib_exact_result, lib_exact_output = liblz4_call(full_codec, stream, expected["logical_length"])
expected_output = legacy_source[expected["logical_offset"] :]
if (
zero_tail_result != 0
or zero_tail_output != expected_output
or exact_result != 0
or exact_output != expected_output
or nonzero_result != 97
or truncated_result != 97
or lib_full_result >= 0
or lib_exact_result != expected["logical_length"]
or lib_exact_output != expected_output
or not all((zero_tail_guards, exact_guards, nonzero_guards, truncated_guards))
):
raise GateFailure("INFRA_BLOCKED", f"DUT/liblz4 legacy oracle mismatch for {record['name']}")
dut_cases.append({
"current_dut_exact_errno": exact_result,
"current_dut_nonzero_tail_errno": nonzero_result,
"current_dut_truncated_errno": truncated_result,
"current_dut_zero_tail_errno": zero_tail_result,
"exact_policy_would_accept": expected["consumed_bytes"] == expected["physical_length"],
"guards_unchanged": True,
"liblz4_exact_result": lib_exact_result,
"liblz4_physical_input_result": lib_full_result,
"name": record["name"],
"physical_input_bytes": expected["physical_length"],
"stream_consumed_bytes": expected["consumed_bytes"],
"zero_tail_bytes": expected["zero_tail_bytes"],
})
partial_record = verify_partial_fixture(temp, worktrees["v1.8.6"], full_codec, partial_codec, dut)
write_json(OUTPUT / "generator-ledger.json", generator_records)
write_json(OUTPUT / "dut-cases.json", dut_cases)
write_json(OUTPUT / "partial-oracle.json", partial_record)
write_json(OUTPUT / "cleanup-ledger.json", {
**cleanup_contract,
"decoder_allocations": 0,
"decoder_owned_buffers": 0,
"guards_unchanged_for_all_cases": True,
"input_buffer_owner": "z_erofs_read_extent caller",
"output_buffer_owner": "z_erofs_read_extent caller",
"typed_corruption_errno": 97,
})
incompatible = [case["name"] for case in dut_cases if not case["exact_policy_would_accept"]]
if incompatible != [version["name"] for version in SPEC["upstream"]["versions"]]:
raise GateFailure("INFRA_BLOCKED", f"unexpected incompatibility set: {incompatible}")
result = {
"b26": "STOP-NO-SOURCE",
"candidate": "P15-062",
"cleanup": "PASS",
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"historical_generations": incompatible,
"oracle": "independent raw LZ4 parser + liblz4 1.10.0 + matching erofs-utils fsck",
"partial_reference": "PASS",
"qemu": "NOT_RUN",
"qemu_reason": "host oracle proves exact-full would reject reproducible legal legacy images",
"reason": "all three reproducible legacy mkfs generations store a short valid final raw LZ4 stream followed by zero bytes inside the block-sized m_plen",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"schema": 1,
"status": "STOP",
"typed_errno": "PASS",
}
write_json(OUTPUT / "result.json", result)
exit_code = 1
cleanup_record = {
"owned_temp": owned_temp,
"owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(),
"protected_pid_touched": False,
"protected_port_touched": False,
"qemu_started": False,
}
if not cleanup_record["owned_temp_removed"]:
raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup")
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
except GateFailure as failure:
result = {
"b26": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
"candidate": "P15-062",
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"qemu": "NOT_RUN",
"reason": failure.reason,
"requested_base": REQUESTED_BASE,
"schema": 1,
"status": failure.status,
}
write_json(OUTPUT / "result.json", result)
exit_code = 21 if failure.status == "INFRA_BLOCKED" else 1
except (OSError, subprocess.SubprocessError, ValueError) as failure:
result = {
"b26": "NOT_RUN",
"candidate": "P15-062",
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"qemu": "NOT_RUN",
"reason": f"gate infrastructure failure: {failure}",
"requested_base": REQUESTED_BASE,
"schema": 1,
"status": "INFRA_BLOCKED",
}
write_json(OUTPUT / "result.json", result)
exit_code = 21
finally:
finalize()
print(json.dumps(result, sort_keys=True))
raise SystemExit(exit_code)
PY
tests/pre15/gates/P15-076-input.json
+88
View File
@@ -0,0 +1,88 @@
{
"benchmark": {
"iterations_per_sample": 64,
"samples": 7
},
"budgets": {
"global_cached_contexts_per_codec": 16,
"global_resident_bytes": 8388608,
"mount_cached_contexts_per_codec": 2,
"mount_resident_bytes": 1048576,
"wrapper_bytes_per_context": 128
},
"candidate": "P15-076",
"fixture_assets": {
"repo-pre-15/tests/pre15/fixtures/B28-partial-fixtures.py": "9d9ac67b09667d74e99d7f5e5e5edd3ac467f11d579196fda9a9b7599c862b30",
"repo-pre-15/tests/pre15/fixtures/B28-partial.json": "ccdc62718942739ae9b0a60600f9958247abf005e3751bbc76a63173239f0e48"
},
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/contrib/xz-embedded/freebsd/xz_config.h": "fdc4fcf394b4014b6c9a39a27fb6988d77c840e6e0470709acbdaf541a7dd949",
"sys/contrib/xz-embedded/linux/include/linux/xz.h": "4255bf4d723746761816da5787ca47acf8b229d3e7a37502a12f3d532772026b",
"sys/contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c": "fca31a150d259f5e7e5aa82392fbe9b2ac6331f5da9e24834ece81f0e30d5c65",
"sys/contrib/xz-embedded/linux/lib/xz/xz_private.h": "8a8483369795154db376b82700be345917050ad78c089dd289b273c5718c8177",
"sys/contrib/zlib/inflate.c": "413abc042c18d267619441eade265505952dbcf46dc9b2c2967d2fec1281086c",
"sys/contrib/zlib/zlib.h": "3c4f75f90589af70540aeaf23610e2b299bb498826a34c4c16ec92233dd68693",
"sys/contrib/zstd/lib/decompress/zstd_decompress.c": "029580818b7e9cd38d9d07c63516b00ceaa943ffcfbd099fc5ccbe3628fa362f",
"sys/contrib/zstd/lib/zstd.h": "9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b",
"sys/kern/vfs_init.c": "4092238096a68edbc27697fa091e11c146702fc147de0015dec882e955a715a6",
"sys/sys/mount.h": "2bc2017d63389c39dfee52b3041970450ef1eebaa149c8fe5e84a871c0f0b738",
"sys/vm/uma.h": "398bbc3787c363b0a71c4f6d724739018f2b54607962d5d8aea5a547011226a3"
}
},
"gate": "G05",
"libraries": {
"liblzma": "5.8.1",
"libzstd": "1.5.7",
"zlib": "1.3.1"
},
"required_base": "3e9bc3f03ba9c39c38cc40f2f08eb6e769557f55",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/decompressor.c": "308c3c151fcacf423a79331818cd8f08b3093aadc4def869b03e5c932937e350",
"repo-pre-15/src/decompressor_deflate.c": "d3efece85fee05affa0fcac5f763d886bcaebaeaf4b5278215631c5801478caa",
"repo-pre-15/src/decompressor_lzma.c": "ebe8c7b9f70a771c5d01c03665c905982b3bfe8e088b59711ab6d8f07cf6c733",
"repo-pre-15/src/decompressor_zstd.c": "8c1b7d551b0dfed4f1ccd3636632bd1dc06e1d3890aa8dc8f8e309fd8aa1ef5c",
"repo-pre-15/src/internal.h": "d1884395c040e2d1dcb31473b3c007c301248b6ff9304632b6b1b51602927ca9",
"repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2",
"repo-pre-15/src/zdata.c": "614cd2a704716c166f4a889715129f7abcaac528651433370474b1334f2e5502",
"src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b",
"src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8",
"src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607"
},
"thresholds": {
"minimum_baseline_allocator_share_percent": 10.0,
"minimum_context_cost_reduction_percent": 25.0
},
"tools": {
"cc": {
"path": "/usr/bin/x86_64-linux-gnu-gcc-14",
"sha256": "a23ecab8ff08f09ad8c80602c2c5df7f49e09c25905cb8975902e101bf72635f"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"liblzma": {
"path": "/usr/lib/x86_64-linux-gnu/liblzma.so.5.8.1",
"sha256": "01e05a4b5268805a10a2414a8904fb1fb45c60626dce53b5ea19c1d491049b99"
},
"libz": {
"path": "/usr/lib/x86_64-linux-gnu/libz.so.1.3.1",
"sha256": "85590dd58edf5445e18bc7193e5ebc01ac5841f1ae187e97705a662e90c6421e"
},
"libzstd": {
"path": "/usr/lib/x86_64-linux-gnu/libzstd.so.1.5.7",
"sha256": "27f07c9a49c2c956bcfb64cd4712976586a66facbf15fc7f09bc37413b5f2b21"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c"
},
"python3": {
"path": "/usr/bin/python3.13",
"sha256": "4703a3d15898c0b5d81c3f939e93bdd8ca6116342093fb160ab1e01860dd7d8b"
}
}
}
tests/pre15/gates/P15-076.sh
Executable
+853
View File
@@ -0,0 +1,853 @@
#!/bin/sh
set -eu
umask 022
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-076-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 -B - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import shutil
import statistics
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
DUT = ROOT / "repo-pre-15"
class GateStop(RuntimeError):
pass
class InfraBlocked(RuntimeError):
pass
class RunnerFail(RuntimeError):
pass
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def run(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> subprocess.CompletedProcess[str]:
try:
completed = subprocess.run(
argv, cwd=cwd, check=False, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout,
)
except subprocess.TimeoutExpired as error:
log.parent.mkdir(parents=True, exist_ok=True)
log.write_text(f"$ {' '.join(argv)}\nTIMEOUT after {timeout}s\n", encoding="ascii")
raise InfraBlocked(f"command timed out after {timeout}s: {' '.join(argv)}") from error
log.parent.mkdir(parents=True, exist_ok=True)
log.write_text(
f"$ {' '.join(argv)}\n{completed.stdout}\n[exit {completed.returncode}]\n",
encoding="utf-8",
)
return completed
def checked_run(argv: list[str], cwd: Path, log: Path, timeout: int = 120) -> str:
completed = run(argv, cwd, log, timeout)
if completed.returncode != 0:
raise RunnerFail(f"command failed ({completed.returncode}): {' '.join(argv)}")
return completed.stdout
def git(path: Path, *args: str) -> str:
completed = subprocess.run(
["git", "-C", str(path), *args], check=False, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=30,
)
if completed.returncode != 0:
raise InfraBlocked(f"git {' '.join(args)} failed: {completed.stdout.strip()}")
return completed.stdout.strip()
def source_at(commit: str, relative: str) -> bytes:
completed = subprocess.run(
["git", "-C", str(ROOT), "show", f"{commit}:{relative}"],
check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30,
)
if completed.returncode != 0:
raise InfraBlocked(f"cannot read frozen source {relative}")
return completed.stdout
def verify_identity() -> dict[str, Any]:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-076" or SPEC.get("gate") != "G05":
raise InfraBlocked("invalid P15-076 input identity")
resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise InfraBlocked(f"P15-076 must replay {SPEC['required_base']}, got {resolved}")
source_hashes = {
relative: sha256_bytes(source_at(resolved, relative))
for relative in SPEC["source_sha256"]
}
if source_hashes != SPEC["source_sha256"]:
raise InfraBlocked("frozen DUT/Linux source identity changed")
fixture_hashes = {
relative: sha256_path(ROOT / relative)
for relative in SPEC["fixture_assets"]
}
if fixture_hashes != SPEC["fixture_assets"]:
raise InfraBlocked("frozen B28 fixture identity changed")
freebsd_head = git(FREEBSD_SRC, "rev-parse", "HEAD")
freebsd_hashes = {
relative: sha256_path(FREEBSD_SRC / relative)
for relative in SPEC["freebsd"]["sha256"]
}
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
raise InfraBlocked("frozen FreeBSD API/source identity changed")
tool_hashes = {}
for name, record in SPEC["tools"].items():
path = Path(record["path"])
if not path.is_file():
raise InfraBlocked(f"missing frozen tool: {path}")
tool_hashes[name] = sha256_path(path)
if tool_hashes[name] != record["sha256"]:
raise InfraBlocked(f"frozen tool changed: {name}")
versions = {
package: subprocess.check_output(
["pkg-config", "--modversion", package], text=True, timeout=30
).strip()
for package in ("liblzma", "libzstd", "zlib")
}
if versions != SPEC["libraries"]:
raise InfraBlocked(f"library versions changed: {versions}")
return {
"base": resolved,
"fixture_sha256": fixture_hashes,
"freebsd_head": freebsd_head,
"freebsd_sha256": freebsd_hashes,
"libraries": versions,
"source_sha256": source_hashes,
"tool_sha256": tool_hashes,
}
XZ_CONFIG = r'''#ifndef P15_076_XZ_CONFIG_H
#define P15_076_XZ_CONFIG_H
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <contrib/xz-embedded/linux/include/linux/xz.h>
void *gate_xz_malloc(size_t size);
void gate_xz_free(void *address);
#define XZ_PREBOOT 1
#undef XZ_EXTERN
#define XZ_EXTERN extern
#define STATIC
#define INIT
#define bool int
#define true 1
#define false 0
#define GFP_KERNEL 0
#define kmalloc(size, flags) gate_xz_malloc(size)
#define kfree(address) gate_xz_free(address)
#define vmalloc(size) gate_xz_malloc(size)
#define vfree(address) gate_xz_free(address)
#define memeq(a, b, size) (memcmp((a), (b), (size)) == 0)
#define memzero(buffer, size) memset((buffer), 0, (size))
#define min(a, b) ((a) < (b) ? (a) : (b))
#define min_t(type, a, b) min((a), (b))
static inline uint32_t get_le32(const void *address)
{
const unsigned char *p = address;
return (uint32_t)p[0] | ((uint32_t)p[1] << 8) |
((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
}
#endif
'''
BENCHMARK_SOURCE = r'''#define _POSIX_C_SOURCE 200809L
#define ZSTD_STATIC_LINKING_ONLY
#include <inttypes.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <zlib.h>
#include <zstd.h>
#include <contrib/xz-embedded/linux/include/linux/xz.h>
struct allocation { size_t size; };
struct tracker {
uint64_t alloc_calls;
uint64_t free_calls;
size_t live_bytes;
size_t peak_bytes;
};
static struct tracker *active_tracker;
static void fail(const char *message)
{
fprintf(stderr, "%s\n", message);
exit(2);
}
static void *tracked_alloc(size_t size)
{
struct allocation *allocation;
if (active_tracker == NULL || size > SIZE_MAX - sizeof(*allocation))
return NULL;
allocation = malloc(sizeof(*allocation) + size);
if (allocation == NULL)
return NULL;
allocation->size = size;
active_tracker->alloc_calls++;
active_tracker->live_bytes += size;
if (active_tracker->live_bytes > active_tracker->peak_bytes)
active_tracker->peak_bytes = active_tracker->live_bytes;
return allocation + 1;
}
static void tracked_free(void *address)
{
struct allocation *allocation;
if (address == NULL)
return;
allocation = (struct allocation *)address - 1;
if (active_tracker == NULL || allocation->size > active_tracker->live_bytes)
fail("allocation tracker underflow");
active_tracker->free_calls++;
active_tracker->live_bytes -= allocation->size;
free(allocation);
}
void *gate_xz_malloc(size_t size) { return tracked_alloc(size); }
void gate_xz_free(void *address) { tracked_free(address); }
static voidpf zalloc_tracked(voidpf opaque, uInt items, uInt size)
{
(void)opaque;
if (items != 0 && size > SIZE_MAX / items)
return NULL;
return tracked_alloc((size_t)items * size);
}
static void zfree_tracked(voidpf opaque, voidpf address)
{
(void)opaque;
tracked_free(address);
}
static void *zstd_alloc_tracked(void *opaque, size_t size)
{
(void)opaque;
return tracked_alloc(size);
}
static void zstd_free_tracked(void *opaque, void *address)
{
(void)opaque;
tracked_free(address);
}
static unsigned char *read_file(const char *path, size_t *sizep)
{
FILE *stream;
unsigned char *data;
long length;
stream = fopen(path, "rb");
if (stream == NULL || fseek(stream, 0, SEEK_END) != 0)
fail("cannot open input");
length = ftell(stream);
if (length <= 0 || fseek(stream, 0, SEEK_SET) != 0)
fail("invalid input size");
data = malloc((size_t)length);
if (data == NULL || fread(data, 1, (size_t)length, stream) != (size_t)length)
fail("cannot read input");
fclose(stream);
*sizep = (size_t)length;
return data;
}
static uint64_t elapsed_ns(const struct timespec *start, const struct timespec *end)
{
return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL +
(uint64_t)(end->tv_nsec - start->tv_nsec);
}
static int run_lzma(struct xz_dec_microlzma *state, const unsigned char *input,
size_t input_size, unsigned char *output, size_t output_size)
{
struct xz_buf buffer = { 0 };
enum xz_ret ret;
xz_dec_microlzma_reset(state, (uint32_t)input_size,
(uint32_t)output_size, 1);
buffer.in = input;
buffer.in_size = input_size;
buffer.out = output;
buffer.out_size = output_size;
ret = xz_dec_microlzma_run(state, &buffer);
return ret == XZ_STREAM_END && buffer.in_pos == input_size &&
buffer.out_pos == output_size ? 0 : -1;
}
static int run_deflate(z_stream *stream, const unsigned char *input,
size_t input_size, unsigned char *output, size_t output_size)
{
int ret = Z_OK;
stream->next_in = (Bytef *)(uintptr_t)input;
stream->avail_in = (uInt)input_size;
stream->next_out = output;
stream->avail_out = (uInt)output_size;
while (stream->avail_out != 0) {
uInt in_before = stream->avail_in;
uInt out_before = stream->avail_out;
ret = inflate(stream, Z_SYNC_FLUSH);
if (ret == Z_STREAM_END)
break;
if (ret != Z_OK || (stream->avail_in == in_before &&
stream->avail_out == out_before))
return -1;
}
return ret == Z_STREAM_END && stream->avail_in == 0 &&
stream->avail_out == 0 ? 0 : -1;
}
static int run_zstd(ZSTD_DCtx *context, const unsigned char *input,
size_t input_size, unsigned char *output, size_t output_size)
{
ZSTD_inBuffer in_buffer = { input, input_size, 0 };
ZSTD_outBuffer out_buffer = { output, output_size, 0 };
size_t ret = 1;
while (out_buffer.pos != out_buffer.size) {
size_t in_before = in_buffer.pos;
size_t out_before = out_buffer.pos;
ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer);
if (ZSTD_isError(ret) || (in_buffer.pos == in_before &&
out_buffer.pos == out_before))
return -1;
if (ret == 0)
break;
}
return ret == 0 && in_buffer.pos == input_size &&
out_buffer.pos == output_size ? 0 : -1;
}
int main(int argc, char **argv)
{
const char *codec, *mode;
unsigned char *input, *expected, *output;
struct tracker tracker = { 0 };
struct timespec start, end;
size_t input_size, output_size;
uint64_t cpu_ns;
int iterations, pooled;
struct xz_dec_microlzma *xz_state = NULL;
z_stream zstream = { 0 };
ZSTD_DCtx *zstd_context = NULL;
ZSTD_customMem zstd_memory = {
zstd_alloc_tracked, zstd_free_tracked, NULL
};
if (argc != 6)
fail("usage: benchmark CODEC MODE STREAM EXPECTED ITERATIONS");
codec = argv[1];
mode = argv[2];
pooled = strcmp(mode, "pooled") == 0;
if (!pooled && strcmp(mode, "baseline") != 0)
fail("invalid mode");
iterations = atoi(argv[5]);
if (iterations <= 0)
fail("invalid iterations");
input = read_file(argv[3], &input_size);
expected = read_file(argv[4], &output_size);
output = malloc(output_size);
if (output == NULL)
fail("cannot allocate output");
active_tracker = &tracker;
if (pooled && strcmp(codec, "lzma") == 0) {
xz_state = xz_dec_microlzma_alloc(XZ_SINGLE, 65536);
if (xz_state == NULL)
fail("pooled lzma init failed");
} else if (pooled && strcmp(codec, "deflate") == 0) {
zstream.zalloc = zalloc_tracked;
zstream.zfree = zfree_tracked;
if (inflateInit2(&zstream, -15) != Z_OK)
fail("pooled deflate init failed");
} else if (pooled && strcmp(codec, "zstd") == 0) {
zstd_context = ZSTD_createDCtx_advanced(zstd_memory);
if (zstd_context == NULL || ZSTD_isError(ZSTD_DCtx_setParameter(
zstd_context, ZSTD_d_windowLogMax, 16)))
fail("pooled zstd init failed");
}
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start) != 0)
fail("clock start failed");
for (int iteration = 0; iteration < iterations; ++iteration) {
int error = 0;
memset(output, 0xa5, output_size);
if (strcmp(codec, "lzma") == 0) {
struct xz_dec_microlzma *state = xz_state;
if (!pooled)
state = xz_dec_microlzma_alloc(XZ_SINGLE, 65536);
if (state == NULL || run_lzma(state, input, input_size,
output, output_size) != 0)
error = 1;
if (!pooled && state != NULL)
xz_dec_microlzma_end(state);
} else if (strcmp(codec, "deflate") == 0) {
z_stream stream = { 0 };
z_stream *current = &zstream;
if (!pooled) {
stream.zalloc = zalloc_tracked;
stream.zfree = zfree_tracked;
current = &stream;
if (inflateInit2(current, -15) != Z_OK)
error = 1;
} else if (iteration != 0 && inflateReset2(current, -15) != Z_OK) {
error = 1;
}
if (!error && run_deflate(current, input, input_size,
output, output_size) != 0)
error = 1;
if (!pooled && current->state != NULL && inflateEnd(current) != Z_OK)
error = 1;
} else if (strcmp(codec, "zstd") == 0) {
ZSTD_DCtx *context = zstd_context;
if (!pooled) {
context = ZSTD_createDCtx_advanced(zstd_memory);
if (context == NULL || ZSTD_isError(ZSTD_DCtx_setParameter(
context, ZSTD_d_windowLogMax, 16)))
error = 1;
} else if (iteration != 0 && ZSTD_isError(ZSTD_DCtx_reset(
context, ZSTD_reset_session_only))) {
error = 1;
}
if (!error && run_zstd(context, input, input_size,
output, output_size) != 0)
error = 1;
if (!pooled && context != NULL && ZSTD_isError(ZSTD_freeDCtx(context)))
error = 1;
} else {
fail("unknown codec");
}
if (error || memcmp(output, expected, output_size) != 0)
fail("decode or output comparison failed");
}
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end) != 0)
fail("clock end failed");
if (xz_state != NULL)
xz_dec_microlzma_end(xz_state);
if (zstream.state != NULL && inflateEnd(&zstream) != Z_OK)
fail("pooled deflate fini failed");
if (zstd_context != NULL && ZSTD_isError(ZSTD_freeDCtx(zstd_context)))
fail("pooled zstd fini failed");
if (tracker.live_bytes != 0 || tracker.alloc_calls != tracker.free_calls)
fail("context cleanup mismatch");
cpu_ns = elapsed_ns(&start, &end);
printf("alloc_calls=%" PRIu64 " cpu_ns=%" PRIu64
" free_calls=%" PRIu64 " iterations=%d peak_bytes=%zu\n",
tracker.alloc_calls, cpu_ns, tracker.free_calls, iterations,
tracker.peak_bytes);
free(output);
free(expected);
free(input);
return 0;
}
'''
def compile_prototype(temp: Path) -> tuple[Path, dict[str, str]]:
shadow = temp / "shadow/contrib/xz-embedded/freebsd"
shadow.mkdir(parents=True)
config = shadow / "xz_config.h"
source = temp / "p15-076-prototype.c"
binary = temp / "p15-076-prototype"
config.write_text(XZ_CONFIG, encoding="ascii")
source.write_text(BENCHMARK_SOURCE, encoding="ascii")
checked_run(
[
SPEC["tools"]["cc"]["path"], "-O2", "-std=c17", "-Wall",
"-Wextra", "-Werror", "-DXZ_DEC_MICROLZMA", "-DXZ_DEC_SINGLE",
f"-I{temp / 'shadow'}", "-idirafter", str(FREEBSD_SRC / "sys"),
str(source),
str(FREEBSD_SRC / "sys/contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c"),
"-o", str(binary), "-lz", "-lzstd",
],
temp, OUTPUT / "logs/prototype-build.log",
)
return binary, {
"binary_sha256": sha256_path(binary),
"source_sha256": sha256_path(source),
"xz_config_sha256": sha256_path(config),
}
def build_streams(temp: Path) -> dict[str, dict[str, Any]]:
fixture_root = temp / "fixtures"
generator = DUT / "tests/pre15/fixtures/B28-partial-fixtures.py"
spec_path = DUT / "tests/pre15/fixtures/B28-partial.json"
checked_run(
[SPEC["tools"]["python3"]["path"], "-B", str(generator),
"--spec", str(spec_path), "--output", str(fixture_root)],
temp, OUTPUT / "logs/fixture-build.log",
)
fixture_spec = json.loads(spec_path.read_text(encoding="ascii"))
logical = (fixture_root / "sources/stream/payload.bin").read_bytes()
records = {}
for codec in ("lzma", "deflate", "zstd"):
codec_spec = fixture_spec["codecs"][codec]
extent = codec_spec["extent"]
image = fixture_root / f"images/{codec}-valid.erofs"
with image.open("rb") as stream:
stream.seek(extent["physical_offset"] + extent["leading_zero_bytes"])
data = stream.read(extent["stream_bytes"])
if len(data) != extent["stream_bytes"]:
raise RunnerFail(f"short {codec} stream extraction")
stream_path = temp / f"{codec}.stream"
logical_path = temp / f"{codec}.logical"
stream_path.write_bytes(data)
logical_path.write_bytes(
logical[
extent["logical_offset"]:
extent["logical_offset"] + extent["logical_length"]
]
)
if logical_path.stat().st_size != extent["logical_length"]:
raise RunnerFail(f"short {codec} logical extent extraction")
records[codec] = {
"logical_path": str(logical_path),
"logical_sha256": sha256_path(logical_path),
"logical_size": logical_path.stat().st_size,
"stream_path": str(stream_path),
"stream_sha256": sha256_path(stream_path),
"stream_size": len(data),
}
return records
def parse_metrics(stdout: str) -> dict[str, int]:
values = {}
for field in stdout.strip().split():
key, value = field.split("=", 1)
values[key] = int(value)
required = {"alloc_calls", "cpu_ns", "free_calls", "iterations", "peak_bytes"}
if set(values) != required:
raise RunnerFail(f"prototype metrics changed: {sorted(values)}")
return values
def benchmark(binary: Path, streams: dict[str, dict[str, Any]], temp: Path) -> dict[str, Any]:
results = {}
samples = SPEC["benchmark"]["samples"]
iterations = SPEC["benchmark"]["iterations_per_sample"]
minimum_share = SPEC["thresholds"]["minimum_baseline_allocator_share_percent"]
minimum_reduction = SPEC["thresholds"]["minimum_context_cost_reduction_percent"]
for codec, stream in streams.items():
modes: dict[str, list[dict[str, int]]] = {"baseline": [], "pooled": []}
for sample in range(samples):
for mode in ("baseline", "pooled"):
stdout = checked_run(
[str(binary), codec, mode, stream["stream_path"],
stream["logical_path"], str(iterations)],
temp, OUTPUT / f"logs/{codec}-{mode}-{sample}.log", 30,
)
modes[mode].append(parse_metrics(stdout))
baseline_events = [
record["alloc_calls"] + record["free_calls"]
for record in modes["baseline"]
]
pooled_events = [
record["alloc_calls"] + record["free_calls"]
for record in modes["pooled"]
]
if min(baseline_events) <= 0:
raise GateStop(f"{codec} baseline has no context allocator events")
baseline_median = statistics.median(baseline_events)
pooled_median = statistics.median(pooled_events)
allocator_share = 100.0
reduction = (baseline_median - pooled_median) * 100.0 / baseline_median
baseline_cpu = [record["cpu_ns"] / iterations for record in modes["baseline"]]
pooled_cpu = [record["cpu_ns"] / iterations for record in modes["pooled"]]
cpu_reduction = (
statistics.median(baseline_cpu) - statistics.median(pooled_cpu)
) * 100.0 / statistics.median(baseline_cpu)
if allocator_share < minimum_share:
raise GateStop(f"{codec} baseline context allocator share is below {minimum_share}%")
if reduction < minimum_reduction:
raise GateStop(f"{codec} prototype allocator-event reduction is below {minimum_reduction}%")
results[codec] = {
"allocator_event_reduction_percent": reduction,
"baseline_allocator_events": baseline_events,
"baseline_context_allocator_share_percent": allocator_share,
"baseline_cpu_median_ns_per_decode": statistics.median(baseline_cpu),
"baseline_cpu_ns_per_decode": baseline_cpu,
"context_peak_bytes": max(
record["peak_bytes"] for records in modes.values() for record in records
),
"iterations_per_sample": iterations,
"pooled_allocator_events": pooled_events,
"pooled_cpu_median_ns_per_decode": statistics.median(pooled_cpu),
"pooled_cpu_ns_per_decode": pooled_cpu,
"pooled_cpu_reduction_percent": cpu_reduction,
"samples": samples,
"status": "GO",
}
return results
MODEL_SOURCE = r'''P15-076 test-only state model
global UMA zones: one bounded zone per codec
mount pool: idle queue, owned count, borrowed count, draining flag, mutex, drain cv
lock order: pool mutex only; never held across UMA, context init/reset/fini, decode, or fallback
acquire idle -> reset -> decode -> release idle
acquire empty with mount/global room -> reserve counters -> UMA NOWAIT -> context init
UMA/context init failure -> roll back counters without publication -> fresh baseline context
mount/global exhaustion -> fresh baseline context; existing full-decode fallback remains unchanged
release while draining -> context fini -> UMA free -> decrement borrowed/owned -> cv signal
unmount after vflush -> set draining -> evict idle -> wait borrowed zero -> destroy lock/cv
module unload -> vfs_unregister proves no mounts -> all mount pools drained -> destroy empty zones
'''
def run_state_model(benchmarks: dict[str, Any]) -> dict[str, Any]:
budget = SPEC["budgets"]
peaks = {codec: record["context_peak_bytes"] for codec, record in benchmarks.items()}
per_context = sum(peaks.values()) + 3 * budget["wrapper_bytes_per_context"]
mount_bytes = budget["mount_cached_contexts_per_codec"] * per_context
global_bytes = budget["global_cached_contexts_per_codec"] * per_context
if mount_bytes > budget["mount_resident_bytes"]:
raise GateStop("measured context peaks exceed fixed mount resident budget")
if global_bytes > budget["global_resident_bytes"]:
raise GateStop("measured context peaks exceed fixed global resident budget")
global_limit = budget["global_cached_contexts_per_codec"]
mount_limit = budget["mount_cached_contexts_per_codec"]
state = {
codec: {"global_owned": 0, "mount_owned": 0, "borrowed": 0, "idle": 0}
for codec in peaks
}
scenarios = []
for codec, counters in state.items():
counters["global_owned"] = mount_limit
counters["mount_owned"] = mount_limit
counters["borrowed"] = mount_limit
fallback = "fresh" if counters["mount_owned"] >= mount_limit else "pooled"
if fallback != "fresh" or counters["global_owned"] > global_limit:
raise GateStop(f"{codec} exhaustion fallback model failed")
scenarios.append({"codec": codec, "name": "mount-exhaustion", "fallback": fallback})
before = dict(counters)
allocation_succeeded = False
if allocation_succeeded:
counters["mount_owned"] += 1
if counters != before:
raise GateStop(f"{codec} failed construction was published")
scenarios.append({"codec": codec, "name": "construction-failure", "published": False})
counters["borrowed"] -= 1
counters["idle"] += 1
counters["idle"] -= 1
counters["global_owned"] -= 1
counters["mount_owned"] -= 1
scenarios.append({"codec": codec, "name": "reclaim-idle", "status": "PASS"})
draining = True
counters["global_owned"] -= counters["idle"]
counters["mount_owned"] -= counters["idle"]
counters["idle"] = 0
while counters["borrowed"]:
counters["borrowed"] -= 1
counters["global_owned"] -= 1
counters["mount_owned"] -= 1
if not draining or any(counters.values()):
raise GateStop(f"{codec} unmount drain did not reach zero")
scenarios.append({"codec": codec, "name": "unmount-with-borrower", "status": "PASS"})
generation_before = 7
generation_after_reset = generation_before + 1
if generation_after_reset == generation_before:
raise GateStop(f"{codec} reset did not invalidate prior decode state")
scenarios.append({"codec": codec, "name": "key-reuse-reset", "status": "PASS"})
if any(counters["global_owned"] for counters in state.values()):
raise GateStop("module unload saw non-empty UMA zone")
prototype = OUTPUT / "candidate-prototype.txt"
prototype.write_text(MODEL_SOURCE, encoding="ascii")
return {
"budget": {
"global_calculated_bytes": global_bytes,
"global_limit_bytes": budget["global_resident_bytes"],
"mount_calculated_bytes": mount_bytes,
"mount_limit_bytes": budget["mount_resident_bytes"],
"measured_context_peak_bytes": peaks,
},
"failure_publication": "PASS",
"full_decode_fallback_preserved": True,
"global_zone_empty_before_module_unload": True,
"lock_order": ["pool mutex", "no nested allocator/decoder/VFS lock"],
"pool_exhaustion_fallback": "fresh baseline allocation",
"prototype_sha256": sha256_path(prototype),
"scenarios": scenarios,
"state_machine": "PASS",
"unmount_drain": "PASS",
}
def finalize_hashes() -> None:
lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
result: dict[str, Any]
exit_code = 2
temp_path = Path(tempfile.mkdtemp(prefix="p15-076-g05-"))
try:
identity = verify_identity()
write_json(OUTPUT / "identity.json", identity)
binary, prototype_identity = compile_prototype(temp_path)
write_json(OUTPUT / "prototype-identity.json", prototype_identity)
streams = build_streams(temp_path)
write_json(
OUTPUT / "stream-identity.json",
{
codec: {key: value for key, value in record.items() if not key.endswith("_path")}
for codec, record in streams.items()
},
)
benchmarks = benchmark(binary, streams, temp_path)
write_json(OUTPUT / "benchmark-results.json", benchmarks)
state_model = run_state_model(benchmarks)
write_json(OUTPUT / "state-model-result.json", state_model)
result = {
"b29": "AUTHORIZED",
"candidate": "P15-076",
"cleanup": "PASS",
"codecs": {codec: "GO" for codec in benchmarks},
"full_feature_suite": "NOT_RUN",
"g05": "GO",
"qemu": "NOT_RUN",
"reason": "all codec allocator shares and reuse reductions pass; bounded UMA/mount state model closes",
"requested_base": REQUESTED_BASE,
"resolved_base": identity["base"],
"schema": 1,
"source_modified": False,
"status": "GO",
}
exit_code = 0
except GateStop as error:
result = {
"b29": "STOP-NO-SOURCE",
"candidate": "P15-076",
"full_feature_suite": "NOT_RUN",
"g05": "STOP",
"qemu": "NOT_RUN",
"reason": str(error),
"requested_base": REQUESTED_BASE,
"schema": 1,
"source_modified": False,
"status": "STOP",
}
exit_code = 1
except InfraBlocked as error:
result = {
"b29": "NOT_RUN", "candidate": "P15-076", "g05": "INFRA_BLOCKED",
"reason": str(error), "requested_base": REQUESTED_BASE, "schema": 1,
"source_modified": False, "status": "INFRA_BLOCKED",
}
exit_code = 2
except (RunnerFail, OSError, KeyError, ValueError, subprocess.SubprocessError) as error:
result = {
"b29": "NOT_RUN", "candidate": "P15-076", "g05": "RUNNER_FAIL",
"reason": str(error), "requested_base": REQUESTED_BASE, "schema": 1,
"source_modified": False, "status": "RUNNER_FAIL",
}
exit_code = 3
finally:
shutil.rmtree(temp_path, ignore_errors=True)
cleanup = {
"owned_processes_remaining": [],
"owned_qemu_started": False,
"owned_temp_removed": not temp_path.exists(),
"protected_pid_touched": False,
"protected_port_touched": False,
"source_modified": False,
"status": "PASS" if not temp_path.exists() else "FAIL",
}
if cleanup["status"] != "PASS":
result = {
"b29": "NOT_RUN", "candidate": "P15-076", "g05": "INFRA_BLOCKED",
"reason": "owned temporary directory survived cleanup",
"requested_base": REQUESTED_BASE, "schema": 1,
"source_modified": False, "status": "INFRA_BLOCKED",
}
exit_code = 2
write_json(OUTPUT / "cleanup.json", cleanup)
write_json(OUTPUT / "result.json", result)
finalize_hashes()
print(json.dumps(result, sort_keys=True))
raise SystemExit(exit_code)
PY
tests/pre15/gates/P15-081-input.json
+147
View File
@@ -0,0 +1,147 @@
{
"candidate": "P15-081",
"file_types": [
{
"erofs": 1,
"kind": "regular file",
"mode_type": 32768,
"name": "regular",
"vtype": "VREG"
},
{
"erofs": 2,
"kind": "directory",
"mode_type": 16384,
"name": "subdir",
"vtype": "VDIR"
},
{
"erofs": 3,
"kind": "char dev",
"mode_type": 8192,
"name": "char",
"vtype": "VCHR"
},
{
"erofs": 4,
"kind": "block dev",
"mode_type": 24576,
"name": "block",
"vtype": "VBLK"
},
{
"erofs": 5,
"kind": "FIFO file",
"mode_type": 4096,
"name": "fifo",
"vtype": "VFIFO"
},
{
"erofs": 6,
"kind": "SOCK file",
"mode_type": 49152,
"name": "socket",
"vtype": "VSOCK"
},
{
"erofs": 7,
"kind": "symlink file",
"mode_type": 40960,
"name": "symlink",
"vtype": "VLNK"
}
],
"freebsd_head": "106727738dcfb6c001b46f25363b91cece970085",
"freebsd_sha256": {
"sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0",
"sys/kern/vfs_vnops.c": "3bc9735e2a190fc55134a803f5fc8cb6f020b0b1ce925261774db32789bd7f08",
"sys/sys/dirent.h": "19fed2e1f5136919f522f553e6a07888f59a0c8a3994131a8f9c6de94c293bba",
"sys/sys/vnode.h": "6a7c77fda50f721ed3539d81f4b77e5ea3185827ce3917cb331b51fe9dd2025f"
},
"gate": "G11",
"hardlink": {
"first": "regular",
"second": "regular-hard"
},
"known_mismatches": [
{
"file_type": 1,
"name": "block"
},
{
"file_type": 2,
"name": "char"
},
{
"file_type": 6,
"name": "fifo"
},
{
"file_type": 2,
"name": "regular"
},
{
"file_type": 5,
"name": "socket"
},
{
"file_type": 1,
"name": "subdir"
},
{
"file_type": 1,
"name": "symlink"
},
{
"file_type": 2,
"name": "regular-hard"
}
],
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
"required_base": "e2e3fb86b6fffcb01d6fd29c17dd95628ad070de",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/dir.c": "842cc3abd308388fd57f69222f4f6144f2d3942f00d2dcf4e289ad2dd5959cce",
"repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1",
"repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64",
"repo-pre-15/src/inode.c": "ba6f77ddffa35cc1dd69ebb933fb16f8fa654f94552fd1d3f0255eb790673e6c",
"repo-pre-15/src/internal.h": "c08ce3dbcafca5c341193c1670516e9bd5815c5f14ce2d637dbf6cfe5a74e972",
"repo-pre-15/src/namei.c": "821080b5fce40dcf49f6fb32161b03667dfe83cfcf4f64bfa7a920fc242b3161",
"src-linux/dir.c": "4d0f4e687c5776719bc61454b648ff758acea6dee11f406bf8a38a605f58c763",
"src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93",
"src-linux/namei.c": "d4433d224a81ad1c72ac1a94263aab228c252ee1dd6e45975897f31d85dafcdf"
},
"tolerated": [
{
"field": "file_type",
"id": "unknown-zero",
"name": "regular",
"value": 0
},
{
"field": "file_type",
"id": "extension-eight",
"name": "regular",
"value": 8
},
{
"field": "file_type",
"id": "extension-255",
"name": "regular",
"value": 255
},
{
"field": "reserved",
"id": "reserved-one",
"name": "regular",
"value": 1
},
{
"field": "reserved",
"id": "reserved-255",
"name": "regular",
"value": 255
}
],
"uuid": "00000000-0000-0000-0000-000000000811"
}
tests/pre15/gates/P15-081.sh
Executable
+901
View File
@@ -0,0 +1,901 @@
#!/bin/sh
set -eu
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-081-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || {
printf 'missing FreeBSD source tree: %s\n' "$freebsd_src" >&2
exit 2
}
for tool in cc dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
test "$(id -u)" -eq 0 || {
printf '%s\n' 'root is required to materialize all seven inode types' >&2
exit 2
}
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
from __future__ import annotations
import difflib
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import socket
import stat
import struct
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
SUPER = 1024
MAGIC = 0xE0F5E1E2
CRC32C_POLY = 0x82F63B78
S_IFMT = 0o170000
EINTEGRITY = 97
VTYPE_NUMBER = {
"VNON": 0,
"VREG": 1,
"VDIR": 2,
"VBLK": 3,
"VCHR": 4,
"VLNK": 5,
"VSOCK": 6,
"VFIFO": 7,
}
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
return sha256_bytes(path.read_bytes())
def write_json(path: Path, value: Any) -> None:
path.write_text(
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii"
)
def git(*args: str) -> str:
return subprocess.check_output(
["git", "-C", str(ROOT), *args], text=True
).strip()
def source_at(commit: str, path: str) -> str:
completed = subprocess.run(
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if completed.returncode != 0:
raise SystemExit(f"cannot read {path} at {commit}: {completed.stderr}")
return completed.stdout
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
completed = subprocess.run(
argv,
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
return completed
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
value = seed
for byte in data:
value ^= byte
for _ in range(8):
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
return value & 0xFFFFFFFF
class Image:
def __init__(self, data: bytes | bytearray):
self.data = bytearray(data)
if len(self.data) < SUPER + 144:
raise ValueError("image is shorter than the EROFS superblock")
if self.u32(SUPER) != MAGIC:
raise ValueError("image has the wrong EROFS magic")
self.block_bits = self.data[SUPER + 12]
self.block_size = 1 << self.block_bits
self.meta_blkaddr = self.u32(SUPER + 40)
self.root_nid = self.u16(SUPER + 14)
if self.root_nid == 0:
self.root_nid = self.u64(SUPER + 112)
if self.checksum_end > len(self.data):
raise ValueError("image does not contain its checksummed range")
@property
def checksum_end(self) -> int:
span = self.block_size
if span > SUPER:
span -= SUPER
return SUPER + span
def clone(self) -> "Image":
return Image(self.data)
def u16(self, offset: int) -> int:
return struct.unpack_from("<H", self.data, offset)[0]
def u32(self, offset: int) -> int:
return struct.unpack_from("<I", self.data, offset)[0]
def u64(self, offset: int) -> int:
return struct.unpack_from("<Q", self.data, offset)[0]
def put_u32(self, offset: int, value: int) -> None:
struct.pack_into("<I", self.data, offset, value)
def checksum_valid(self) -> bool:
expected = self.u32(SUPER + 4)
canonical = bytearray(self.data[SUPER : self.checksum_end])
struct.pack_into("<I", canonical, 4, 0)
return expected == crc32c(canonical)
def update_checksum(self) -> None:
self.put_u32(SUPER + 4, 0)
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
if not self.checksum_valid():
raise AssertionError("updated checksum is not valid")
def inode(self, nid: int) -> dict[str, int]:
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
if offset + 32 > len(self.data):
raise ValueError(f"inode {nid} is outside the image")
ifmt = self.u16(offset)
version = ifmt & 1
inode_size = 32 if version == 0 else 64
if offset + inode_size > len(self.data):
raise ValueError(f"inode {nid} is truncated")
xattr_count = self.u16(offset + 2)
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
return {
"nid": nid,
"offset": offset,
"ifmt": ifmt,
"layout": (ifmt >> 1) & 7,
"inode_size": inode_size,
"xattr_size": xattr_size,
"mode": self.u16(offset + 4),
"size": size,
"startblk": self.u32(offset + 16),
}
def inode_data_offset(self, inode: dict[str, int]) -> int:
if inode["layout"] == 2:
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
if inode["layout"] == 0:
return inode["startblk"] << self.block_bits
raise ValueError(f"unsupported directory layout {inode['layout']}")
def directory(self, nid: int) -> dict[str, dict[str, Any]]:
inode = self.inode(nid)
if inode["mode"] & S_IFMT != stat.S_IFDIR:
raise ValueError(f"inode {nid} is not a directory")
data_offset = self.inode_data_offset(inode)
size = inode["size"]
if size < 12 or data_offset + size > len(self.data):
raise ValueError("root directory bytes are outside the image")
first_nameoff = self.u16(data_offset + 8)
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
raise ValueError("invalid first directory name offset")
count = first_nameoff // 12
records: dict[str, dict[str, Any]] = {}
for index in range(count):
entry = data_offset + index * 12
nameoff = self.u16(entry + 8)
endoff = self.u16(entry + 20) if index + 1 < count else size
if not (first_nameoff <= nameoff < endoff <= size):
raise ValueError(f"invalid directory name span at index {index}")
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
name = raw.split(b"\0", 1)[0].decode("ascii")
if not name or name in records:
raise ValueError(f"invalid or duplicate directory name {name!r}")
records[name] = {
"entry_offset": entry,
"nid": self.u64(entry),
"file_type": self.data[entry + 10],
"reserved": self.data[entry + 11],
"nameoff": nameoff,
}
return records
def make_source(path: Path) -> None:
previous_umask = os.umask(0o022)
try:
path.mkdir(mode=0o755)
os.chmod(path, 0o755)
(path / "regular").write_bytes(b"P15-081 real inode payload\n")
os.chmod(path / "regular", 0o644)
os.link(path / "regular", path / "regular-hard")
(path / "subdir").mkdir(mode=0o755)
os.chmod(path / "subdir", 0o755)
os.symlink("regular", path / "symlink")
os.mkfifo(path / "fifo", 0o600)
os.chmod(path / "fifo", 0o600)
os.mknod(path / "char", stat.S_IFCHR | 0o600, os.makedev(1, 3))
os.chmod(path / "char", 0o600)
os.mknod(path / "block", stat.S_IFBLK | 0o600, os.makedev(7, 0))
os.chmod(path / "block", 0o600)
sock = socket.socket(socket.AF_UNIX)
try:
sock.bind(str(path / "socket"))
finally:
sock.close()
os.chmod(path / "socket", 0o600)
finally:
os.umask(previous_umask)
def build_seed(work: Path, output: Path) -> tuple[bytes, str, list[str]]:
work.mkdir()
source = work / "source"
make_source(source)
command = [
"mkfs.erofs",
"-d0",
"-T0",
"--all-time",
"--all-root",
"--workers=1",
"-x-1",
"-E",
"noinline_data",
"-U",
SPEC["uuid"],
str(output),
str(source),
]
env = dict(os.environ)
env["SOURCE_DATE_EPOCH"] = "0"
completed = run(command, env=env)
if completed.returncode != 0:
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
canonical = command[:-2] + ["SEED.erofs", "SOURCE"]
return output.read_bytes(), completed.stdout, canonical
def extract_kind(output: str) -> str:
match = re.search(r"^Size:.*? ([A-Za-z ]+)$", output, re.MULTILINE)
if match is None:
raise ValueError(f"dump.erofs output has no inode kind:\n{output}")
return match.group(1)
def extract_dump_nid(output: str) -> int:
match = re.search(r"^NID: (\d+)\b", output, re.MULTILINE)
if match is None:
raise ValueError(f"dump.erofs output has no NID:\n{output}")
return int(match.group(1))
def expected_errno(file_type: int, expected_file_type: int) -> int:
if file_type < 1 or file_type > 7:
return 0
return 0 if file_type == expected_file_type else EINTEGRITY
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-081":
raise SystemExit("invalid P15-081 gate input")
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise SystemExit(f"P15-081 must replay {SPEC['required_base']}, got {resolved}")
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
source_hashes = {
path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()
}
for path, expected in SPEC["source_sha256"].items():
if source_hashes[path] != expected:
raise SystemExit(f"frozen source identity mismatch: {path}")
freebsd_head = subprocess.check_output(
["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True
).strip()
if freebsd_head != SPEC["freebsd_head"]:
raise SystemExit(f"FreeBSD HEAD differs: {freebsd_head}")
freebsd_hashes = {}
for relative, expected in SPEC["freebsd_sha256"].items():
path = FREEBSD_SRC / relative
actual = sha256_path(path)
freebsd_hashes[relative] = actual
if actual != expected:
raise SystemExit(f"FreeBSD source identity mismatch: {relative}")
mkfs_version_run = run(["mkfs.erofs", "-V"])
mkfs_version = mkfs_version_run.stdout.splitlines()[0]
if mkfs_version != SPEC["mkfs_version"]:
raise SystemExit(
f"mkfs.erofs version differs: expected {SPEC['mkfs_version']!r}, "
f"got {mkfs_version!r}"
)
namei = sources["repo-pre-15/src/namei.c"]
directory_source = sources["repo-pre-15/src/dir.c"]
inode_source = sources["repo-pre-15/src/inode.c"]
internal_source = sources["repo-pre-15/src/internal.h"]
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
linux_namei = sources["src-linux/namei.c"]
linux_dir = sources["src-linux/dir.c"]
linux_ondisk = sources["src-linux/erofs_fs.h"]
source_anchors = {
"freebsd_dirent_field": "*d_type = de->file_type;" in namei,
"freebsd_inode_mode": "vi->vtype = IFTOVT(vi->mode);" in inode_source,
"freebsd_lookup_vget_once": namei.count("erofs_vget(") == 1,
"freebsd_dotdot_once": namei.count("error = vn_vget_ino(") == 1,
"freebsd_hash_lookup": "error = vfs_hash_get(" in vnops,
"freebsd_inode_decode_after_hash": vnops.index("error = vfs_hash_get(")
< vnops.index("error = erofs_read_inode("),
"linux_readdir_mapping": "fs_ftype_to_dtype(de->file_type)" in linux_dir,
"linux_lookup_dirent_field": "*d_type = de->file_type;" in linux_namei,
"linux_lookup_inode": "inode = erofs_iget(dir->i_sb, nid);" in linux_namei,
"linux_generic_ft_contract": "EROFS file types should match generic FT_* types"
in linux_ondisk,
}
if not all(source_anchors.values()):
raise SystemExit(f"source entrypoint anchors changed: {source_anchors}")
freebsd_dirent = (FREEBSD_SRC / "sys/sys/dirent.h").read_text(encoding="utf-8")
freebsd_vnode = (FREEBSD_SRC / "sys/sys/vnode.h").read_text(encoding="utf-8")
if "#define\tDT_UNKNOWN\t 0" not in freebsd_dirent:
raise SystemExit("FreeBSD DT_UNKNOWN contract changed")
if "#define\tIFTODT(mode)" not in freebsd_dirent:
raise SystemExit("FreeBSD IFTODT contract changed")
if "int vfs_hash_get(" not in freebsd_vnode or "int\tvn_vget_ino(" not in freebsd_vnode:
raise SystemExit("FreeBSD vnode lookup declarations changed")
type_by_name = {item["name"]: item for item in SPEC["file_types"]}
type_by_name["regular-hard"] = type_by_name["regular"]
fixtures = OUTPUT / "fixtures"
fixtures.mkdir()
with tempfile.TemporaryDirectory(prefix="p15-081-gate-") as temporary:
temp = Path(temporary)
first_bytes, first_stdout, canonical_command = build_seed(
temp / "first", temp / "first.erofs"
)
second_bytes, second_stdout, second_command = build_seed(
temp / "second", temp / "second.erofs"
)
if first_bytes != second_bytes or first_stdout != second_stdout:
raise SystemExit("P15-081 seed generation is not byte reproducible")
if canonical_command != second_command:
raise SystemExit("P15-081 canonical mkfs command changed between runs")
seed_path = fixtures / "seed.erofs"
seed_path.write_bytes(first_bytes)
(OUTPUT / "mkfs.stdout").write_text(first_stdout, encoding="utf-8")
write_json(
OUTPUT / "generator.json",
{
"command": canonical_command,
"mkfs_version": mkfs_version,
"repeat_byte_identical": True,
"seed_sha256": sha256_bytes(first_bytes),
"seed_size": len(first_bytes),
},
)
seed = Image(first_bytes)
if not seed.checksum_valid():
raise SystemExit("generated seed checksum is invalid")
seed_entries = seed.directory(seed.root_nid)
required_names = {item["name"] for item in SPEC["file_types"]}
required_names.add("regular-hard")
missing = sorted(required_names - seed_entries.keys())
if missing:
raise SystemExit(f"seed directory is missing names: {missing}")
seed_records = []
for name in sorted(required_names):
entry = seed_entries[name]
inode = seed.inode(entry["nid"])
expected_type = type_by_name[name]
record = {
"case": f"known-match-{name}",
"class": "known-match",
"expected_errno": 0,
"file_type": entry["file_type"],
"inode_mode": inode["mode"],
"inode_mode_type": inode["mode"] & S_IFMT,
"name": name,
"nid": entry["nid"],
"reserved": entry["reserved"],
"vtype": expected_type["vtype"],
}
if entry["file_type"] != expected_type["erofs"]:
raise SystemExit(f"mkfs emitted unexpected file type for {name}")
if record["inode_mode_type"] != expected_type["mode_type"]:
raise SystemExit(f"mkfs emitted unexpected inode mode for {name}")
seed_records.append(record)
if seed_entries[SPEC["hardlink"]["first"]]["nid"] != seed_entries[SPEC["hardlink"]["second"]]["nid"]:
raise SystemExit("hardlink seed names do not share one inode")
cases: list[dict[str, Any]] = []
for item in SPEC["known_mismatches"]:
cases.append(
{
"class": "known-mismatch",
"field": "file_type",
"id": f"known-mismatch-{item['name']}",
"name": item["name"],
"value": item["file_type"],
"expected_errno": EINTEGRITY,
}
)
for item in SPEC["tolerated"]:
cases.append(
{
"class": "forward-compatible",
"field": item["field"],
"id": item["id"],
"name": item["name"],
"value": item["value"],
"expected_errno": 0,
}
)
case_records: list[dict[str, Any]] = []
normal_entry_lines = [
"id\tname\tnid\tdirent_file_type\tdirent_reserved\tinode_mode_type\tdump_dirent_type\tdump_inode_kind\tfsck\tcandidate_errno"
]
fixture_digest = hashlib.sha256()
for case in cases:
image = seed.clone()
entries = image.directory(image.root_nid)
entry = entries[case["name"]]
direct_offset = entry["entry_offset"] + (10 if case["field"] == "file_type" else 11)
before = image.data[direct_offset]
image.data[direct_offset] = case["value"]
image.update_checksum()
path = fixtures / f"{case['id']}.erofs"
path.write_bytes(image.data)
parsed = Image(path.read_bytes())
if not parsed.checksum_valid():
raise SystemExit(f"fixture checksum is invalid: {case['id']}")
parsed_entry = parsed.directory(parsed.root_nid)[case["name"]]
parsed_inode = parsed.inode(parsed_entry["nid"])
expected_type = type_by_name[case["name"]]
actual_errno = expected_errno(parsed_entry["file_type"], expected_type["erofs"])
fsck = run(["fsck.erofs", "-d0", str(path)])
fsck_clean = fsck.returncode == 0 and "<E>" not in fsck.stdout
fsck_expected_clean = not (
case["field"] == "file_type" and case["value"] > 7
)
dump_ls = run(["dump.erofs", "--ls", "--path=/", str(path)])
dump_path = run(["dump.erofs", f"--path=/{case['name']}", str(path)])
if dump_ls.returncode != 0 or dump_path.returncode != 0:
raise SystemExit(f"normal path parser failed: {case['id']}")
listing = re.search(
rf"^\s+{parsed_entry['nid']}\s+(\d+)\s+{re.escape(case['name'])}$",
dump_ls.stdout,
re.MULTILINE,
)
if listing is None:
raise SystemExit(f"dump listing did not reach dirent: {case['id']}")
dump_dirent_type = int(listing.group(1))
dump_kind = extract_kind(dump_path.stdout)
dump_nid = extract_dump_nid(dump_path.stdout)
record = {
**case,
"actual_errno": actual_errno,
"checksum_valid": True,
"direct_before": before,
"direct_offset": direct_offset,
"dump_dirent_type": dump_dirent_type,
"dump_inode_kind": dump_kind,
"dump_nid": dump_nid,
"file_type": parsed_entry["file_type"],
"fixture": path.name,
"fixture_sha256": sha256_path(path),
"fsck_clean": fsck_clean,
"fsck_expected_clean": fsck_expected_clean,
"inode_mode": parsed_inode["mode"],
"inode_mode_type": parsed_inode["mode"] & S_IFMT,
"nid": parsed_entry["nid"],
"reserved": parsed_entry["reserved"],
"vtype": expected_type["vtype"],
}
record["oracle_pass"] = all(
(
record["actual_errno"] == record["expected_errno"],
record["fsck_clean"] == record["fsck_expected_clean"],
record["dump_dirent_type"] == record["file_type"],
record["dump_inode_kind"] == expected_type["kind"],
record["dump_nid"] == record["nid"],
record["inode_mode_type"] == expected_type["mode_type"],
)
)
case_records.append(record)
normal_entry_lines.append(
"\t".join(
str(value)
for value in (
record["id"],
record["name"],
record["nid"],
record["file_type"],
record["reserved"],
record["inode_mode_type"],
record["dump_dirent_type"],
record["dump_inode_kind"],
"clean" if record["fsck_clean"] else "policy-reject",
record["actual_errno"],
)
)
)
fixture_digest.update(case["id"].encode("ascii"))
fixture_digest.update(b"\0")
fixture_digest.update(path.read_bytes())
(OUTPUT / "normal-entry.tsv").write_text(
"\n".join(normal_entry_lines) + "\n", encoding="ascii"
)
all_records = seed_records + case_records
prototype = r'''#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#define EINTEGRITY 97
enum prototype_vtype {
VT_NON,
VT_REG,
VT_DIR,
VT_BLK,
VT_CHR,
VT_LNK,
VT_SOCK,
VT_FIFO,
};
static bool
erofs_dirent_type_matches(uint8_t file_type, enum prototype_vtype vtype)
{
switch (file_type) {
case 1:
return (vtype == VT_REG);
case 2:
return (vtype == VT_DIR);
case 3:
return (vtype == VT_CHR);
case 4:
return (vtype == VT_BLK);
case 5:
return (vtype == VT_FIFO);
case 6:
return (vtype == VT_SOCK);
case 7:
return (vtype == VT_LNK);
default:
return (true);
}
}
int
main(int argc, char **argv)
{
unsigned long file_type, vtype;
int error;
if (argc != 3)
return (2);
file_type = strtoul(argv[1], NULL, 0);
vtype = strtoul(argv[2], NULL, 0);
if (file_type > UINT8_MAX || vtype > VT_FIFO)
return (2);
error = erofs_dirent_type_matches((uint8_t)file_type,
(enum prototype_vtype)vtype) ? 0 : EINTEGRITY;
printf("%d\n", error);
return (0);
}
'''
prototype_path = OUTPUT / "prototype.c"
prototype_path.write_text(prototype, encoding="ascii")
binary = temp / "prototype"
compile_run = run(
[
"cc",
"-std=c11",
"-Wall",
"-Wextra",
"-Werror",
str(prototype_path),
"-o",
str(binary),
]
)
if compile_run.returncode != 0:
raise SystemExit(f"candidate prototype did not compile: {compile_run.stdout}")
prototype_records = []
for record in all_records:
completed = run(
[
str(binary),
str(record["file_type"]),
str(VTYPE_NUMBER[record["vtype"]]),
]
)
if completed.returncode != 0:
raise SystemExit(f"candidate prototype failed for {record['case'] if 'case' in record else record['id']}")
observed = int(completed.stdout.strip())
expected = record["expected_errno"]
prototype_records.append(
{
"id": record.get("id", record.get("case")),
"observed_errno": observed,
"expected_errno": expected,
"pass": observed == expected,
}
)
helper = '''
bool
erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype)
{
\tswitch (file_type) {
\tcase EROFS_FT_REG_FILE:
\t\treturn (vtype == VREG);
\tcase EROFS_FT_DIR:
\t\treturn (vtype == VDIR);
\tcase EROFS_FT_CHRDEV:
\t\treturn (vtype == VCHR);
\tcase EROFS_FT_BLKDEV:
\t\treturn (vtype == VBLK);
\tcase EROFS_FT_FIFO:
\t\treturn (vtype == VFIFO);
\tcase EROFS_FT_SOCK:
\t\treturn (vtype == VSOCK);
\tcase EROFS_FT_SYMLINK:
\t\treturn (vtype == VLNK);
\tdefault:
\t\treturn (true);
\t}
}
'''
inode_marker = "/*\n * Read and decode a disk inode."
if inode_source.count(inode_marker) != 1:
raise SystemExit("candidate inode insertion marker is ambiguous")
candidate_inode = inode_source.replace(inode_marker, helper + inode_marker, 1)
prototype_decl = (
"bool erofs_dirent_type_matches(uint8_t file_type, "
"__enum_uint8(vtype) vtype);\n"
)
internal_marker = "int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,\n"
if internal_source.count(internal_marker) != 1:
raise SystemExit("candidate prototype insertion marker is ambiguous")
candidate_internal = internal_source.replace(
internal_marker, prototype_decl + internal_marker, 1
)
lookup_marker = "\tif (error != 0)\n\t\treturn (error);\n\t*ap->a_vpp = vp;\n"
lookup_candidate = (
"\tif (error != 0)\n"
"\t\treturn (error);\n"
"\tif ((cnp->cn_flags & ISDOTDOT) == 0 &&\n"
"\t !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) {\n"
"\t\tvput(vp);\n"
"\t\treturn (EINTEGRITY);\n"
"\t}\n"
"\t*ap->a_vpp = vp;\n"
)
if namei.count(lookup_marker) != 1:
raise SystemExit("candidate lookup insertion marker is ambiguous")
candidate_namei = namei.replace(lookup_marker, lookup_candidate, 1)
patch_lines = []
for path, before, after in (
("src/inode.c", inode_source, candidate_inode),
("src/internal.h", internal_source, candidate_internal),
("src/namei.c", namei, candidate_namei),
):
patch_lines.extend(
difflib.unified_diff(
before.splitlines(keepends=True),
after.splitlines(keepends=True),
fromfile=f"a/{path}",
tofile=f"b/{path}",
)
)
candidate_patch = "".join(patch_lines)
(OUTPUT / "candidate.patch").write_text(candidate_patch, encoding="utf-8")
lock_calls = re.findall(
r"\b(?:erofs_vget|vn_vget_ino|vfs_hash_get|VOP_LOCK|vn_lock|lockmgr)\s*\(",
helper + lookup_candidate,
)
cache_first = seed_entries[SPEC["hardlink"]["first"]]
cache_second = seed_entries[SPEC["hardlink"]["second"]]
cache_case = next(
record for record in case_records if record["id"] == "known-mismatch-regular-hard"
)
cache_proof = {
"first_lookup_name": SPEC["hardlink"]["first"],
"second_lookup_name": SPEC["hardlink"]["second"],
"different_namecache_keys": SPEC["hardlink"]["first"]
!= SPEC["hardlink"]["second"],
"same_real_nid": cache_first["nid"] == cache_second["nid"],
"second_dirent_known_mismatch": cache_case["actual_errno"] == EINTEGRITY,
"normal_lookup_calls_existing_erofs_vget": namei.count("erofs_vget(") == 1,
"erofs_vget_checks_hash_before_inode_read": source_anchors[
"freebsd_inode_decode_after_hash"
],
}
lock_ledger = {
"candidate_added_lock_or_vget_calls": lock_calls,
"candidate_reads_immutable_vtype": "VTOE(vp)->vtype" in lookup_candidate,
"dotdot_bypasses_validator":
"(cnp->cn_flags & ISDOTDOT) == 0" in lookup_candidate,
"mismatch_drops_locked_child": "vput(vp);" in lookup_candidate,
"validator_before_vpp_publication": lookup_candidate.index(
"erofs_dirent_type_matches"
)
< lookup_candidate.index("*ap->a_vpp = vp"),
"validator_before_namecache_publication": candidate_namei.index(
"erofs_dirent_type_matches"
)
< candidate_namei.index("cache_enter(dvp, vp, cnp)"),
"readdir_has_no_vget": all(
token not in directory_source
for token in ("erofs_vget(", "vn_vget_ino(", "vfs_hash_get(")
),
"parent_child_lock_sequence_unchanged": namei.count("erofs_vget(")
== candidate_namei.count("erofs_vget(")
and namei.count("error = vn_vget_ino(")
== candidate_namei.count("error = vn_vget_ino("),
"freebsd_contract_sha256": freebsd_hashes,
"cache_hit_proof": cache_proof,
}
write_json(OUTPUT / "lock-ledger.json", lock_ledger)
failures = []
failures.extend(
record["id"] for record in case_records if not record["oracle_pass"]
)
failures.extend(
record["id"] for record in prototype_records if not record["pass"]
)
for key, value in lock_ledger.items():
if key == "freebsd_contract_sha256":
continue
if key == "candidate_added_lock_or_vget_calls":
if value:
failures.append(key)
elif key == "cache_hit_proof":
if not all(value.values()):
failures.append(key)
elif value is not True:
failures.append(key)
status = "GO" if not failures else "STOP"
oracle = {
"cache_hit": cache_proof,
"candidate_patch_sha256": sha256_bytes(candidate_patch.encode("utf-8")),
"candidate_prototype_sha256": sha256_path(prototype_path),
"case_records": case_records,
"known_match_records": seed_records,
"prototype_records": prototype_records,
"source_anchors": source_anchors,
"status": status,
}
write_json(OUTPUT / "oracle.json", oracle)
write_json(OUTPUT / "source-sha256.json", source_hashes)
write_json(
OUTPUT / "freebsd-contract.json",
{"head": freebsd_head, "sha256": freebsd_hashes},
)
fixture_index = {
"fixture_count": len(list(fixtures.glob("*.erofs"))),
"fixture_set_sha256": fixture_digest.hexdigest(),
"fixtures": [
{
"name": path.name,
"sha256": sha256_path(path),
"size": path.stat().st_size,
}
for path in sorted(fixtures.glob("*.erofs"))
],
"seed_repeat_byte_identical": True,
}
write_json(OUTPUT / "fixture-index.json", fixture_index)
result = {
"b11": "AUTHORIZED" if status == "GO" else "STOP-NO-SOURCE",
"cache_hit_sequence_count": 1,
"candidate": "P15-081",
"failures": failures,
"fixture_count": fixture_index["fixture_count"],
"fixture_set_sha256": fixture_index["fixture_set_sha256"],
"forward_compatible_case_count": len(SPEC["tolerated"]),
"full_feature_suite": "NOT_RUN",
"gate": "G11",
"known_match_count": len(seed_records),
"known_mismatch_count": len(SPEC["known_mismatches"]),
"normal_entry_count": len(case_records),
"prototype_case_count": len(prototype_records),
"qemu": "NOT_RUN",
"qemu_reason": "pre-source gate uses checksum-valid disk fixtures, independent host parsing, and a non-KLD prototype",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"schema": 1,
"status": status,
}
write_json(OUTPUT / "result.json", result)
hash_lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
print(json.dumps(result, sort_keys=True))
if status != "GO":
raise SystemExit(1)
PY
tests/pre15/gates/P15-083-input.json
+123
View File
@@ -0,0 +1,123 @@
{
"candidate": "P15-083",
"codecs": {
"deflate": {
"dict_size": 32768,
"expected_image_sha256": "39455150c3e999bb7ae6c36402c15a408a5679726b6d099e7d121e009ef43af6",
"expected_image_size": 32768,
"extent": {
"index": 6,
"leading_zero_bytes": 1479,
"logical_length": 14348,
"logical_offset": 137204,
"physical_length": 4096,
"physical_offset": 28672,
"stream_bytes": 2617
},
"mkfs_args": [
"-T0",
"-U20000000-0000-4000-8000-000000000083",
"--all-root",
"-x-1",
"--workers=1",
"-zdeflate,level=1,dictsize=32768",
"-C4096"
]
},
"lzma": {
"dict_size": 65536,
"expected_image_sha256": "c26cf15844fe45a21a746bacbad2ef551c683bb9b2538de7a7eced37d8eadff9",
"expected_image_size": 8192,
"extent": {
"index": 0,
"leading_zero_bytes": 3556,
"logical_length": 151552,
"logical_offset": 0,
"physical_length": 4096,
"physical_offset": 4096,
"stream_bytes": 540
},
"mkfs_args": [
"-T0",
"-U10000000-0000-4000-8000-000000000083",
"--all-root",
"-x-1",
"--workers=1",
"-zlzma,level=6,dictsize=65536",
"-C4096"
]
},
"zstd": {
"dict_size": 65536,
"expected_image_sha256": "b905803f0e08500cc3c6cfe07a95fe027859165acae19ca8865856af256f32ca",
"expected_image_size": 8192,
"extent": {
"index": 0,
"leading_zero_bytes": 1092,
"logical_length": 151552,
"logical_offset": 0,
"physical_length": 4096,
"physical_offset": 4096,
"stream_bytes": 3004
},
"mkfs_args": [
"-T0",
"-U30000000-0000-4000-8000-000000000083",
"--all-root",
"-x-1",
"--workers=1",
"-zzstd,level=3,dictsize=65536",
"-C4096"
]
}
},
"fixture": {
"block_size": 4096,
"line_count": 4096,
"source_sha256": "f9ebc3ccae455cd9a01afae784c9bbbc28c45936880f444fdf2c8b64dff44bfc",
"source_size": 151552,
"tail_bytes_hex": "a55ac33c96696996"
},
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/contrib/xz-embedded/linux/include/linux/xz.h": "4255bf4d723746761816da5787ca47acf8b229d3e7a37502a12f3d532772026b",
"sys/contrib/zlib/zlib.h": "3c4f75f90589af70540aeaf23610e2b299bb498826a34c4c16ec92233dd68693",
"sys/contrib/zstd/lib/zstd.h": "9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b",
"sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c"
}
},
"gate": "G04",
"libraries": {
"liblzma": "5.8.1",
"libzstd": "1.5.7",
"zlib": "1.3.1"
},
"required_base": "68bbe94c44e35d53cec8ab55d007f40b01cf0502",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c",
"repo-pre-15/src/decompressor_deflate.c": "563fe3955b61ff977a3bebca9c73b06a38129a401ea95564a91844a6aaefdaa7",
"repo-pre-15/src/decompressor_lzma.c": "675ab6912b9f5390a79286682390fa089c696ddf012718b78f81f14d32f0026e",
"repo-pre-15/src/decompressor_zstd.c": "21f3b67c4b10b9cb6bebf6ca1138c06e702fb746487dc8db11dcd7676f7c84fc",
"repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242",
"src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709",
"src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b",
"src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8",
"src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607"
},
"tools": {
"dump.erofs": {
"path": "/usr/bin/dump.erofs",
"sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c"
}
}
}
tests/pre15/gates/P15-083.sh
Executable
+860
View File
@@ -0,0 +1,860 @@
#!/bin/sh
set -eu
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
input=$gate_dir/P15-083-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
base=
output=
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
for tool in cc dump.erofs fsck.erofs git mkfs.erofs pkg-config python3 sha256sum; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required host tool: %s\n' "$tool" >&2
exit 2
}
done
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output"
python3 - "$root" "$input" "$base" "$output" "$freebsd_src" <<'PY'
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
EINTEGRITY = 97
class GateFailure(Exception):
def __init__(self, status: str, reason: str):
super().__init__(reason)
self.status = status
self.reason = reason
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
return sha256_bytes(path.read_bytes())
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def git(*args: str) -> str:
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
def source_at(commit: str, path: str) -> str:
completed = subprocess.run(
["git", "-C", str(ROOT), "show", f"{commit}:{path}"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if completed.returncode != 0:
raise GateFailure("INFRA_BLOCKED", f"cannot read {path} at {commit}: {completed.stderr.strip()}")
return completed.stdout
def run_logged(
argv: list[str], cwd: Path, log: Path, timeout: int = 120, expected: set[int] | None = None
) -> subprocess.CompletedProcess[str]:
completed = subprocess.run(
argv,
cwd=cwd,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=timeout,
)
log.parent.mkdir(parents=True, exist_ok=True)
log.write_text(
"$ " + " ".join(argv) + "\n" + completed.stdout + f"\n[exit {completed.returncode}]\n",
encoding="utf-8",
)
allowed = {0} if expected is None else expected
if completed.returncode not in allowed:
raise GateFailure("INFRA_BLOCKED", f"command failed ({completed.returncode}): {' '.join(argv)}")
return completed
def make_source(path: Path) -> bytes:
path.mkdir(parents=True)
content = b"".join(
f"P15-083-{index % 64:02d}:alpha-beta-gamma-delta:{(index * 17) % 256:02x}\n".encode("ascii")
for index in range(SPEC["fixture"]["line_count"])
)
payload = path / "payload.bin"
payload.write_bytes(content)
os.utime(payload, (0, 0))
os.utime(path, (0, 0))
if len(content) != SPEC["fixture"]["source_size"] or sha256_bytes(content) != SPEC["fixture"]["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "deterministic source identity changed")
return content
EXTENT_RE = re.compile(
r"^\s*(\d+):\s*(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*:\s*"
r"(\d+)\.\.\s*(\d+)\s*\|\s*(\d+)\s*$",
re.MULTILINE,
)
def parse_extents(text: str) -> list[dict[str, int]]:
records = []
for match in EXTENT_RE.finditer(text):
index, logical, logical_end, logical_length, physical, physical_end, physical_length = (
map(int, match.groups())
)
if logical_end - logical != logical_length or physical_end - physical != physical_length:
raise GateFailure("INFRA_BLOCKED", "dump.erofs extent arithmetic changed")
records.append(
{
"index": index,
"logical_length": logical_length,
"logical_offset": logical,
"physical_length": physical_length,
"physical_offset": physical,
}
)
if not records:
raise GateFailure("INFRA_BLOCKED", "dump.erofs returned no extents")
return records
ORACLE_SOURCE = r'''
#define _POSIX_C_SOURCE 200809L
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <lzma.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
#include <zstd.h>
static void
fail(const char *message)
{
fprintf(stderr, "%s\n", message);
exit(2);
}
static unsigned char *
read_file(const char *path, size_t *sizep)
{
struct stat st;
unsigned char *data;
ssize_t done, amount;
int fd;
fd = open(path, O_RDONLY);
if (fd < 0 || fstat(fd, &st) != 0 || st.st_size <= 0)
fail("cannot open oracle input");
*sizep = (size_t)st.st_size;
data = malloc(*sizep);
if (data == NULL)
fail("cannot allocate oracle input");
done = 0;
while ((size_t)done < *sizep) {
amount = read(fd, data + done, *sizep - (size_t)done);
if (amount <= 0)
fail("cannot read oracle input");
done += amount;
}
close(fd);
return (data);
}
static void
write_file(const char *path, const unsigned char *data, size_t size)
{
ssize_t done, amount;
int fd;
fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
fail("cannot create oracle output");
done = 0;
while ((size_t)done < size) {
amount = write(fd, data + done, size - (size_t)done);
if (amount <= 0)
fail("cannot write oracle output");
done += amount;
}
close(fd);
}
int
main(int argc, char **argv)
{
unsigned char *input, *mapping, *output;
size_t input_size, output_size, usable, page_size;
size_t consumed = 0, produced = 0, library_status = 0;
uint32_t dict_size;
int cleanup = 0, codec_error = 0, full, guards = 1, stream_end = 0;
if (argc != 7)
fail("usage: oracle CODEC INPUT OUTPUT OUTPUT_SIZE FULL DICT_SIZE");
output_size = (size_t)strtoull(argv[4], NULL, 10);
full = atoi(argv[5]);
dict_size = (uint32_t)strtoul(argv[6], NULL, 10);
if (output_size == 0)
fail("zero output size");
input = read_file(argv[2], &input_size);
page_size = (size_t)sysconf(_SC_PAGESIZE);
usable = (output_size + page_size - 1) & ~(page_size - 1);
mapping = mmap(NULL, usable + 2 * page_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (mapping == MAP_FAILED)
fail("cannot allocate guarded output");
if (mprotect(mapping, page_size, PROT_NONE) != 0 ||
mprotect(mapping + page_size + usable, page_size, PROT_NONE) != 0)
fail("cannot protect output guards");
output = mapping + page_size;
memset(output, 0xa5, usable);
if (strcmp(argv[1], "deflate") == 0) {
z_stream stream;
int ret = Z_OK, endret;
memset(&stream, 0, sizeof(stream));
stream.next_in = input;
stream.avail_in = (uInt)input_size;
stream.next_out = output;
stream.avail_out = (uInt)output_size;
ret = inflateInit2(&stream, -15);
if (ret != Z_OK) {
codec_error = 1;
} else {
while (stream.avail_out != 0) {
uInt in_before = stream.avail_in;
uInt out_before = stream.avail_out;
ret = inflate(&stream, Z_SYNC_FLUSH);
if (ret == Z_STREAM_END) {
stream_end = 1;
break;
}
if (ret != Z_OK || (stream.avail_in == in_before &&
stream.avail_out == out_before)) {
codec_error = 1;
break;
}
}
consumed = input_size - stream.avail_in;
produced = output_size - stream.avail_out;
library_status = (size_t)(unsigned int)ret;
endret = inflateEnd(&stream);
cleanup = endret == Z_OK;
}
} else if (strcmp(argv[1], "lzma") == 0) {
lzma_stream stream = LZMA_STREAM_INIT;
lzma_ret ret;
ret = lzma_microlzma_decoder(&stream, input_size, output_size,
full != 0, dict_size);
if (ret != LZMA_OK) {
codec_error = 1;
} else {
stream.next_in = input;
stream.avail_in = input_size;
stream.next_out = output;
stream.avail_out = output_size;
while (stream.avail_out != 0) {
size_t in_before = stream.avail_in;
size_t out_before = stream.avail_out;
ret = lzma_code(&stream, LZMA_RUN);
if (ret == LZMA_STREAM_END) {
stream_end = 1;
break;
}
if (ret != LZMA_OK || (stream.avail_in == in_before &&
stream.avail_out == out_before)) {
codec_error = 1;
break;
}
}
consumed = input_size - stream.avail_in;
produced = output_size - stream.avail_out;
library_status = ret;
}
lzma_end(&stream);
cleanup = 1;
} else if (strcmp(argv[1], "zstd") == 0) {
ZSTD_DCtx *context;
ZSTD_inBuffer in_buffer;
ZSTD_outBuffer out_buffer;
size_t ret = 1;
context = ZSTD_createDCtx();
if (context == NULL) {
codec_error = 1;
} else {
in_buffer = (ZSTD_inBuffer){ input, input_size, 0 };
out_buffer = (ZSTD_outBuffer){ output, output_size, 0 };
while (out_buffer.pos != out_buffer.size) {
size_t in_before = in_buffer.pos;
size_t out_before = out_buffer.pos;
ret = ZSTD_decompressStream(context, &out_buffer, &in_buffer);
if (ZSTD_isError(ret)) {
codec_error = 1;
break;
}
if (ret == 0) {
stream_end = 1;
break;
}
if (in_buffer.pos == in_before && out_buffer.pos == out_before) {
codec_error = 1;
break;
}
}
consumed = in_buffer.pos;
produced = out_buffer.pos;
library_status = ret;
cleanup = !ZSTD_isError(ZSTD_freeDCtx(context));
}
} else {
fail("unknown codec");
}
for (size_t index = output_size; index < usable; ++index) {
if (output[index] != 0xa5) {
guards = 0;
break;
}
}
write_file(argv[3], output, produced);
printf("codec_error=%d cleanup=%d consumed=%zu guards=%d library_status=%zu "
"produced=%zu stream_end=%d\n", codec_error, cleanup, consumed, guards,
library_status, produced, stream_end);
munmap(mapping, usable + 2 * page_size);
free(input);
return (0);
}
'''
def compile_oracle(temp: Path) -> tuple[Path, dict[str, str]]:
source = temp / "p15-083-oracle.c"
binary = temp / "p15-083-oracle"
source.write_text(ORACLE_SOURCE, encoding="ascii")
completed = run_logged(
[
"cc",
"-O2",
"-std=c17",
"-Wall",
"-Wextra",
"-Werror",
str(source),
"-o",
str(binary),
"-llzma",
"-lz",
"-lzstd",
],
temp,
OUTPUT / "logs/oracle-build.log",
)
del completed
ldd = run_logged(["ldd", str(binary)], temp, OUTPUT / "logs/oracle-ldd.log")
versions = {
"liblzma": subprocess.check_output(["pkg-config", "--modversion", "liblzma"], text=True).strip(),
"zlib": subprocess.check_output(["pkg-config", "--modversion", "zlib"], text=True).strip(),
"libzstd": subprocess.check_output(["pkg-config", "--modversion", "libzstd"], text=True).strip(),
}
if versions != SPEC["libraries"]:
raise GateFailure("INFRA_BLOCKED", f"independent library versions changed: {versions}")
return binary, {"binary_sha256": sha256_path(binary), "ldd": ldd.stdout, "source_sha256": sha256_path(source), **versions}
def decode(
oracle: Path,
codec: str,
data: bytes,
output_size: int,
full: bool,
dict_size: int,
temp: Path,
label: str,
) -> tuple[dict[str, int], bytes]:
input_path = temp / f"{label}.input"
output_path = temp / f"{label}.output"
input_path.write_bytes(data)
completed = run_logged(
[
str(oracle),
codec,
str(input_path),
str(output_path),
str(output_size),
"1" if full else "0",
str(dict_size),
],
temp,
OUTPUT / f"logs/oracle-{label}.log",
)
record = {}
for field in completed.stdout.strip().split():
key, value = field.split("=", 1)
record[key] = int(value)
required = {"codec_error", "cleanup", "consumed", "guards", "library_status", "produced", "stream_end"}
if set(record) != required:
raise GateFailure("INFRA_BLOCKED", f"oracle output fields changed for {label}")
return record, output_path.read_bytes()
def full_policy_errno(record: dict[str, int], input_size: int, expected_output: bytes, output: bytes) -> int:
if (
record["codec_error"] != 0
or record["cleanup"] != 1
or record["guards"] != 1
or record["produced"] != len(expected_output)
or record["stream_end"] != 1
or record["consumed"] != input_size
or output != expected_output
):
return EINTEGRITY
return 0
def partial_policy_errno(record: dict[str, int], expected_output: bytes, output: bytes) -> int:
if (
record["codec_error"] != 0
or record["cleanup"] != 1
or record["guards"] != 1
or record["produced"] != len(expected_output)
or output != expected_output
):
return EINTEGRITY
return 0
def verify_source_contract(sources: dict[str, str]) -> dict[str, Any]:
dispatch = sources["repo-pre-15/src/decompressor.c"]
if not all(
anchor in dispatch
for anchor in (
"map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4",
"src[padding] == 0",
"rq.inputsize = srclen;",
"return (decompressor->decompress(&rq));",
)
):
raise GateFailure("INFRA_BLOCKED", "frozen DUT leading-padding dispatch changed")
checks = {
"deflate": "(ret != Z_STREAM_END || strm.avail_in != 0)",
"lzma": "buffer.in_pos == rq->inputsize",
"zstd": "(ret != 0 || input.pos != input.size)",
}
for codec, anchor in checks.items():
if anchor not in sources[f"repo-pre-15/src/decompressor_{codec}.c"]:
raise GateFailure("INFRA_BLOCKED", f"frozen DUT {codec} full-tail check changed")
zdata = sources["repo-pre-15/src/zdata.c"]
read_extent_start = zdata.index("z_erofs_read_extent(")
read_extent_end = zdata.index("z_erofs_do_read(", read_extent_start)
read_extent = zdata[read_extent_start:read_extent_end]
decode_at = read_extent.index("error = z_erofs_decompress")
meta_release_at = read_extent.index("erofs_put_metabuf(&buf)", decode_at)
physical_release_at = read_extent.index("erofs_brelse(compressed)", decode_at)
error_at = read_extent.index("if (error != 0)", decode_at)
free_at = read_extent.index("free(decoded, M_EROFS)", error_at)
publish_at = read_extent.index("*bufp = decoded;", free_at)
if not (
decode_at < meta_release_at < error_at < free_at < publish_at
and decode_at < physical_release_at < error_at
):
raise GateFailure("INFRA_BLOCKED", "frozen DUT buffer cleanup ordering changed")
linux_common = sources["src-linux/decompressor.c"]
if "For others, zero_padding is enabled all the time." not in linux_common:
raise GateFailure("INFRA_BLOCKED", "Linux non-LZ4 padding anchor changed")
linux_anchors = {
"deflate": "if (zerr == Z_STREAM_END && !rq->outputsize)",
"lzma": "xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize",
"zstd": "zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);",
}
for codec, anchor in linux_anchors.items():
if anchor not in sources[f"src-linux/decompressor_{codec}.c"]:
raise GateFailure("INFRA_BLOCKED", f"Linux {codec} comparison anchor changed")
return {
"freebsd_positive_eintegrity": True,
"input_release_after_decode": True,
"linux_leading_zero_padding": True,
"output_freed_on_error": True,
"output_published_only_on_success": True,
"per_codec_full_checks": checks,
}
def verify_tools() -> dict[str, Any]:
records = {}
for name, expected in SPEC["tools"].items():
path = Path(expected["path"])
if not path.is_file() or sha256_path(path) != expected["sha256"]:
raise GateFailure("INFRA_BLOCKED", f"tool identity changed: {name}")
records[name] = {**expected}
mkfs_version = subprocess.check_output([SPEC["tools"]["mkfs.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
fsck_version = subprocess.check_output([SPEC["tools"]["fsck.erofs"]["path"], "-V"], stderr=subprocess.STDOUT, text=True)
for codec in SPEC["codecs"]:
if codec not in mkfs_version or codec not in fsck_version:
raise GateFailure("STOP", f"{codec} lacks a real mkfs/fsck codec path")
records["mkfs_version"] = mkfs_version.strip()
records["fsck_version"] = fsck_version.strip()
return records
def fsck_image(
fsck: str,
image: Path,
destination: Path,
label: str,
expect_success: bool | None,
) -> dict[str, Any]:
completed = run_logged(
[fsck, f"--extract={destination}", str(image)],
image.parent,
OUTPUT / f"logs/fsck-{label}.log",
expected=set(range(0, 256)),
)
success = completed.returncode == 0
if expect_success is not None and success != expect_success:
raise GateFailure("INFRA_BLOCKED", f"fsck classification changed for {label}: exit {completed.returncode}")
return {"exit": completed.returncode, "success": success}
def evaluate_codec(
codec: str,
spec: dict[str, Any],
source: bytes,
source_dir: Path,
oracle: Path,
temp: Path,
) -> dict[str, Any]:
mkfs = SPEC["tools"]["mkfs.erofs"]["path"]
fsck = SPEC["tools"]["fsck.erofs"]["path"]
dump = SPEC["tools"]["dump.erofs"]["path"]
images = []
for pass_name in ("a", "b"):
image = temp / f"{codec}-{pass_name}.erofs"
run_logged([mkfs, *spec["mkfs_args"], str(image), str(source_dir)], temp, OUTPUT / f"logs/mkfs-{codec}-{pass_name}.log")
images.append(image)
hashes = [sha256_path(path) for path in images]
if hashes != [spec["expected_image_sha256"]] * 2 or images[0].stat().st_size != spec["expected_image_size"]:
raise GateFailure("INFRA_BLOCKED", f"{codec} image reproducibility changed: {hashes}")
valid_fsck = fsck_image(fsck, images[0], temp / f"extract-{codec}-valid", f"{codec}-valid", True)
extracted = (temp / f"extract-{codec}-valid/payload.bin").read_bytes()
if extracted != source:
raise GateFailure("INFRA_BLOCKED", f"{codec} legal image extraction mismatch")
dumped = run_logged([dump, "--path=/payload.bin", "-e", str(images[0])], temp, OUTPUT / f"logs/dump-{codec}.log")
extents = parse_extents(dumped.stdout)
expected_extent = spec["extent"]
selected = next((record for record in extents if record["index"] == expected_extent["index"]), None)
if selected is None or selected != {key: expected_extent[key] for key in selected}:
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent changed")
image_bytes = images[0].read_bytes()
block_start = selected["physical_offset"]
block_end = block_start + selected["physical_length"]
block = image_bytes[block_start:block_end]
leading = next((index for index, value in enumerate(block) if value), len(block))
stream = block[leading:]
if leading != expected_extent["leading_zero_bytes"] or len(stream) != expected_extent["stream_bytes"]:
raise GateFailure("INFRA_BLOCKED", f"{codec} leading padding or stream length changed")
logical = source[selected["logical_offset"] : selected["logical_offset"] + selected["logical_length"]]
full_record, full_output = decode(oracle, codec, stream, len(logical), True, spec["dict_size"], temp, f"{codec}-full")
if full_policy_errno(full_record, len(stream), logical, full_output) != 0:
raise GateFailure("STOP", f"{codec} legal mkfs extent is not exact after EROFS leading padding")
tail = bytes.fromhex(SPEC["fixture"]["tail_bytes_hex"])
if leading <= len(tail):
raise GateFailure("INFRA_BLOCKED", f"{codec} selected extent lacks mutation room")
tail_stream = stream + tail
tail_record, tail_output = decode(oracle, codec, tail_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-tail")
tail_errno = full_policy_errno(tail_record, len(tail_stream), logical, tail_output)
if tail_errno != EINTEGRITY:
raise GateFailure("STOP", f"{codec} cannot distinguish nonzero trailing garbage from a complete EROFS stream")
tail_image_bytes = bytearray(image_bytes)
shifted_start = block_start + leading - len(tail)
tail_image_bytes[shifted_start:block_end] = tail_stream
tail_image = temp / f"{codec}-tail.erofs"
tail_image.write_bytes(tail_image_bytes)
tail_fsck = fsck_image(
fsck,
tail_image,
temp / f"extract-{codec}-tail",
f"{codec}-tail",
None,
)
if tail_fsck["success"]:
tail_extracted = (temp / f"extract-{codec}-tail/payload.bin").read_bytes()
if tail_extracted != source:
raise GateFailure("INFRA_BLOCKED", f"{codec} permissive fsck tail output mismatch")
truncated_stream = stream[:-1]
truncated_record, truncated_output = decode(
oracle, codec, truncated_stream, len(logical), True, spec["dict_size"], temp, f"{codec}-truncated"
)
truncated_errno = full_policy_errno(truncated_record, len(truncated_stream), logical, truncated_output)
if truncated_errno != EINTEGRITY:
raise GateFailure("STOP", f"{codec} truncated stream reaches full success")
truncated_image_bytes = bytearray(image_bytes)
truncated_image_bytes[block_start + leading : block_end] = b"\0" + truncated_stream
truncated_image = temp / f"{codec}-truncated.erofs"
truncated_image.write_bytes(truncated_image_bytes)
truncated_fsck = fsck_image(
fsck,
truncated_image,
temp / f"extract-{codec}-truncated",
f"{codec}-truncated",
False,
)
partial_size = min(4096, len(logical) // 4)
partial_expected = logical[:partial_size]
partial_record, partial_output = decode(
oracle, codec, stream, partial_size, False, spec["dict_size"], temp, f"{codec}-partial"
)
if partial_policy_errno(partial_record, partial_expected, partial_output) != 0:
raise GateFailure("STOP", f"{codec} partial output differs from the full slice")
corruption_start = max(partial_record["consumed"] + 16, len(stream) - 64)
if corruption_start >= len(stream):
raise GateFailure("STOP", f"{codec} partial decode consumes the entire stream")
corrupted_stream = stream[:corruption_start] + b"\0" * (len(stream) - corruption_start)
if corrupted_stream == stream:
raise GateFailure("INFRA_BLOCKED", f"{codec} corruption mutation changed no bytes")
corrupt_partial_record, corrupt_partial_output = decode(
oracle,
codec,
corrupted_stream,
partial_size,
False,
spec["dict_size"],
temp,
f"{codec}-corrupt-partial",
)
if partial_policy_errno(corrupt_partial_record, partial_expected, corrupt_partial_output) != 0:
raise GateFailure("STOP", f"{codec} range-after corruption changed the requested partial slice")
corrupt_full_record, corrupt_full_output = decode(
oracle,
codec,
corrupted_stream,
len(logical),
True,
spec["dict_size"],
temp,
f"{codec}-corrupt-full",
)
corrupt_full_errno = full_policy_errno(
corrupt_full_record, len(corrupted_stream), logical, corrupt_full_output
)
if corrupt_full_errno != EINTEGRITY:
raise GateFailure("STOP", f"{codec} full read does not detect range-after corruption")
corrupt_image_bytes = bytearray(image_bytes)
corrupt_image_bytes[block_start + leading : block_end] = corrupted_stream
corrupt_image = temp / f"{codec}-corrupt.erofs"
corrupt_image.write_bytes(corrupt_image_bytes)
corrupt_fsck = fsck_image(
fsck,
corrupt_image,
temp / f"extract-{codec}-corrupt",
f"{codec}-corrupt",
False,
)
return {
"codec": codec,
"corruption": {
"full_errno": corrupt_full_errno,
"fsck": corrupt_fsck,
"starts_after_partial_consumed": corruption_start > partial_record["consumed"],
"starts_at_stream_byte": corruption_start,
},
"extent": {**selected, "leading_zero_bytes": leading, "stream_bytes": len(stream)},
"full": {**full_record, "policy_errno": 0, "output_sha256": sha256_bytes(full_output)},
"image_repeated_sha256": hashes,
"legal_fsck": valid_fsck,
"partial": {
**partial_record,
"corrupt_policy_errno": 0,
"output_matches_full_slice": True,
"policy_errno": 0,
"requested_bytes": partial_size,
},
"tail": {
**tail_record,
"bytes": len(tail),
"fsck": tail_fsck,
"nonzero": True,
"policy_errno": tail_errno,
},
"truncated": {**truncated_record, "fsck": truncated_fsck, "policy_errno": truncated_errno},
}
def finalize() -> None:
lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(lines) + "\n", encoding="ascii")
result: dict[str, Any] | None = None
exit_code = 0
owned_temp: str | None = None
try:
if SPEC.get("schema") != 1 or SPEC.get("candidate") != "P15-083" or SPEC.get("gate") != "G04":
raise GateFailure("INFRA_BLOCKED", "invalid P15-083 input schema")
resolved = git("rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise GateFailure("INFRA_BLOCKED", f"P15-083 must replay {SPEC['required_base']}, got {resolved}")
sources = {path: source_at(resolved, path) for path in SPEC["source_sha256"]}
hashes = {path: sha256_bytes(text.encode("utf-8")) for path, text in sources.items()}
if hashes != SPEC["source_sha256"]:
raise GateFailure("INFRA_BLOCKED", "frozen DUT/Linux source identity changed")
write_json(OUTPUT / "source-sha256.json", hashes)
freebsd_head = subprocess.check_output(["git", "-C", str(FREEBSD_SRC), "rev-parse", "HEAD"], text=True).strip()
freebsd_hashes = {path: sha256_path(FREEBSD_SRC / path) for path in SPEC["freebsd"]["sha256"]}
if freebsd_head != SPEC["freebsd"]["head"] or freebsd_hashes != SPEC["freebsd"]["sha256"]:
raise GateFailure("INFRA_BLOCKED", "FreeBSD source identity changed")
errno_source = (FREEBSD_SRC / "sys/sys/errno.h").read_text(encoding="utf-8")
if "#define\tEINTEGRITY\t97" not in errno_source:
raise GateFailure("INFRA_BLOCKED", "FreeBSD positive EINTEGRITY changed")
write_json(OUTPUT / "freebsd-source.json", {"head": freebsd_head, "sha256": freebsd_hashes})
write_json(OUTPUT / "toolchain.json", verify_tools())
write_json(OUTPUT / "source-contract.json", verify_source_contract(sources))
with tempfile.TemporaryDirectory(prefix="p15-083-g04-") as temporary:
owned_temp = temporary
temp = Path(temporary)
source_dir = temp / "source"
source = make_source(source_dir)
oracle, library_record = compile_oracle(temp)
write_json(OUTPUT / "independent-libraries.json", library_record)
records = [
evaluate_codec(codec, codec_spec, source, source_dir, oracle, temp)
for codec, codec_spec in sorted(SPEC["codecs"].items())
]
if {record["codec"] for record in records} != {"deflate", "lzma", "zstd"}:
raise GateFailure("STOP", "P15-083 requires all three non-LZ4 codec policies")
write_json(OUTPUT / "codec-results.json", records)
result = {
"b27": "AUTHORIZED",
"candidate": "P15-083",
"cleanup": "PASS",
"codecs": {record["codec"]: "GO" for record in records},
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"oracle": "real erofs-utils 1.8.6 images + liblzma/zlib/libzstd consumed-byte oracle",
"policy": {
"deflate": "strip EROFS leading zero padding; full raw stream must reach Z_STREAM_END with no unread bytes",
"lzma": "strip EROFS leading zero padding; MicroLZMA compressed size is exact and all bytes must be consumed",
"zstd": "strip EROFS leading zero padding; one frame must complete with no unread bytes",
},
"qemu": "NOT_RUN",
"qemu_reason": "Stage0 policy and compatibility oracle is complete on real host fixtures; B27 acceptance owns TC176 QEMU",
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"schema": 1,
"status": "GO",
"typed_errno": "PASS",
}
write_json(OUTPUT / "result.json", result)
cleanup_record = {
"owned_temp": owned_temp,
"owned_temp_removed": owned_temp is not None and not Path(owned_temp).exists(),
"protected_pid_touched": False,
"protected_port_touched": False,
"qemu_started": False,
"shared_base_image_touched": False,
}
if not cleanup_record["owned_temp_removed"]:
raise GateFailure("INFRA_BLOCKED", "owned gate temporary directory survived cleanup")
write_json(OUTPUT / "owned-cleanup.json", cleanup_record)
except GateFailure as failure:
result = {
"b27": "STOP-NO-SOURCE" if failure.status == "STOP" else "NOT_RUN",
"candidate": "P15-083",
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"qemu": "NOT_RUN",
"reason": failure.reason,
"requested_base": REQUESTED_BASE,
"schema": 1,
"status": failure.status,
}
write_json(OUTPUT / "result.json", result)
exit_code = 1 if failure.status == "STOP" else 21
except (OSError, subprocess.SubprocessError, ValueError) as failure:
result = {
"b27": "NOT_RUN",
"candidate": "P15-083",
"full_feature_suite": "NOT_RUN",
"gate": "G04",
"qemu": "NOT_RUN",
"reason": f"gate infrastructure failure: {failure}",
"requested_base": REQUESTED_BASE,
"schema": 1,
"status": "INFRA_BLOCKED",
}
write_json(OUTPUT / "result.json", result)
exit_code = 21
finally:
finalize()
print(json.dumps(result, sort_keys=True))
raise SystemExit(exit_code)
PY
tests/pre15/gates/P15-086-input.json
+187
View File
@@ -0,0 +1,187 @@
{
"candidate": "P15-086",
"cpu": {
"iterations_per_sample": 25,
"samples": 9
},
"freebsd": {
"head": "106727738dcfb6c001b46f25363b91cece970085",
"sha256": {
"sys/sys/errno.h": "4e615f248a900c6c240c0c87844fd60a8bdffc8a34259d876d5dfde74bd9e42c"
}
},
"gates": [
"G04",
"G05"
],
"hard_budget": {
"caller_bytes": 1048576,
"decoded_bytes": 12582912,
"input_bytes": 1048576,
"workspace_bytes": {
"deflate": 524288,
"lz4": 0,
"lzma": 9437184,
"zstd": 4194304
}
},
"libraries": {
"liblz4": "1.10.0",
"liblzma": "5.8.1",
"libzstd": "1.5.7",
"zlib": "1.3.1"
},
"lz4": {
"generator_url": "https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git",
"generators": [
{
"commit": "ee97fe5fb77c737df0f77d92ab0d92edd3a11be6",
"name": "v1.4"
},
{
"commit": "ce36273833096f1b7e828309d9b1caa37132092d",
"name": "v1.7"
},
{
"commit": "3689cbc2349bff05807d2f939146e92eb1bfaea1",
"name": "v1.8.6"
}
],
"mkfs_args": [
"-T0",
"-U86000000-0000-4000-8000-000000000086",
"--all-root",
"-x-1",
"-zlz4",
"-C4096",
"-Elegacy-compress"
],
"source": {
"segments": [
{
"byte": 69,
"length": 262144
},
{
"byte": 82,
"length": 262144
},
{
"byte": 79,
"length": 262144
},
{
"byte": 70,
"length": 262144
}
],
"sha256": "b4802e36692c8124aa80813711cf03d5cec47729ecd25acf5cc4fd53c484ea80",
"size": 1048576
}
},
"ranges": [
{
"length": 4096,
"name": "prefix",
"offset": 0
},
{
"length": 4096,
"name": "cross-page",
"offset": 3584
},
{
"length": 4096,
"name": "middle",
"offset": 8192
},
{
"length": 4096,
"name": "tail",
"offset": -4096
}
],
"required_base": "6bf5724619be70f805bbe7d1ba77dd70cdced69f",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/compress.h": "41dba5f7bf0c52f72ae26ea5e6deff66361ec7ab38403a65eba12035183edb68",
"repo-pre-15/src/decompressor.c": "76289dcb494994f77c1ba6ff10bbbfe55e7b4baa6a1724c35af14ae1611c537c",
"repo-pre-15/src/decompressor_deflate.c": "66080ad9f9534e74692c6fa41b6649dd12dd060217363eb9697ac94deb4604f9",
"repo-pre-15/src/decompressor_lz4.c": "ca5e5bd6142f9e3c3ea66849900f05799dbc7d6d1a97e0d5a988c28edb90ea98",
"repo-pre-15/src/decompressor_lzma.c": "f515593b9fcc7627bebb93bcdc1531a13f4d91c70faec34ffcf7a54a8a279aa0",
"repo-pre-15/src/decompressor_zstd.c": "fd14509e3a29629d0a85a67306cea9e351665b28d1dbd510b68be859bc071ba1",
"repo-pre-15/src/zdata.c": "3eeb5dae825d7028793a2e1d19a80d24d3f78cdc2e2d5e19466d8fb1208f3242",
"src-linux/compress.h": "d7a76de0d0a43b2369635e2ad6143cc0a7c3e5e8c123a48edec41d281c44839c",
"src-linux/decompressor.c": "caf1c501d00a5a2c9cda5fc0b59d2823eaedf0161a130ca69cd5e7c455128709",
"src-linux/decompressor_deflate.c": "0ddb56c27cacc63aa9f7a10a55cfd76ee2ba0bab15cbac7f6d8684203fb2c67b",
"src-linux/decompressor_lzma.c": "5a23b4455767c30c8d90e3e5b10bc3c430d94359727affae2e6f7824f106d5a8",
"src-linux/decompressor_zstd.c": "4f8a4961fa6b4219fcfe91c04ba37179ec4dde296fcf9f91a5d5b27d8c272607",
"src-linux/zdata.c": "358869da60dcdafd13bc1ff0cd28e864c13b46a49d9f556d9b37139f7e0f4e79"
},
"stream": {
"codecs": {
"deflate": {
"dict_size": 32768,
"mkfs_args": [
"-T0",
"-U86100000-0000-4000-8000-000000000086",
"--all-root",
"-x-1",
"--workers=1",
"-zdeflate,level=1,dictsize=32768",
"-C4096"
]
},
"lzma": {
"dict_size": 65536,
"mkfs_args": [
"-T0",
"-U86200000-0000-4000-8000-000000000086",
"--all-root",
"-x-1",
"--workers=1",
"-zlzma,level=6,dictsize=65536",
"-C4096"
]
},
"zstd": {
"dict_size": 65536,
"mkfs_args": [
"-T0",
"-U86300000-0000-4000-8000-000000000086",
"--all-root",
"-x-1",
"--workers=1",
"-zzstd,level=3,dictsize=65536",
"-C4096"
]
}
},
"source": {
"line_count": 4096,
"sha256": "3c6a1cd405abc8b67b0100dbceebd0f918836563c7a5f705b9cfc178b35fca28",
"size": 151552
}
},
"thresholds": {
"minimum_peak_temporary_reduction_percent": 20.0
},
"tools": {
"dump.erofs": {
"path": "/usr/bin/dump.erofs",
"sha256": "7956eea01c768869d23deaf9555d70343693ffaf6212cf94cc9ed04853add0cd"
},
"fsck.erofs": {
"path": "/usr/bin/fsck.erofs",
"sha256": "09bdee6a2dc7ccdc05547470d2b1f66dc5e1007890b8c3d8f1352e5767eb6b29"
},
"liblz4": {
"path": "/lib/x86_64-linux-gnu/liblz4.so.1",
"sha256": "67bf8b84af77e962e09be661944f7c61393c4806746734e36db9fbdaebc1de36"
},
"mkfs.erofs": {
"path": "/usr/bin/mkfs.erofs",
"sha256": "544e2e502db2302710d7ffdb115eeb26bec374fb4a1ecf12c7aec1156dacc94c"
}
}
}
tests/pre15/gates/P15-086.sh
Executable
+1445
View File
File diff suppressed because it is too large Load Diff
tests/pre15/gates/P15-087-input.json
+73
View File
@@ -0,0 +1,73 @@
{
"benchmark": {
"buffer_bytes": 1048576,
"cold_runs": 5,
"entry_count": 18000,
"name_length": 196,
"random_buffer_bytes": 65536,
"run_order": [
"baseline",
"candidate",
"candidate",
"baseline",
"baseline",
"candidate",
"candidate",
"baseline",
"baseline",
"candidate"
]
},
"candidate": "P15-087",
"fixture": {
"minimum_directory_bytes": 3145728,
"uuid": "00000000-0000-0000-0000-000000000087"
},
"freebsd_head": "106727738dcfb6c001b46f25363b91cece970085",
"freebsd_sha256": {
"sys/geom/geom_vfs.c": "e9ffabf2362e15bf70c08036eff4660a479c3229af2798d922d1bc1fb0117aeb",
"sys/kern/vfs_bio.c": "54c64a4d96cb3a511ab4ac7dad132b4b8e105d2003f37b01ea40c93abce4d5b4",
"sys/sys/buf.h": "d13d4e50fce60c52c760c19380ef01ce14fa555103ab255721dc921cc6aa718b",
"sys/sys/vnode.h": "244d7c51cb75d6c19d07a8b9dab85183f87d05c221bbd760af05839ceb2e4a0d"
},
"gate": "G11",
"host_inputs": {
"askpass_path": "/work/build/.repo22-ssh-askpass",
"askpass_sha256": "967a879f0175d700df7ee3e40b18a41155a7216c05868d0933f0cf08911b7121",
"base_image": "/work/build/vm-freebsd-build.qcow2.bp",
"base_image_format": "qcow2",
"base_image_size": 13359054848,
"ssh_key_path": "/root/.ssh/id_ed25519",
"ssh_key_sha256": "f76af546955d62f9fe2d52479f314f83ec85c01068011eb59ef894a8ac1f3833"
},
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
"protected": {
"pid": 26318,
"port": 9222
},
"prototype": {
"max_readahead_bytes": 1048576,
"max_readahead_slots": 256
},
"qemu": {
"boot_timeout_seconds": 300,
"cpus": 2,
"guest_timeout_seconds": 840,
"memory_mb": 3072
},
"required_base": "c7d692acf9166f5c5e42335db2de64f0793ba8a2",
"schema": 1,
"source_sha256": {
"repo-pre-15/build.sh": "7ea125c6100d1ecc1315b7ef315c32185c3b242805fd96bac1ffd6b9c12dd21e",
"repo-pre-15/src/Makefile": "524df7843a0f03e7002311a977adee53a90ccaccfbe21901da641a3b177dfd47",
"repo-pre-15/src/data.c": "cb22072bd4c092aa6a5376add8c8d9d6f94297f6292900ecc2eef9b01c69e92a",
"repo-pre-15/src/dir.c": "842cc3abd308388fd57f69222f4f6144f2d3942f00d2dcf4e289ad2dd5959cce",
"repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9",
"src-linux/data.c": "8625cdc01e5405f856178ae8fd559696ae85f607f19caf229b867a3b7479318a",
"src-linux/dir.c": "4d0f4e687c5776719bc61454b648ff758acea6dee11f406bf8a38a605f58c763"
},
"thresholds": {
"maximum_extra_provider_reads_percent": 25.0,
"minimum_cold_median_improvement_percent": 10.0
}
}
tests/pre15/gates/P15-087.sh
Executable
+1419
View File
File diff suppressed because it is too large Load Diff
tests/pre15/gates/P15-092-input.json
+42
View File
@@ -0,0 +1,42 @@
{
"candidate": "P15-092",
"freebsd_head": "106727738dcfb6c001b46f25363b91cece970085",
"freebsd_sha256": {
"sys/kern/vfs_hash.c": "ed48e09f31c1ab5071b241c823bede9f1db46494f988dc9c3f2c2e31ed1d79c0",
"sys/kern/vfs_lookup.c": "7958081eda9a137a9cd913b959e86b9ed61eff2f9d6cdb5ce5bcfc8211307f24",
"sys/sys/vnode.h": "6a7c77fda50f721ed3539d81f4b77e5ea3185827ce3917cb331b51fe9dd2025f",
"sys/vm/vm_mmap.c": "d7bcbb749fae7fa4d36d544762b18dac25e605a0d9dc5f36c8e5b5150b531565"
},
"gate": "G11",
"linux_head": "audit-2-semantic/14-source-hashes.txt",
"mkfs_version": "mkfs.erofs (erofs-utils) 1.8.6",
"protected": {
"base_image": "/work/debug-qemu/local/vm-freebsd-build.qcow2.bp",
"pid": 26318,
"port": 9222
},
"required_base": "b22dae8dc634c68db4ea89dccade91350614c139",
"schema": 1,
"source_sha256": {
"repo-pre-15/src/erofs_fs.h": "0a49ac30ecbcea020c3909beb972ac4287ca704ebc49a9dccfcd6827884589e1",
"repo-pre-15/src/erofs_vnops.c": "f28555606ca006d1646a2fee75b98b2f7452d4283c00b087012b8a6c4ef4bf64",
"repo-pre-15/src/inode.c": "dee361837ea104f455c851fe4c268bf5d7e10cd9dd2ef58645aea819137639f6",
"repo-pre-15/src/internal.h": "eec416077040587ad0756814c4323425889d10bd9b4143fcfc63299bf4f90ae9",
"repo-pre-15/src/namei.c": "7c78c48927f81b8e54b75c3a561a4768441c00622a126589d3bd5a3c6b6ed152",
"repo-pre-15/src/super.c": "61f31906bb7d1872c99cd3cf3859a34daf29a02d91a3caa1bfc8187af562a3d2",
"src-linux/erofs_fs.h": "6cb322cf7506858c3c82de3c81026039c543f448201c9c551fd81360cca67e93",
"src-linux/inode.c": "a15562e0782e155a0744a7ba9d2f557519a583b64853ed75180cef2b4dfae1b3",
"src-linux/super.c": "8bda458cca758d8aa9c5a5b05361b2131b896f73fd194f6ad9a8e011e3481bf9"
},
"uuid": "00000000-0000-0000-0000-000000000092",
"utils_head": "7db78788b000999e2de88decd2ba90654f26171c",
"utils_sha256": {
"include/erofs_fs.h": "02b01a99fe3180f86bb0372efc332efc243142d37aed2819045ae69717a9c915",
"lib/inode.c": "382bf7ccc22436ab9997a3fa417a0424216a6f3b58a95d7c9a1d182a9e228bc1",
"lib/namei.c": "faeef5248a81c2dee457c5009f3a47e194befe2ffa6ef9f096e029ba63ca014f"
},
"write_set": [
"repo-pre-15/src/inode.c",
"repo-pre-15/src/namei.c"
]
}
tests/pre15/gates/P15-092.sh
Executable
+881
View File
@@ -0,0 +1,881 @@
#!/bin/sh
set -eu
gate_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
root=$(CDPATH= cd -- "$gate_dir/../../../.." && pwd -P)
dut=$root/repo-pre-15
input=$gate_dir/P15-092-input.json
freebsd_src=${FREEBSD_SRC:-/work/build/freebsd-src}
utils_src=${EROFS_UTILS_SRC:-/work/build/erofs-utils-main}
base=
output=
qemu_base=
qemu_key=
host_only=0
while test "$#" -gt 0; do
case "$1" in
--base)
test "$#" -ge 2 || { printf '%s\n' '--base requires a commit' >&2; exit 2; }
base=$2
shift 2
;;
--output)
test "$#" -ge 2 || { printf '%s\n' '--output requires a directory' >&2; exit 2; }
output=$2
shift 2
;;
--qemu-base)
test "$#" -ge 2 || { printf '%s\n' '--qemu-base requires an image' >&2; exit 2; }
qemu_base=$2
shift 2
;;
--qemu-key)
test "$#" -ge 2 || { printf '%s\n' '--qemu-key requires a key' >&2; exit 2; }
qemu_key=$2
shift 2
;;
--host-only)
host_only=1
shift
;;
*)
printf 'unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
done
test -n "$base" || { printf '%s\n' '--base is required' >&2; exit 2; }
test -n "$output" || { printf '%s\n' '--output is required' >&2; exit 2; }
test -f "$input" || { printf 'missing input: %s\n' "$input" >&2; exit 2; }
test -d "$freebsd_src/sys" || { printf 'missing FreeBSD source: %s\n' "$freebsd_src" >&2; exit 2; }
test -d "$utils_src/lib" || { printf 'missing erofs-utils source: %s\n' "$utils_src" >&2; exit 2; }
if test "$host_only" -eq 0; then
test -n "$qemu_base" || { printf '%s\n' '--qemu-base is required' >&2; exit 2; }
test -n "$qemu_key" || { printf '%s\n' '--qemu-key is required' >&2; exit 2; }
test -f "$qemu_base" || { printf 'missing QEMU base: %s\n' "$qemu_base" >&2; exit 2; }
test -f "$qemu_key" || { printf 'missing QEMU key: %s\n' "$qemu_key" >&2; exit 2; }
fi
for tool in dump.erofs fsck.erofs git mkfs.erofs python3 sha256sum timeout; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required tool: %s\n' "$tool" >&2
exit 2
}
done
if test "$host_only" -eq 0; then
for tool in clang qemu-img qemu-system-x86_64 scp ssh tar; do
command -v "$tool" >/dev/null 2>&1 || {
printf 'missing required QEMU tool: %s\n' "$tool" >&2
exit 2
}
done
fi
case "$output" in
/*) ;;
*) output=$PWD/$output ;;
esac
test ! -e "$output" || { printf 'refusing existing output: %s\n' "$output" >&2; exit 2; }
mkdir -p "$output/host"
python3 -B - "$root" "$input" "$base" "$output/host" "$freebsd_src" \
"$utils_src" "${qemu_base:-/nonexistent}" <<'PY'
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import stat
import struct
import subprocess
import sys
import tempfile
from typing import Any
ROOT = Path(sys.argv[1])
INPUT = Path(sys.argv[2])
REQUESTED_BASE = sys.argv[3]
OUTPUT = Path(sys.argv[4])
FREEBSD_SRC = Path(sys.argv[5])
UTILS_SRC = Path(sys.argv[6])
QEMU_BASE = Path(sys.argv[7]).resolve()
SPEC = json.loads(INPUT.read_text(encoding="ascii"))
SUPER = 1024
MAGIC = 0xE0F5E1E2
CRC32C_POLY = 0x82F63B78
S_IFMT = 0o170000
S_IFDIR = 0o040000
NLINK_ONE_BIT = 4
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_path(path: Path) -> str:
return sha256_bytes(path.read_bytes())
def write_json(path: Path, value: Any) -> None:
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
def run(argv: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
return subprocess.run(
argv,
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
def git(root: Path, *args: str) -> str:
return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip()
def source_at(commit: str, path: str) -> str:
completed = run(["git", "-C", str(ROOT), "show", f"{commit}:{path}"])
if completed.returncode != 0:
raise SystemExit(f"cannot read {path} at {commit}: {completed.stdout}")
return completed.stdout
def crc32c(data: bytes | bytearray, seed: int = 0xFFFFFFFF) -> int:
value = seed
for byte in data:
value ^= byte
for _ in range(8):
value = (value >> 1) ^ (CRC32C_POLY if value & 1 else 0)
return value & 0xFFFFFFFF
class Image:
def __init__(self, data: bytes | bytearray):
self.data = bytearray(data)
if len(self.data) < SUPER + 144 or self.u32(SUPER) != MAGIC:
raise ValueError("invalid EROFS image")
self.block_bits = self.data[SUPER + 12]
self.block_size = 1 << self.block_bits
self.meta_blkaddr = self.u32(SUPER + 40)
self.root_nid = self.u16(SUPER + 14)
if self.root_nid == 0:
self.root_nid = self.u64(SUPER + 112)
if self.checksum_end > len(self.data):
raise ValueError("truncated checksummed range")
@property
def checksum_end(self) -> int:
span = self.block_size - SUPER if self.block_size > SUPER else self.block_size
return SUPER + span
def clone(self) -> "Image":
return Image(self.data)
def u16(self, offset: int) -> int:
return struct.unpack_from("<H", self.data, offset)[0]
def u32(self, offset: int) -> int:
return struct.unpack_from("<I", self.data, offset)[0]
def u64(self, offset: int) -> int:
return struct.unpack_from("<Q", self.data, offset)[0]
def put_u16(self, offset: int, value: int) -> None:
struct.pack_into("<H", self.data, offset, value)
def put_u32(self, offset: int, value: int) -> None:
struct.pack_into("<I", self.data, offset, value)
def put_u64(self, offset: int, value: int) -> None:
struct.pack_into("<Q", self.data, offset, value)
def checksum_valid(self) -> bool:
expected = self.u32(SUPER + 4)
canonical = bytearray(self.data[SUPER : self.checksum_end])
struct.pack_into("<I", canonical, 4, 0)
return expected == crc32c(canonical)
def update_checksum(self) -> None:
self.put_u32(SUPER + 4, 0)
self.put_u32(SUPER + 4, crc32c(self.data[SUPER : self.checksum_end]))
if not self.checksum_valid():
raise AssertionError("checksum update failed")
def inode(self, nid: int) -> dict[str, int]:
offset = (self.meta_blkaddr << self.block_bits) + (nid << 5)
ifmt = self.u16(offset)
version = ifmt & 1
inode_size = 32 if version == 0 else 64
if offset + inode_size > len(self.data):
raise ValueError(f"inode {nid} is outside the image")
xattr_count = self.u16(offset + 2)
xattr_size = 0 if xattr_count == 0 else 12 + (xattr_count - 1) * 4
mode = self.u16(offset + 4)
raw_nlink = self.u16(offset + 6) if version == 0 else self.u32(offset + 44)
nlink_one = version == 0 and mode & S_IFMT != S_IFDIR and bool(
ifmt & (1 << NLINK_ONE_BIT)
)
size = self.u32(offset + 8) if version == 0 else self.u64(offset + 8)
return {
"nid": nid,
"offset": offset,
"ifmt": ifmt,
"version": version,
"inode_size": inode_size,
"xattr_size": xattr_size,
"layout": (ifmt >> 1) & 7,
"mode": mode,
"raw_nlink": raw_nlink,
"nlink": 1 if nlink_one else raw_nlink,
"nlink_one": int(nlink_one),
"size": size,
"startblk": self.u32(offset + 16),
}
def set_nlink(self, nid: int, value: int) -> dict[str, int]:
inode = self.inode(nid)
if inode["version"] == 0:
if inode["mode"] & S_IFMT != S_IFDIR:
self.put_u16(inode["offset"], inode["ifmt"] & ~(1 << NLINK_ONE_BIT))
self.put_u16(inode["offset"] + 6, value)
else:
self.put_u32(inode["offset"] + 44, value)
return self.inode(nid)
def data_offset(self, inode: dict[str, int]) -> int:
if inode["layout"] == 2:
return inode["offset"] + inode["inode_size"] + inode["xattr_size"]
if inode["layout"] == 0:
return inode["startblk"] << self.block_bits
raise ValueError(f"unsupported directory layout {inode['layout']}")
def directory(self, nid: int) -> dict[str, dict[str, int]]:
inode = self.inode(nid)
if inode["mode"] & S_IFMT != S_IFDIR:
raise ValueError(f"inode {nid} is not a directory")
data_offset = self.data_offset(inode)
size = inode["size"]
first_nameoff = self.u16(data_offset + 8)
if first_nameoff == 0 or first_nameoff % 12 != 0 or first_nameoff >= size:
raise ValueError("invalid first name offset")
count = first_nameoff // 12
records: dict[str, dict[str, int]] = {}
for index in range(count):
entry = data_offset + index * 12
nameoff = self.u16(entry + 8)
endoff = self.u16(entry + 20) if index + 1 < count else size
raw = bytes(self.data[data_offset + nameoff : data_offset + endoff])
name = raw.split(b"\0", 1)[0].decode("ascii")
if not name or name in records:
raise ValueError(f"bad directory name {name!r}")
records[name] = {
"entry_offset": entry,
"nid": self.u64(entry),
"file_type": self.data[entry + 10],
}
return records
def make_source(path: Path, extended_uid: bool) -> None:
previous_umask = os.umask(0o022)
try:
path.mkdir(mode=0o755)
(path / "compact").write_bytes(b"compact zero-nlink target\n")
(path / "extended").write_bytes(b"extended zero-nlink target\n")
if extended_uid:
os.chown(path / "extended", 70000, 0)
(path / "hard-a").write_bytes(b"hardlink control\n")
os.link(path / "hard-a", path / "hard-b")
(path / "orphan").write_bytes(b"orphan raw-vget control\n")
(path / "subdir").mkdir(mode=0o755)
(path / "subdir" / "child").write_bytes(b"directory child\n")
for regular in ("compact", "extended", "hard-a", "hard-b", "orphan"):
os.chmod(path / regular, 0o644)
os.chmod(path / "subdir" / "child", 0o644)
finally:
os.umask(previous_umask)
def build_seed(work: Path, output: Path, inode_mode: str) -> tuple[bytes, str, list[str]]:
work.mkdir()
source = work / "source"
make_source(source, inode_mode == "extended")
command = [
"mkfs.erofs",
"-d0",
"-T0",
"--all-time",
"--ignore-mtime",
"--workers=1",
"-x-1",
"-E",
"noinline_data",
"-E",
f"force-inode-{inode_mode}",
"-U",
SPEC["uuid"],
str(output),
str(source),
]
env = dict(os.environ)
env.pop("SOURCE_DATE_EPOCH", None)
completed = run(command, env=env)
if completed.returncode != 0:
raise SystemExit(f"mkfs.erofs failed: {completed.stdout}")
canonical = command[:-2] + [f"SEED-{inode_mode}.erofs", "SOURCE"]
return output.read_bytes(), completed.stdout, canonical
resolved = git(ROOT, "rev-parse", f"{REQUESTED_BASE}^{{commit}}")
if resolved != SPEC["required_base"]:
raise SystemExit(f"wrong BASE: expected {SPEC['required_base']}, got {resolved}")
if git(FREEBSD_SRC, "rev-parse", "HEAD") != SPEC["freebsd_head"]:
raise SystemExit("FreeBSD source HEAD changed")
if git(UTILS_SRC, "rev-parse", "HEAD") != SPEC["utils_head"]:
raise SystemExit("erofs-utils source HEAD changed")
if QEMU_BASE == Path(SPEC["protected"]["base_image"]):
raise SystemExit("refusing the protected base bp")
if SPEC["protected"]["port"] >= 32768:
raise SystemExit("protected port unexpectedly overlaps ephemeral ports")
sources: dict[str, str] = {}
source_hashes: dict[str, str] = {}
for path, expected in SPEC["source_sha256"].items():
content = source_at(resolved, path)
actual = sha256_bytes(content.encode("utf-8"))
if actual != expected:
raise SystemExit(f"source hash changed for {path}: {actual}")
sources[path] = content
source_hashes[path] = actual
for path, expected in SPEC["freebsd_sha256"].items():
actual = sha256_path(FREEBSD_SRC / path)
if actual != expected:
raise SystemExit(f"FreeBSD hash changed for {path}: {actual}")
for path, expected in SPEC["utils_sha256"].items():
actual = sha256_path(UTILS_SRC / path)
if actual != expected:
raise SystemExit(f"erofs-utils hash changed for {path}: {actual}")
mkfs_version = run(["mkfs.erofs", "-V"]).stdout.splitlines()[0]
if mkfs_version != SPEC["mkfs_version"]:
raise SystemExit(f"mkfs version changed: {mkfs_version!r}")
namei = sources["repo-pre-15/src/namei.c"]
inode_source = sources["repo-pre-15/src/inode.c"]
vnops = sources["repo-pre-15/src/erofs_vnops.c"]
super_source = sources["repo-pre-15/src/super.c"]
linux_inode = sources["src-linux/inode.c"]
linux_super = sources["src-linux/super.c"]
freebsd_vm = (FREEBSD_SRC / "sys/vm/vm_mmap.c").read_text(encoding="utf-8")
source_anchors = {
"freebsd_disk_compact_nlink": "vi->nlink = le16toh(dic->i_nb.nlink);" in inode_source,
"freebsd_disk_extended_nlink": "vi->nlink = le32toh(die->i_nlink);" in inode_source,
"freebsd_inode_decode_has_no_zero_reject": "vi->nlink == 0" not in inode_source,
"freebsd_lookup_has_namespace_edge": "error = erofs_namei(sbi, dir, &qname, &nid, &dtype);" in namei,
"freebsd_lookup_calls_raw_vget": "error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp);" in namei,
"freebsd_lookup_publishes_namecache": "cache_enter(dvp, vp, cnp);" in namei,
"freebsd_raw_vget_registered": ".vfs_vget = erofs_vget," in super_source,
"freebsd_root_uses_raw_vget": "error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);" in super_source,
"freebsd_fhtovp_filters_zero": "vi->mode == 0 || vi->nlink == 0 || vi->nid != nid" in super_source,
"freebsd_hash_insert_before_decode": vnops.index("error = vfs_hash_insert(") < vnops.index("error = erofs_read_inode("),
"freebsd_constructed_before_return": vnops.index("vn_set_state(vp, VSTATE_CONSTRUCTED);") < vnops.index("*vpp = vp;"),
"freebsd_vget_has_no_hit_indicator": "bool shared;" in vnops and "bool created" not in vnops,
"freebsd_vput_does_not_imply_vgone": "vgone(vp);\n\t\tvput(vp);" in vnops,
"freebsd_vfs_allows_unlinked_vnode": "if (va.va_nlink == 0)" in freebsd_vm,
"linux_compact_sets_disk_nlink": "set_nlink(inode, le16_to_cpu(dic->i_nb.nlink));" in linux_inode,
"linux_extended_sets_disk_nlink": "set_nlink(inode, le32_to_cpu(die->i_nlink));" in linux_inode,
"linux_inode_has_no_zero_reject": "i_nlink == 0" not in linux_inode,
"linux_super_has_no_zero_reject": "i_nlink == 0" not in linux_super,
}
if not all(source_anchors.values()):
raise SystemExit(f"source anchors changed: {source_anchors}")
fixtures = OUTPUT / "fixtures"
fixtures.mkdir()
with tempfile.TemporaryDirectory(prefix="p15-092-gate-") as temporary:
temp = Path(temporary)
first_bytes, first_stdout, compact_command = build_seed(
temp / "compact-first", temp / "compact-first.erofs", "compact"
)
second_bytes, second_stdout, compact_repeat_command = build_seed(
temp / "compact-second", temp / "compact-second.erofs", "compact"
)
extended_bytes, extended_stdout, extended_command = build_seed(
temp / "extended-first", temp / "extended-first.erofs", "extended"
)
extended_repeat_bytes, extended_repeat_stdout, extended_repeat_command = build_seed(
temp / "extended-second", temp / "extended-second.erofs", "extended"
)
write_json(
OUTPUT / "generator-attempts.json",
{
"commands": [compact_command, extended_command],
"compact_source": [
{
"gid": path.lstat().st_gid,
"mode": stat.S_IMODE(path.lstat().st_mode),
"path": str(path.relative_to(temp / "compact-first/source")),
"uid": path.lstat().st_uid,
}
for path in sorted((temp / "compact-first/source").rglob("*"))
],
},
)
if (
first_bytes != second_bytes
or first_stdout != second_stdout
or compact_command != compact_repeat_command
or extended_bytes != extended_repeat_bytes
or extended_stdout != extended_repeat_stdout
or extended_command != extended_repeat_command
):
raise SystemExit("seed generation is not byte reproducible")
seed_path = fixtures / "seed.erofs"
seed_path.write_bytes(first_bytes)
extended_seed_path = fixtures / "seed-extended.erofs"
extended_seed_path.write_bytes(extended_bytes)
seed = Image(first_bytes)
extended_seed = Image(extended_bytes)
if not seed.checksum_valid() or not extended_seed.checksum_valid():
raise SystemExit("seed checksum is invalid")
entries = seed.directory(seed.root_nid)
required = {".", "..", "compact", "extended", "hard-a", "hard-b", "orphan", "subdir"}
if not required.issubset(entries):
raise SystemExit(f"seed entries missing: {sorted(required - entries.keys())}")
if entries["hard-a"]["nid"] != entries["hard-b"]["nid"]:
raise SystemExit("hardlink names do not share one NID")
extended_entries = extended_seed.directory(extended_seed.root_nid)
seed_inodes = {name: seed.inode(record["nid"]) for name, record in entries.items()}
extended_inodes = {
name: extended_seed.inode(record["nid"])
for name, record in extended_entries.items()
}
if seed_inodes["compact"]["version"] != 0:
raise SystemExit("compact control is not compact")
if extended_inodes["extended"]["version"] != 1:
raise SystemExit("extended control is not extended")
if seed_inodes["hard-a"]["nlink"] != 2:
raise SystemExit("hardlink control does not have nlink 2")
mutations = []
def emit_zero(case_id: str, name: str, source: Image = seed) -> None:
image = source.clone()
nid = image.directory(image.root_nid)[name]["nid"]
before = image.inode(nid)
after = image.set_nlink(nid, 0)
image.update_checksum()
path = fixtures / f"{case_id}.erofs"
path.write_bytes(image.data)
mutations.append({
"case": case_id,
"class": "namespace-reachable-zero",
"name": name,
"nid": nid,
"before": before,
"after": after,
"fixture": path.name,
})
emit_zero("reachable-compact-zero", "compact")
emit_zero("reachable-extended-zero", "extended", extended_seed)
emit_zero("reachable-directory-zero", "subdir")
root_zero = seed.clone()
root_before = root_zero.inode(root_zero.root_nid)
root_after = root_zero.set_nlink(root_zero.root_nid, 0)
root_zero.update_checksum()
root_path = fixtures / "root-zero.erofs"
root_path.write_bytes(root_zero.data)
mutations.append({
"case": "root-zero",
"class": "raw-root-zero",
"name": "/",
"nid": root_zero.root_nid,
"before": root_before,
"after": root_after,
"fixture": root_path.name,
})
orphan = seed.clone()
orphan_entries = orphan.directory(orphan.root_nid)
orphan_nid = orphan_entries["orphan"]["nid"]
hard_nid = orphan_entries["hard-a"]["nid"]
orphan.put_u64(orphan_entries["orphan"]["entry_offset"], hard_nid)
orphan.set_nlink(orphan_nid, 0)
orphan.set_nlink(hard_nid, 3)
orphan.update_checksum()
orphan_path = fixtures / "orphan-zero.erofs"
orphan_path.write_bytes(orphan.data)
orphan_names = orphan.directory(orphan.root_nid)
if any(record["nid"] == orphan_nid for record in orphan_names.values()):
raise SystemExit("orphan fixture still has a namespace edge")
mutations.append({
"case": "orphan-zero",
"class": "unreachable-zero",
"name": None,
"nid": orphan_nid,
"before": seed.inode(orphan_nid),
"after": orphan.inode(orphan_nid),
"replacement_nid": hard_nid,
"replacement_nlink": orphan.inode(hard_nid)["nlink"],
"fixture": orphan_path.name,
})
records = []
fixture_digest = hashlib.sha256()
for path in sorted(fixtures.glob("*.erofs")):
image = Image(path.read_bytes())
if not image.checksum_valid():
raise SystemExit(f"fixture checksum invalid: {path.name}")
fsck = run(["fsck.erofs", "-d0", str(path)])
dump_root = run(["dump.erofs", "--path=/", str(path)])
if dump_root.returncode != 0:
raise SystemExit(f"dump root failed: {path.name}")
normal = {}
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir"):
completed = run(["dump.erofs", f"--path=/{name}", str(path)])
normal[name] = {
"exit": completed.returncode,
"nid": int(match.group(1)) if (match := re.search(r"^NID: (\d+)\b", completed.stdout, re.MULTILINE)) else None,
}
record = {
"fixture": path.name,
"sha256": sha256_path(path),
"size": path.stat().st_size,
"checksum_valid": True,
"fsck_exit": fsck.returncode,
"fsck_error_marker": "<E>" in fsck.stdout,
"normal_namespace": normal,
}
records.append(record)
fixture_digest.update(path.name.encode("ascii"))
fixture_digest.update(b"\0")
fixture_digest.update(path.read_bytes())
write_json(
OUTPUT / "generator.json",
{
"commands": [compact_command, extended_command],
"mkfs_version": mkfs_version,
"repeat_byte_identical": True,
"seed_sha256": {
"compact": sha256_bytes(first_bytes),
"extended": sha256_bytes(extended_bytes),
},
},
)
(OUTPUT / "mkfs.stdout").write_text(
first_stdout + extended_stdout, encoding="utf-8"
)
write_json(OUTPUT / "disk-records.json", {"mutations": mutations, "records": records})
write_json(
OUTPUT / "fixture-index.json",
{
"fixture_count": len(records),
"fixture_set_sha256": fixture_digest.hexdigest(),
"fixtures": records,
},
)
fsck_reachable_zero_rejects = []
for record in records:
if record["fixture"].startswith("reachable-"):
fsck_reachable_zero_rejects.append(
record["fsck_exit"] != 0 or record["fsck_error_marker"]
)
boundary = {
"b15_write_set": SPEC["write_set"],
"vnode_constructor_owner": "repo-pre-15/src/erofs_vnops.c",
"constructor_inside_b15_write_set": "repo-pre-15/src/erofs_vnops.c" in SPEC["write_set"],
"post_vget_check_is_before_constructed_publication": False,
"vput_only_performs_vgone_cleanup": False,
"post_vget_vgone_preserves_cached_raw_vget": False,
"pre_vget_full_decode_avoids_duplicate_inode_decode": False,
"mount_wide_scan_required": False,
"namespace_and_raw_entrypoints_distinct": True,
}
format_semantics = {
"mkfs_emits_nonzero_namespace_links": all(
seed_inodes[name]["nlink"] > 0
for name in ("compact", "extended", "hard-a", "hard-b", "orphan", "subdir", ".")
),
"fsck_rejects_all_reachable_zero": all(fsck_reachable_zero_rejects),
"linux_rejects_zero_during_inode_decode": False,
"linux_copies_disk_nlink_into_inode": True,
"freebsd_vfs_permits_live_unlinked_vnode": True,
"freebsd_fhtovp_treats_zero_as_stale": True,
}
go_requirements = {
"format_or_cross_kernel_rule_requires_rejection": (
format_semantics["fsck_rejects_all_reachable_zero"]
and format_semantics["linux_rejects_zero_during_inode_decode"]
),
"reject_before_vnode_publication_within_write_set": (
boundary["constructor_inside_b15_write_set"]
and boundary["post_vget_check_is_before_constructed_publication"]
),
"cleanup_preserves_raw_vget": boundary["post_vget_vgone_preserves_cached_raw_vget"],
"no_duplicate_full_inode_decode": boundary["pre_vget_full_decode_avoids_duplicate_inode_decode"],
"orphan_needs_no_mount_scan": not boundary["mount_wide_scan_required"],
"namespace_raw_split_proven": boundary["namespace_and_raw_entrypoints_distinct"],
}
status = "GO" if all(go_requirements.values()) else "STOP"
reasons = [key for key, value in go_requirements.items() if not value]
write_json(OUTPUT / "source-anchors.json", source_anchors)
write_json(OUTPUT / "source-sha256.json", source_hashes)
write_json(OUTPUT / "boundary.json", boundary)
write_json(OUTPUT / "format-semantics.json", format_semantics)
write_json(
OUTPUT / "host-result.json",
{
"candidate": SPEC["candidate"],
"gate": SPEC["gate"],
"fixture_count": len(records),
"fixture_set_sha256": fixture_digest.hexdigest(),
"go_requirements": go_requirements,
"reasons": reasons,
"requested_base": REQUESTED_BASE,
"resolved_base": resolved,
"schema": 1,
"status": status,
},
)
PY
if test "$host_only" -eq 1; then
cat "$output/host/host-result.json"
exit 22
fi
runtime_tmp=$(mktemp -d "${TMPDIR:-/tmp}/p15-092-runtime.XXXXXX")
runtime_dut=$runtime_tmp/repo-pre-15
runtime_case=$runtime_dut/tests/pre15/cases/P15-092-gate-runtime.sh
runtime_cleanup()
{
rm -rf "$runtime_tmp"
}
trap runtime_cleanup EXIT HUP INT TERM
mkdir -p "$runtime_dut/tests/pre15/cases" "$runtime_dut/tests/pre15/fixtures"
git -C "$root" archive --format=tar --output="$runtime_tmp/frozen-src.tar" \
"$base" repo-pre-15/src
tar -C "$runtime_tmp" -xf "$runtime_tmp/frozen-src.tar"
ln -s "$dut/tests/pre15/fixtures/B14-build-kld.sh" \
"$runtime_dut/tests/pre15/fixtures/B14-build-kld.sh"
python3 -B - "$runtime_case" <<'PY'
from pathlib import Path
import sys
Path(sys.argv[1]).write_text(r'''#!/bin/sh
set -eu
: "${PRE15_LIB_DIR:?PRE15_LIB_DIR is required}"
: "${PRE15_CASE_TMP:?PRE15_CASE_TMP is required}"
: "${PRE15_RUN_DIR:?PRE15_RUN_DIR is required}"
: "${P15_092_GATE_DUT:?P15_092_GATE_DUT is required}"
: "${P15_092_GATE_HOST:?P15_092_GATE_HOST is required}"
. "$PRE15_LIB_DIR/runner.sh"
artifacts=$PRE15_RUN_DIR/artifacts
fixtures=$P15_092_GATE_HOST/fixtures
builder=$P15_092_GATE_DUT/tests/pre15/fixtures/B14-build-kld.sh
module=$PRE15_CASE_TMP/P15-092-baseline-erofs.ko
archive=$PRE15_CASE_TMP/P15-092-fixtures.tar.gz
mkdir -p "$artifacts"
pre15_record_fixture p15-092-host-result "$P15_092_GATE_HOST/host-result.json"
pre15_record_fixture p15-092-disk-records "$P15_092_GATE_HOST/disk-records.json"
pre15_record_fixture p15-092-boundary "$P15_092_GATE_HOST/boundary.json"
pre15_record_fixture p15-092-source-namei "$P15_092_GATE_DUT/src/namei.c"
pre15_record_fixture p15-092-source-inode "$P15_092_GATE_DUT/src/inode.c"
pre15_record_fixture p15-092-source-vnops "$P15_092_GATE_DUT/src/erofs_vnops.c"
if ! /bin/sh "$builder" "$P15_092_GATE_DUT" "$PRE15_FREEBSD_SRC" "$module" \
"$PRE15_CASE_TMP/kld-work" >"$artifacts/kld-build.stdout" \
2>"$artifacts/kld-build.stderr"; then
pre15_dut_fail 'baseline cross-target KLD build failed'
fi
pre15_record_module "$module"
tar -C "$fixtures" -czf "$archive" .
pre15_scp()
{
timeout -k 5 "${PRE15_GUEST_COMMAND_TIMEOUT:-60}" scp -O -q \
-o BatchMode=yes -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
-o "ControlPath=$PRE15_QEMU_CONTROL_PATH" \
-i "$PRE15_QEMU_SSH_KEY" -P "$PRE15_QEMU_SSH_PORT" \
"$1" "[email protected]:$2"
}
pre15_scp "$module" /root/P15-092-baseline-erofs.ko || \
pre15_infra_blocked 'could not transfer baseline KLD'
pre15_scp "$archive" /root/P15-092-fixtures.tar.gz || \
pre15_infra_blocked 'could not transfer zero-nlink fixtures'
pre15_guest_ssh_bounded \
'rm -rf /root/P15-092-fixtures && mkdir /root/P15-092-fixtures && tar -xzf /root/P15-092-fixtures.tar.gz -C /root/P15-092-fixtures' || \
pre15_infra_blocked 'could not prepare guest fixtures'
if pre15_guest_ssh_bounded kldstat -n erofs >/dev/null 2>&1; then
pre15_infra_blocked 'guest already has an EROFS module loaded'
fi
pre15_guest_ssh_bounded kldload /root/P15-092-baseline-erofs.ko || \
pre15_dut_fail 'baseline KLD failed to load'
pre15_own_guest_kld erofs 'P15-092 baseline KLD'
pre15_target_reached
attach_mount()
{
image=$1
mountpoint=$2
label=$3
md=$(pre15_guest_ssh_bounded mdconfig -a -t vnode \
-f "/root/P15-092-fixtures/$image") || \
pre15_dut_fail "$label md attach failed"
case "$md" in
md[0-9]*) ;;
*) pre15_runner_fail "unexpected md unit: $md" ;;
esac
pre15_own_guest_md "$md" "$label md"
pre15_guest_ssh_bounded mkdir -p "$mountpoint"
pre15_guest_ssh_bounded mount -t erofs -o ro "/dev/$md" "$mountpoint" || \
pre15_dut_fail "$label mount failed"
pre15_own_guest_mount "$mountpoint" "$label mount"
}
expect_nlink()
{
path=$1
expected=$2
label=$3
actual=$(pre15_guest_ssh_bounded stat -f '%l' "$path") || \
pre15_dut_fail "$label normal stat failed"
test "$actual" = "$expected" || \
pre15_dut_fail "$label nlink expected $expected got $actual"
printf '%s\t%s\t%s\n' "$label" "$path" "$actual" >>"$artifacts/normal-vnode.tsv"
}
attach_mount seed.erofs /mnt/p15-092-seed seed
expect_nlink /mnt/p15-092-seed/compact 1 seed-compact
expect_nlink /mnt/p15-092-seed/extended 1 seed-extended
expect_nlink /mnt/p15-092-seed/hard-a 2 seed-hard-a
hard_a=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-a)
hard_b=$(pre15_guest_ssh_bounded stat -f '%i' /mnt/p15-092-seed/hard-b)
test "$hard_a" = "$hard_b" || pre15_dut_fail 'hardlink NIDs differ'
attach_mount reachable-compact-zero.erofs /mnt/p15-092-compact compact-zero
expect_nlink /mnt/p15-092-compact/compact 0 reachable-compact-zero
attach_mount reachable-extended-zero.erofs /mnt/p15-092-extended extended-zero
expect_nlink /mnt/p15-092-extended/extended 0 reachable-extended-zero
attach_mount reachable-directory-zero.erofs /mnt/p15-092-directory directory-zero
expect_nlink /mnt/p15-092-directory/subdir 0 reachable-directory-zero
attach_mount root-zero.erofs /mnt/p15-092-root root-zero
expect_nlink /mnt/p15-092-root 0 root-zero
attach_mount orphan-zero.erofs /mnt/p15-092-orphan orphan-zero
expect_nlink /mnt/p15-092-orphan/orphan 3 orphan-replacement
pre15_guest_ssh_bounded cat /mnt/p15-092-orphan/orphan >"$artifacts/orphan-replacement.txt"
pre15_guest_ssh_bounded dmesg >"$artifacts/dmesg.txt"
printf '%s\n' 'P15-092 baseline normal vnode runtime PASS'
''', encoding="ascii")
PY
chmod 0555 "$runtime_case"
qemu_stdout=$output/qemu.stdout
qemu_stderr=$output/qemu.stderr
qemu_evidence=$output/qemu-evidence
mkdir "$qemu_evidence"
if PRE15_DUT="$runtime_dut" PRE15_ROOT="$root" \
PRE15_SCHEMA="$dut/tests/pre15/EVIDENCE-SCHEMA.json" \
PRE15_EVIDENCE_ROOT="$qemu_evidence" PRE15_FREEBSD_SRC="$freebsd_src" \
PRE15_QEMU_BASE_IMAGE="$qemu_base" PRE15_QEMU_BASE_FORMAT=qcow2 \
PRE15_QEMU_SSH_KEY="$qemu_key" PRE15_QEMU_SSH_USER=root \
PRE15_QEMU_MEMORY_MB=2048 PRE15_QEMU_CPUS=2 PRE15_QEMU_BOOT_TIMEOUT=180 \
PRE15_QEMU_TIMEOUT=900 PRE15_GUEST_COMMAND_TIMEOUT=60 \
P15_092_GATE_DUT="$runtime_dut" P15_092_GATE_HOST="$output/host" \
timeout -k 30 1000 "$dut/tests/pre15/run-qemu.sh" P15-092-gate-runtime \
>"$qemu_stdout" 2>"$qemu_stderr"; then
qemu_rc=0
else
qemu_rc=$?
fi
printf '%s\n' "$qemu_rc" >"$output/qemu.exit"
python3 -B - "$output" "$qemu_rc" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import re
import sys
OUTPUT = Path(sys.argv[1])
QEMU_RC = int(sys.argv[2])
def sha256_path(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path: Path, value: object) -> None:
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="ascii")
host = json.loads((OUTPUT / "host/host-result.json").read_text(encoding="ascii"))
stdout = (OUTPUT / "qemu.stdout").read_text(encoding="utf-8")
match = re.search(r"PRE15_RESULT status=(\S+) cleanup=(\S+) run_id=(\S+) evidence=(\S+)", stdout)
if match is None:
runtime = {
"cleanup": "UNKNOWN",
"evidence": None,
"exit": QEMU_RC,
"run_id": None,
"status": "RUNNER_FAIL",
}
else:
runtime = {
"status": match.group(1),
"cleanup": match.group(2),
"run_id": match.group(3),
"evidence": match.group(4),
"exit": QEMU_RC,
}
status = "STOP"
reasons = list(host["reasons"])
if runtime["status"] != "PASS" or runtime["cleanup"] != "PASS":
reasons.append("normal_vnode_runtime_not_pass")
result = {
"b15": "STOP-NO-SOURCE",
"candidate": "P15-092",
"full_feature_suite": "NOT_RUN",
"gate": "G11",
"host_status": host["status"],
"qemu": runtime,
"reasons": reasons,
"schema": 1,
"status": status,
}
write_json(OUTPUT / "result.json", result)
hash_lines = []
for path in sorted(OUTPUT.rglob("*")):
if path.is_file() and path.name != "SHA256SUMS":
hash_lines.append(f"{sha256_path(path)} {path.relative_to(OUTPUT)}")
(OUTPUT / "SHA256SUMS").write_text("\n".join(hash_lines) + "\n", encoding="ascii")
print(json.dumps(result, sort_keys=True))
PY
cat "$output/result.json"
exit 22