From 770d83819bfbca9d76833041e27e48ee2b411d6a Mon Sep 17 00:00:00 2001 From: imrcpan Date: Tue, 18 Aug 2026 09:38:06 +0200 Subject: [PATCH] update --- .clang-format | 199 ++++++ .gitignore | 18 + Makefile | 41 ++ compress.h | 34 + data.c | 696 ++++++++++++++++++++ decompressor.c | 224 +++++++ decompressor_deflate.c | 185 ++++++ decompressor_lz4.c | 104 +++ decompressor_lzma.c | 181 ++++++ decompressor_zstd.c | 181 ++++++ dir.c | 441 +++++++++++++ erofs_fs.h | 496 ++++++++++++++ erofs_vnops.c | 570 +++++++++++++++++ inode.c | 428 +++++++++++++ internal.h | 423 ++++++++++++ namei.c | 436 +++++++++++++ super.c | 1387 ++++++++++++++++++++++++++++++++++++++++ xattr.c | 1272 ++++++++++++++++++++++++++++++++++++ xattr.h | 18 + zdata.c | 625 ++++++++++++++++++ zmap.c | 1108 ++++++++++++++++++++++++++++++++ 21 files changed, 9067 insertions(+) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 compress.h create mode 100644 data.c create mode 100644 decompressor.c create mode 100644 decompressor_deflate.c create mode 100644 decompressor_lz4.c create mode 100644 decompressor_lzma.c create mode 100644 decompressor_zstd.c create mode 100644 dir.c create mode 100644 erofs_fs.h create mode 100644 erofs_vnops.c create mode 100644 inode.c create mode 100644 internal.h create mode 100644 namei.c create mode 100644 super.c create mode 100644 xattr.c create mode 100644 xattr.h create mode 100644 zdata.c create mode 100644 zmap.c diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..3d436fa --- /dev/null +++ b/.clang-format @@ -0,0 +1,199 @@ +# Basic .clang-format +--- +BasedOnStyle: WebKit +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: AcrossEmptyLines +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: TopLevelDefinitions +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeTernaryOperators: false +# TODO: BreakStringLiterals can cause very strange formatting so turn it off? +BreakStringLiterals: false +# Prefer: +# some_var = function(arg1, +# arg2) +# over: +# some_var = +# function(arg1, arg2) +PenaltyBreakAssignment: 100 +# Prefer: +# some_long_function(arg1, arg2 +# arg3) +# over: +# some_long_function( +# arg1, arg2, arg3) +PenaltyBreakBeforeFirstCallParameter: 100 +CompactNamespaces: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: + - ARB_ARRFOREACH + - ARB_ARRFOREACH_REVWCOND + - ARB_ARRFOREACH_REVERSE + - ARB_FOREACH + - ARB_FOREACH_FROM + - ARB_FOREACH_SAFE + - ARB_FOREACH_REVERSE + - ARB_FOREACH_REVERSE_FROM + - ARB_FOREACH_REVERSE_SAFE + - BIT_FOREACH_ISCLR + - BIT_FOREACH_ISSET + - CPU_FOREACH + - CPU_FOREACH_ISCLR + - CPU_FOREACH_ISSET + - FOREACH_THREAD_IN_PROC + - FOREACH_PROC_IN_SYSTEM + - FOREACH_PRISON_CHILD + - FOREACH_PRISON_DESCENDANT + - FOREACH_PRISON_DESCENDANT_LOCKED + - FOREACH_PRISON_DESCENDANT_LOCKED_LEVEL + - MNT_VNODE_FOREACH_ALL + - MNT_VNODE_FOREACH_ACTIVE + - RB_FOREACH + - RB_FOREACH_FROM + - RB_FOREACH_SAFE + - RB_FOREACH_REVERSE + - RB_FOREACH_REVERSE_FROM + - RB_FOREACH_REVERSE_SAFE + - SLIST_FOREACH + - SLIST_FOREACH_FROM + - SLIST_FOREACH_FROM_SAFE + - SLIST_FOREACH_SAFE + - SLIST_FOREACH_PREVPTR + - SPLAY_FOREACH + - LIST_FOREACH + - LIST_FOREACH_FROM + - LIST_FOREACH_FROM_SAFE + - LIST_FOREACH_SAFE + - STAILQ_FOREACH + - STAILQ_FOREACH_FROM + - STAILQ_FOREACH_FROM_SAFE + - STAILQ_FOREACH_SAFE + - TAILQ_FOREACH + - TAILQ_FOREACH_FROM + - TAILQ_FOREACH_FROM_SAFE + - TAILQ_FOREACH_REVERSE + - TAILQ_FOREACH_REVERSE_FROM + - TAILQ_FOREACH_REVERSE_FROM_SAFE + - TAILQ_FOREACH_REVERSE_SAFE + - TAILQ_FOREACH_SAFE + - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH +SpaceBeforeParens: ControlStatementsExceptForEachMacros +IndentCaseLabels: false +IndentPPDirectives: None +Language: Cpp +NamespaceIndentation: None +PointerAlignment: Right +ContinuationIndentWidth: 4 +IndentWidth: 8 +TabWidth: 8 +ColumnLimit: 80 +UseTab: Always +SpaceAfterCStyleCast: false +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^\"opt_.*\.h\"' + Priority: 1 + SortPriority: 10 + - Regex: '^' + Priority: 2 + SortPriority: 20 + - Regex: '^' + Priority: 2 + SortPriority: 21 + - Regex: '^' + Priority: 2 + SortPriority: 22 + - Regex: '^' + Priority: 2 + SortPriority: 23 + - Regex: '^' + Priority: 3 + SortPriority: 30 + - Regex: '^ diff --git a/compress.h b/compress.h new file mode 100644 index 0000000..e423a03 --- /dev/null +++ b/compress.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __EROFS_FS_COMPRESS_H +#define __EROFS_FS_COMPRESS_H + +#include "internal.h" + +struct z_erofs_decompress_req { + struct erofs_sb_info *sbi; + const struct erofs_map_blocks *map; + const void *in; + size_t inputsize; + void *out; + size_t outputsize; + bool partial_decoding; +}; + +struct z_erofs_decompressor { + /* Callbacks return zero or a positive FreeBSD errno. */ + int (*config)(struct erofs_sb_info *, const struct erofs_super_block *, + const void *, size_t); + int (*decompress)(const struct z_erofs_decompress_req *); + bool supports_subextent; + const char *name; +}; + +bool z_erofs_decompress_supports_subextent( + const struct erofs_map_blocks *map); +int z_erofs_lz4_decompress(const struct z_erofs_decompress_req *rq); +extern const struct z_erofs_decompressor z_erofs_lzma_decomp; +extern const struct z_erofs_decompressor z_erofs_deflate_decomp; +extern const struct z_erofs_decompressor z_erofs_zstd_decomp; + +#endif /* __EROFS_FS_COMPRESS_H */ diff --git a/data.c b/data.c new file mode 100644 index 0000000..47945c1 --- /dev/null +++ b/data.c @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define EROFS_DIR_READAHEAD_BYTES (1024 * 1024) +#define EROFS_DIR_READAHEAD_SLOTS (EROFS_DIR_READAHEAD_BYTES / PAGE_SIZE) + +/* + * For flat inline files, compute the "inline tail" start offset. + * Linux EROFS semantics: the last logical block may be tailpacked + * into the inode metadata area. + */ +static erofs_off_t +erofs_inline_tail_start(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + if (vi->size == 0) + return (0); + return (roundup2(vi->size, (uint64_t)sbi->block_size) - sbi->block_size); +} + +static int +erofs_check_device_range(const struct erofs_sb_info *sbi, + const struct erofs_device_info *dif, erofs_off_t off, uint64_t len) +{ + erofs_off_t end, limit; + + if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(off, len, &end)) + return (EINTEGRITY); + limit = dif->blocks << sbi->blkszbits; + return (end > limit ? EINTEGRITY : 0); +} + +static void +erofs_fill_from_devinfo(struct erofs_map_dev *map, + struct erofs_device_info *dif, erofs_off_t pa) +{ + map->m_dif = dif; + map->m_pa = pa; +} + +static int +erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + struct erofs_device_info *dif; + uint64_t start; + unsigned int id; + int error; + + erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa); + if (map->m_deviceid != 0) { + if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL) + return (ENODEV); + dif = &sbi->devs[map->m_deviceid - 1]; + error = erofs_check_device_range(sbi, dif, map->m_pa, + map->m_plen); + if (error != 0) + return (error); + if (sbi->flatdev) { + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (__builtin_add_overflow(map->m_pa, start, &map->m_pa)) + return (EINTEGRITY); + return (0); + } + if (dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa); + return (0); + } + + if (sbi->extra_devices == 0) + return (0); + if (sbi->flatdev) { + error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa, + map->m_plen); + if (error == 0) + return (0); + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0 || + dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + continue; + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa < start) + continue; + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error == 0) + return (0); + if (map->m_pa - start < + (dif->blocks << sbi->blkszbits)) + return (error); + } + return (EINTEGRITY); + } + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0) + continue; + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa >= start && + map->m_pa - start < (dif->blocks << sbi->blkszbits)) { + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error != 0) + return (error); + if (dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa - start); + break; + } + } + return (0); +} + +/* Map chunk-based file to physical extent */ +static int +erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_inode_chunk_index *idx; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t chunk_idx, chunk_size, entry_size; + erofs_off_t loff, idx_off, chunk_off, idx_base; + uint64_t image_size, addrmask; + uint64_t blkaddr; + uint16_t raw_device_id; + int error; + + loff = map->m_la; + chunk_size = 1ULL << vi->chunkbits; + chunk_idx = loff >> vi->chunkbits; + chunk_off = loff & (chunk_size - 1); + + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) + entry_size = sizeof(struct erofs_inode_chunk_index); + else + entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE; + if (vi->inode_off > UINT64_MAX - vi->inode_isize || + vi->inode_off + vi->inode_isize > UINT64_MAX - vi->xattr_isize) + return (EOVERFLOW); + idx_base = vi->inode_off + vi->inode_isize + vi->xattr_isize; + if (idx_base > UINT64_MAX - (entry_size - 1)) + return (EOVERFLOW); + idx_base = roundup2(idx_base, entry_size); + if (chunk_idx > (UINT64_MAX - idx_base) / entry_size) + return (EOVERFLOW); + idx_off = idx_base + chunk_idx * entry_size; + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || idx_off > sbi->metabox_en->size || + entry_size > sbi->metabox_en->size - idx_off) + return (EINTEGRITY); + } else { + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + image_size = sbi->blocks << sbi->blkszbits; + if (idx_off > image_size || entry_size > image_size - idx_off) + return (EINTEGRITY); + } + + error = erofs_read_metadata(sbi, vi->nid, idx_off, entry_size, &buf); + if (error != 0) + return (error); + + idx = buf.data; + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) { + blkaddr = le32toh(idx->startblk_lo); + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0) + blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32; + raw_device_id = le16toh(idx->device_id); + addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ? + ((1ULL << 48) - 1) : UINT32_MAX; + } else { + blkaddr = le32dec(idx); + raw_device_id = 0; + addrmask = UINT32_MAX; + } + erofs_put_metabuf(&buf); + + if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) { + map->m_pa = 0; + map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff); + map->m_plen = map->m_llen; + return (0); + } + + map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff); + map->m_plen = map->m_llen; + map->m_deviceid = raw_device_id & sbi->device_id_mask; + if (blkaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + map->m_pa = blkaddr << sbi->blkszbits; + if (chunk_off > UINT64_MAX - map->m_pa) + return (EOVERFLOW); + map->m_pa += chunk_off; + map->m_flags |= EROFS_MAP_MAPPED; + return (0); +} + +static int +erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno, + int *rabsize, int racnt, void **bufp) +{ + struct buf *bp; + erofs_off_t end, limit; + off_t blkoff, current; + size_t blklen, done, iosize; + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (dif == NULL || dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + if (__builtin_add_overflow(off, (uint64_t)len, &end)) + return (EINTEGRITY); + if (blocks != 0) { + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + limit = blocks << sbi->blkszbits; + if (end > limit) + return (EINTEGRITY); + } + if (end > dif->mediasize) + return (ENXIO); + if (off > INT64_MAX || end > (uint64_t)INT64_MAX + 1) + return (EOVERFLOW); + + iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize; + if (iosize == 0 || (iosize & (iosize - 1)) != 0) + return (EINVAL); + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + current = (off_t)(off + done); + blkoff = rounddown2(current, (off_t)iosize); + blklen = MIN(iosize - (size_t)(current - blkoff), len - done); + if (done == 0 && racnt != 0) + error = breadn(dif->devvp, btodb(blkoff), iosize, + rablkno, rabsize, racnt, NOCRED, &bp); + else + error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, &bp); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (bp->b_data == NULL) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + memcpy(out + done, (char *)bp->b_data + (current - blkoff), + blklen); + brelse(bp); + done += blklen; + } + *bufp = out; + return (0); +} + +int +erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp) +{ + return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len, + NULL, NULL, 0, bufp)); +} + +int +erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp) +{ + struct erofs_map_dev map; + erofs_blk_t blocks; + int error; + + map = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, &map); + if (error != 0) + return (error); + blocks = map.m_dif->blocks; + if (map.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len, + NULL, NULL, 0, bufp)); +} + +static int +erofs_read_physical_readahead(struct erofs_sb_info *sbi, + unsigned int device_id, erofs_off_t off, size_t len, + unsigned int rablocks, void **bufp) +{ + struct erofs_map_dev current, future; + daddr_t rablkno[EROFS_DIR_READAHEAD_SLOTS]; + int rabsize[EROFS_DIR_READAHEAD_SLOTS]; + erofs_off_t step; + erofs_blk_t blocks; + unsigned int count; + int error; + + current = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, ¤t); + if (error != 0) + return (error); + blocks = current.m_dif->blocks; + if (current.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + rablocks = MIN(rablocks, (unsigned int)nitems(rablkno)); + for (count = 0; count < rablocks; count++) { + step = (erofs_off_t)(count + 1) * sbi->block_size; + if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step) + break; + future = (struct erofs_map_dev) { + .m_pa = off + step, + .m_deviceid = device_id, + .m_plen = sbi->block_size, + }; + if (erofs_map_dev(sbi, &future) != 0 || + future.m_dif != current.m_dif || + future.m_pa != current.m_pa + step) + break; + rablkno[count] = btodb(future.m_pa); + rabsize[count] = sbi->block_size; + } + return (erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len, + rablkno, rabsize, count, bufp)); +} + +/* Release a contiguous buffer returned by erofs_bread(). */ +void +erofs_brelse(void *buf) +{ + free(buf, M_EROFS); +} + +void +erofs_put_metabuf(struct erofs_buf *buf) +{ + void (*release)(void *); + void *data; + + if (buf == NULL) + return; + data = buf->data; + release = buf->release; + buf->data = NULL; + buf->release = NULL; + if (data != NULL && release != NULL) + release(data); +} + +/* Read inode metadata from either the primary image or the metabox file. */ +int +erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, erofs_off_t off, + size_t len, struct erofs_buf *buf) +{ + void *data; + int error; + + if (!erofs_nid_in_metabox(nid)) { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } else { + if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + erofs_off_t loff, tail_start; + uint64_t remain, block_rem; + + loff = map->m_la; + if (loff >= vi->size) + return (0); + + remain = vi->size - loff; + switch (vi->datalayout) { + case EROFS_INODE_CHUNK_BASED: + return (erofs_map_blocks_chunk(sbi, vi, map)); + case EROFS_INODE_FLAT_PLAIN: + map->m_llen = remain; + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff, + &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + case EROFS_INODE_FLAT_INLINE: + tail_start = erofs_inline_tail_start(sbi, vi); + if (loff < tail_start) { + map->m_llen = MIN(remain, tail_start - loff); + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, + loff, &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + } + block_rem = sbi->block_size - + ((loff - tail_start) & (sbi->block_size - 1)); + map->m_llen = MIN(remain, block_rem); + map->m_plen = map->m_llen; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &map->m_pa) || __builtin_add_overflow(map->m_pa, vi->xattr_isize, + &map->m_pa) || __builtin_add_overflow(map->m_pa, loff - tail_start, + &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META; + return (0); + case EROFS_INODE_COMPRESSED_FULL: + case EROFS_INODE_COMPRESSED_COMPACT: + return (EOPNOTSUPP); + default: + return (EOPNOTSUPP); + } +} + +int +erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_map_blocks next = { .m_la = map->m_la }; + int error; + + if (erofs_inode_is_data_compressed(vi->datalayout)) { + error = z_erofs_map_blocks(sbi, vi, &next); + } else { + error = erofs_map_blocks_flatmode(sbi, vi, &next); + } + *map = next; + return (error); +} +/* + * Read a small range at a logical file offset into a contiguous buffer. + * Primarily used for directory block reads, lookup, and symlink fragment + * parsing. + */ +static int +erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char *out; + void *blk; + size_t done, want; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (loff > UINT64_MAX - (uint64_t)len) + return (EOVERFLOW); + if (loff > vi->size || (uint64_t)len > vi->size - loff) + return (EINTEGRITY); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) + return (z_erofs_read_data(sbi, vi, loff, len, bufp)); + + /* Uncompressed file path */ + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (map.m_llen == 0) { + free(out, M_EROFS); + return (EINTEGRITY); + } + want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS), + len - done); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map.m_pa, + want, &buf); + } else if (done == 0 && rablocks != 0 && + map.m_flags == EROFS_MAP_MAPPED) { + error = erofs_read_physical_readahead(sbi, + map.m_deviceid, map.m_pa, want, rablocks, &blk); + } else { + error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa, + want, &blk); + } + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + memcpy(out + done, buf.data, want); + erofs_put_metabuf(&buf); + } else { + memcpy(out + done, blk, want); + erofs_brelse(blk); + } + } + done += want; + } + *bufp = out; + return (0); +} + +int +erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp)); +} + +int +erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, bool sequential, void **bufp) +{ + uint64_t remaining; + unsigned int rablocks; + + rablocks = 0; + if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN && + sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 && + len <= sbi->block_size && loff <= vi->size && len <= vi->size - loff) { + remaining = vi->size - loff - len; + rablocks = MIN(howmany(remaining, sbi->block_size), + (uint64_t)EROFS_DIR_READAHEAD_SLOTS); + } + return (erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp)); +} + +/* + * Transfer the logical content of an inode directly into a uio. + * Regular files and symlinks both use this read path. + */ +static int +erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, struct uio *uio) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char zerobuf[PAGE_SIZE]; + void *blk; + size_t want, chunk; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + if ((uint64_t)uio->uio_offset >= vi->size) + return (0); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) + return (z_erofs_read_uio(sbi, vi, uio)); + + /* Uncompressed file path */ + bzero(zerobuf, sizeof(zerobuf)); + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + map = (struct erofs_map_blocks) { .m_la = uio->uio_offset }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0) + break; + want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS), + (size_t)uio->uio_resid); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + chunk = want; + while (chunk > 0) { + size_t zlen = MIN(chunk, sizeof(zerobuf)); + + error = uiomove(zerobuf, zlen, uio); + if (error != 0) + return (error); + chunk -= zlen; + } + continue; + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map.m_pa, want, + &buf); + } else { + error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa, + want, &blk); + } + if (error != 0) + return (error); + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = uiomove(buf.data, want, uio); + erofs_put_metabuf(&buf); + } else { + error = uiomove(blk, want, uio); + erofs_brelse(blk); + } + if (error != 0) + return (error); + } + return (0); +} + +/* Read regular file data. */ +int +erofs_read_file(struct vnode *vp, struct uio *uio, int ioflag) +{ + (void)ioflag; + return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio)); +} + +int +erofs_validate_symlink_target(struct erofs_sb_info *sbi, + struct erofs_inode *vi) +{ + void *target; + int error; + + if (vi->vtype != VLNK) + return (EINVAL); + if (vi->size == 0) + return (EINTEGRITY); + if (vi->size > MAXPATHLEN) + return (ENAMETOOLONG); + error = erofs_read_data(sbi, vi, 0, (size_t)vi->size, &target); + if (error != 0) + return (error); + if (memchr(target, '\0', (size_t)vi->size) != NULL) + error = EINTEGRITY; + erofs_brelse(target); + return (error); +} + +/* Read symlink target string. */ +int +erofs_readlink_target(struct vnode *vp, struct uio *uio) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + if (vi->size == 0) + return (EINTEGRITY); + if (vi->size > MAXPATHLEN) + return (ENAMETOOLONG); + return (erofs_read_uio(MTOE(vp->v_mount), vi, uio)); +} diff --git a/decompressor.c b/decompressor.c new file mode 100644 index 0000000..3988969 --- /dev/null +++ b/decompressor.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ + +#include +#include +#include +#include +#include + +#include "compress.h" + +static int +z_erofs_load_lz4_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lz4_cfgs *lz4; + uint32_t max_pclusterblks; + uint16_t distance; + + if (data != NULL) { + if (size < sizeof(*lz4)) + return (EINTEGRITY); + lz4 = data; + max_pclusterblks = le16toh(lz4->max_pclusterblks); + if (max_pclusterblks == 0) + max_pclusterblks = 1; + else if (max_pclusterblks > + (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits)) + return (EOPNOTSUPP); + } else { + distance = le16toh(dsb->u1.lz4_max_distance); + if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi)) + return (0); + sbi->available_compr_algs = 1U << Z_EROFS_COMPRESSION_LZ4; + } + return (0); +} + +static int +z_erofs_transform_plain(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *src; + uint8_t *dst; + size_t first, offset; + + if (rq->outputsize > rq->inputsize) + return (EINTEGRITY); + src = rq->in; + dst = rq->out; + if (rq->map->m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) { + memmove(dst, src, rq->outputsize); + return (0); + } + first = MIN((size_t)(rq->sbi->block_size - + (rq->map->m_la & (rq->sbi->block_size - 1))), rq->outputsize); + offset = (rq->inputsize - first) & (rq->sbi->block_size - 1); + if (offset > rq->inputsize || first > rq->inputsize - offset) + return (EINTEGRITY); + memmove(dst, src + offset, first); + if (first < rq->outputsize) + memmove(dst + first, src, rq->outputsize - first); + return (0); +} + +static const struct z_erofs_decompressor z_erofs_shifted_decomp = { + .decompress = z_erofs_transform_plain, + .name = "shifted", +}; + +static const struct z_erofs_decompressor z_erofs_interlaced_decomp = { + .decompress = z_erofs_transform_plain, + .name = "interlaced", +}; + +static const struct z_erofs_decompressor z_erofs_lz4_decomp = { + .config = z_erofs_load_lz4_config, + .decompress = z_erofs_lz4_decompress, + .supports_subextent = 1, + .name = "lz4", +}; + +static const struct z_erofs_decompressor * const z_erofs_decomp[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp, + [Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp, + [Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp, + [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, + [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, + [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, +}; + +bool +z_erofs_decompress_supports_subextent(const struct erofs_map_blocks *map) +{ + uint8_t algorithm; + + algorithm = map->m_algorithmformat; + return (algorithm < nitems(z_erofs_decomp) && + z_erofs_decomp[algorithm] != NULL && + z_erofs_decomp[algorithm]->supports_subextent); +} + +static int +z_erofs_read_cfg(struct erofs_sb_info *sbi, uint64_t *offset, + struct erofs_buf *buf, + size_t *sizep) +{ + struct erofs_buf metabuf = EROFS_BUF_INITIALIZER; + uint8_t length_buf[2]; + uint64_t aligned; + uint16_t length; + int error; + + aligned = roundup2(*offset, 4); + if (aligned > UINT64_MAX - sizeof(length_buf)) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, 0, aligned, sizeof(length_buf), &metabuf); + if (error != 0) + return (error); + memcpy(length_buf, metabuf.data, sizeof(length_buf)); + erofs_put_metabuf(&metabuf); + length = le16dec(length_buf); + *sizep = length != 0 ? length : UINT16_MAX + 1U; + if (*sizep > 65536 || aligned + sizeof(length_buf) > + UINT64_MAX - *sizep) + return (EINTEGRITY); + *offset = aligned + sizeof(length_buf); + error = erofs_read_metadata(sbi, 0, *offset, *sizep, buf); + if (error == 0) + *offset += *sizep; + return (error); +} + +int +z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf data = EROFS_BUF_INITIALIZER; + const struct z_erofs_decompressor *decompressor; + uint64_t offset; + uint16_t algorithms; + size_t size; + int algorithm, error; + + if (!erofs_sb_has_compr_cfgs(sbi)) + return (z_erofs_load_lz4_config(sbi, dsb, NULL, 0)); + algorithms = le16toh(dsb->u1.available_compr_algs); + sbi->available_compr_algs = algorithms; + if ((algorithms & ~Z_EROFS_ALL_COMPR_ALGS) != 0) + return (EOPNOTSUPP); + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; + ++algorithm) { + if ((algorithms & (1U << algorithm)) == 0) + continue; + error = z_erofs_read_cfg(sbi, &offset, &data, &size); + if (error != 0) + return (error); + decompressor = z_erofs_decomp[algorithm]; + if (decompressor == NULL || decompressor->config == NULL) + error = EOPNOTSUPP; + else + error = decompressor->config(sbi, dsb, data.data, size); + erofs_put_metabuf(&data); + if (error != 0) + return (error); + } + return (0); +} + +int +z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src0, size_t srclen, + void *dst, size_t dstlen, bool partial) +{ + const struct z_erofs_decompressor *decompressor; + struct z_erofs_decompress_req rq; + const uint8_t *src; + uint8_t algorithm; + size_t padding, padding_limit; + + algorithm = map->m_algorithmformat; + if (algorithm >= nitems(z_erofs_decomp) || + z_erofs_decomp[algorithm] == NULL || + z_erofs_decomp[algorithm]->decompress == NULL) + return (EOPNOTSUPP); + decompressor = z_erofs_decomp[algorithm]; + rq = (struct z_erofs_decompress_req) { + .sbi = sbi, + .map = map, + .in = src0, + .inputsize = srclen, + .out = dst, + .outputsize = dstlen, + .partial_decoding = partial, + }; + if (algorithm == Z_EROFS_COMPRESSION_SHIFTED || + algorithm == Z_EROFS_COMPRESSION_INTERLACED) + return (decompressor->decompress(&rq)); + + src = src0; + if (map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4 || + erofs_sb_has_lz4_0padding(sbi)) { + padding_limit = MIN(srclen, sbi->block_size - + (map->m_pa & (sbi->block_size - 1))); + for (padding = 0; padding < padding_limit && src[padding] == 0; + ++padding) + ; + if (padding == padding_limit) + return (EINTEGRITY); + src += padding; + srclen -= padding; + } + if (algorithm == Z_EROFS_COMPRESSION_LZMA) { + if (sbi->lzma_dict_size == 0) + return (EINTEGRITY); + } + rq.in = src; + rq.inputsize = srclen; + return (decompressor->decompress(&rq)); +} diff --git a/decompressor_deflate.c b/decompressor_deflate.c new file mode 100644 index 0000000..8a4bd05 --- /dev/null +++ b/decompressor_deflate.c @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal DEFLATE decompressor for EROFS FreeBSD */ +#include +#include +#include +#include + +#include "compress.h" + +struct z_erofs_deflate_ctx { + struct erofs_stream_ctx pool; + z_stream stream; + bool initialized; +}; + +_Static_assert(sizeof(struct z_erofs_deflate_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Deflate stream wrapper exceeds UMA item"); + +static voidpf +z_erofs_deflate_alloc(voidpf opaque, uInt items, uInt size) +{ + struct erofs_stream_ctx *pool; + size_t bytes; + + pool = opaque; + if (__builtin_mul_overflow((size_t)items, (size_t)size, &bytes)) { + pool->allocation_failed = true; + return (NULL); + } + return (z_erofs_stream_ctx_alloc(pool, bytes)); +} + +static void +z_erofs_deflate_free(voidpf opaque, voidpf address) +{ + + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_load_deflate_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_deflate_cfgs *deflate; + + (void)dsb; + if (size < sizeof(*deflate)) + return (EINTEGRITY); + deflate = data; + if (deflate->windowbits < 8 || deflate->windowbits > 15) + return (EOPNOTSUPP); + sbi->deflate_windowbits = deflate->windowbits; + return (0); +} + +static int +z_erofs_deflate_error(int ret) +{ + + switch (ret) { + case Z_MEM_ERROR: + return (ENOMEM); + case Z_VERSION_ERROR: + return (EOPNOTSUPP); + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_BUF_ERROR: + return (EINTEGRITY); + default: + return (EIO); + } +} + +static int +z_erofs_deflate_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + int ret; + + ctx = (struct z_erofs_deflate_ctx *)pool; + bzero(&ctx->stream, sizeof(ctx->stream)); + ctx->stream.zalloc = z_erofs_deflate_alloc; + ctx->stream.zfree = z_erofs_deflate_free; + ctx->stream.opaque = pool; + ret = inflateInit2(&ctx->stream, -pool->sbi->deflate_windowbits); + if (ret != Z_OK) + return (z_erofs_deflate_error(ret)); + ctx->initialized = true; + return (0); +} + +static void +z_erofs_deflate_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + + ctx = (struct z_erofs_deflate_ctx *)pool; + if (ctx->initialized) { + (void)inflateEnd(&ctx->stream); + ctx->initialized = false; + } +} + +static int +z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret, + uInt avail_in) +{ + + if (rq->partial_decoding) + return (0); + if (ret != Z_STREAM_END || avail_in != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_deflate_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_deflate_ctx *ctx; + z_stream *strm; + uInt in_before, out_before; + int error, ret; + + if (rq->sbi->deflate_windowbits < 8 || + rq->sbi->deflate_windowbits > MAX_WBITS) + return (EOPNOTSUPP); + if (rq->inputsize > (size_t)(uInt)-1 || + rq->outputsize > (size_t)(uInt)-1) + return (EOVERFLOW); + if (rq->outputsize == 0) + return (EINTEGRITY); + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_DEFLATE, + sizeof(*ctx), z_erofs_deflate_ctx_init, z_erofs_deflate_ctx_fini, + &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_deflate_ctx *)pool; + strm = &ctx->stream; + pool->allocation_failed = false; + ret = inflateReset2(strm, -rq->sbi->deflate_windowbits); + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + z_erofs_stream_ctx_put(pool, false); + return (error); + } + strm->next_in = __DECONST(void *, rq->in); + strm->avail_in = rq->inputsize; + strm->next_out = rq->out; + strm->avail_out = rq->outputsize; + + error = 0; + ret = Z_OK; + while (strm->avail_out != 0) { + in_before = strm->avail_in; + out_before = strm->avail_out; + ret = inflate(strm, Z_SYNC_FLUSH); + if (ret == Z_STREAM_END) + break; + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + break; + } + if (strm->avail_in == in_before && strm->avail_out == out_before) { + error = EINTEGRITY; + break; + } + } + if (error == 0 && strm->avail_out != 0) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_deflate_finish(rq, ret, strm->avail_in); + if (pool->allocation_failed) + error = ENOMEM; + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, + .supports_subextent = 1, + .name = "deflate", +}; diff --git a/decompressor_lz4.c b/decompressor_lz4.c new file mode 100644 index 0000000..39bd897 --- /dev/null +++ b/decompressor_lz4.c @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal LZ4 decompressor for EROFS FreeBSD */ +#include +#include +#include + +#include "compress.h" + +#define EROFS_LZ4_TOKEN_LITERAL_SHIFT 4 +#define EROFS_LZ4_TOKEN_MATCH_MASK 0x0f +#define EROFS_LZ4_MAX_RUN 15 +#define EROFS_LZ4_EXT_SENTINEL 255 +#define EROFS_LZ4_MIN_MATCH 4 +#define EROFS_LZ4_OFFSET_BYTES 2 + +static int +lz4_finish(const uint8_t *ip, const uint8_t *iend, int partial) +{ + if (partial) + return (0); + while (ip < iend) { + if (*ip++ != 0) + return (EINTEGRITY); + } + return (0); +} + +int +z_erofs_lz4_decompress(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *ip, *iend; + uint8_t *op, *oend; + unsigned int token; + size_t length, copylen; + size_t offset; + + ip = rq->in; + iend = ip + rq->inputsize; + op = rq->out; + oend = op + rq->outputsize; + if (iend < ip || oend < op) + return (EINTEGRITY); + + while (ip < iend) { + token = *ip++; + length = token >> EROFS_LZ4_TOKEN_LITERAL_SHIFT; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (ip >= iend) + return (EINTEGRITY); + value = *ip++; + if (length > SIZE_MAX - value) + return (EINTEGRITY); + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > (size_t)(iend - ip)) + return (EINTEGRITY); + if (!rq->partial_decoding && length > (size_t)(oend - op)) + return (EINTEGRITY); + copylen = MIN(length, (size_t)(oend - op)); + memcpy(op, ip, copylen); + ip += length; + op += copylen; + if (op == oend) + return (lz4_finish(ip, iend, rq->partial_decoding)); + if (ip >= iend) + break; + if (ip + EROFS_LZ4_OFFSET_BYTES > iend) + return (EINTEGRITY); + offset = le16dec(ip); + ip += EROFS_LZ4_OFFSET_BYTES; + if (offset == 0 || offset > + (size_t)(op - (uint8_t *)rq->out)) + return (EINTEGRITY); + length = token & EROFS_LZ4_TOKEN_MATCH_MASK; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (ip >= iend) + return (EINTEGRITY); + value = *ip++; + if (length > SIZE_MAX - value) + return (EINTEGRITY); + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > SIZE_MAX - EROFS_LZ4_MIN_MATCH) + return (EINTEGRITY); + length += EROFS_LZ4_MIN_MATCH; + if (!rq->partial_decoding && length > (size_t)(oend - op)) + return (EINTEGRITY); + copylen = MIN(length, (size_t)(oend - op)); + while (copylen-- != 0) { + *op = *(op - offset); + ++op; + } + if (op == oend) + return (lz4_finish(ip, iend, rq->partial_decoding)); + } + return (op == oend ? + lz4_finish(ip, iend, rq->partial_decoding) : EINTEGRITY); +} diff --git a/decompressor_lzma.c b/decompressor_lzma.c new file mode 100644 index 0000000..2ce258a --- /dev/null +++ b/decompressor_lzma.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * EROFS MicroLZMA wrapper around FreeBSD's bundled XZ Embedded decoder. + * The decoder source is compiled with private symbol names because the + * stock xz.ko does not enable its optional MicroLZMA entry points. + */ + +#include +#include +#include + +#include "compress.h" + +#define XZ_DEC_MICROLZMA +#define xz_dec_lzma2_create erofs_xz_dec_lzma2_create +#define xz_dec_lzma2_reset erofs_xz_dec_lzma2_reset +#define xz_dec_lzma2_run erofs_xz_dec_lzma2_run +#define xz_dec_lzma2_end erofs_xz_dec_lzma2_end +#define xz_dec_microlzma_alloc erofs_xz_dec_microlzma_alloc +#define xz_dec_microlzma_reset erofs_xz_dec_microlzma_reset +#define xz_dec_microlzma_run erofs_xz_dec_microlzma_run +#define xz_dec_microlzma_end erofs_xz_dec_microlzma_end +#define xz_malloc erofs_xz_malloc +#define xz_free erofs_xz_free + +static void * +erofs_xz_malloc(unsigned long size) +{ + return (malloc(size, M_EROFS, M_NOWAIT)); +} + +static void +erofs_xz_free(void *ptr) +{ + free(ptr, M_EROFS); +} + +#include + +#undef bool +#undef false +#undef true +#undef min + +struct z_erofs_lzma_ctx { + struct erofs_stream_ctx pool; + struct xz_dec_microlzma *state; + size_t state_bytes; +}; + +_Static_assert(sizeof(struct z_erofs_lzma_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "LZMA stream wrapper exceeds UMA item"); + +static int +z_erofs_lzma_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + int error; + + ctx = (struct z_erofs_lzma_ctx *)pool; + ctx->state_bytes = sizeof(*ctx->state); + error = z_erofs_stream_ctx_charge(pool, ctx->state_bytes); + if (error != 0) + return (error); + ctx->state = xz_dec_microlzma_alloc(XZ_SINGLE, + pool->sbi->lzma_dict_size); + if (ctx->state == NULL) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + pool->allocation_failed = 1; + return (ENOMEM); + } + return (0); +} + +static void +z_erofs_lzma_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + + ctx = (struct z_erofs_lzma_ctx *)pool; + if (ctx->state != NULL) { + xz_dec_microlzma_end(ctx->state); + ctx->state = NULL; + } + if (ctx->state_bytes != 0) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + } +} + +static int +z_erofs_load_lzma_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lzma_cfgs *lzma; + uint32_t dict_size; + + (void)dsb; + if (size < sizeof(*lzma)) + return (EINTEGRITY); + lzma = data; + if (le16toh(lzma->format) != 0) + return (EOPNOTSUPP); + dict_size = le32toh(lzma->dict_size); + if (dict_size < 4096) + return (EINTEGRITY); + if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) + return (EOPNOTSUPP); + sbi->lzma_dict_size = dict_size; + return (0); +} + +static int +z_erofs_lzma_error(enum xz_ret ret) +{ + + switch (ret) { + case XZ_MEM_ERROR: + return (ENOMEM); + case XZ_MEMLIMIT_ERROR: + case XZ_OPTIONS_ERROR: + case XZ_UNSUPPORTED_CHECK: + return (EOPNOTSUPP); + default: + return (EINTEGRITY); + } +} + +static int +z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret, + size_t input_pos) +{ + + if (rq->partial_decoding && + (ret == XZ_OK || ret == XZ_STREAM_END)) + return (0); + if (!rq->partial_decoding && ret == XZ_STREAM_END && + input_pos == rq->inputsize) + return (0); + return (z_erofs_lzma_error(ret)); +} + +static int +z_erofs_lzma_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_lzma_ctx *ctx; + struct xz_buf buffer; + enum xz_ret ret; + int error; + + if (rq->inputsize > UINT32_MAX || rq->outputsize > UINT32_MAX) + return (EOVERFLOW); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_LZMA, + sizeof(*ctx), z_erofs_lzma_ctx_init, z_erofs_lzma_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_lzma_ctx *)pool; + bzero(&buffer, sizeof(buffer)); + buffer.in = rq->in; + buffer.in_size = rq->inputsize; + buffer.out = rq->out; + buffer.out_size = rq->outputsize; + xz_dec_microlzma_reset(ctx->state, (uint32_t)rq->inputsize, + (uint32_t)rq->outputsize, !rq->partial_decoding); + ret = xz_dec_microlzma_run(ctx->state, &buffer); + if (buffer.out_pos != rq->outputsize) + error = EINTEGRITY; + else + error = z_erofs_lzma_finish(rq, ret, buffer.in_pos); + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_lzma_decomp = { + .config = z_erofs_load_lzma_config, + .decompress = z_erofs_lzma_decompress, + .supports_subextent = 1, + .name = "lzma", +}; diff --git a/decompressor_zstd.c b/decompressor_zstd.c new file mode 100644 index 0000000..ee6f7ac --- /dev/null +++ b/decompressor_zstd.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal zstd decompressor for EROFS FreeBSD */ +#include +#include +#include +#include + +#include "compress.h" + +#ifdef ZSTDIO +#define ZSTD_STATIC_LINKING_ONLY +#include + +struct z_erofs_zstd_ctx { + struct erofs_stream_ctx pool; + ZSTD_DCtx *dctx; +}; + +_Static_assert(sizeof(struct z_erofs_zstd_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Zstd stream wrapper exceeds UMA item"); +#endif + +static bool +erofs_zstd_available(void) +{ +#ifdef ZSTDIO + return (true); +#else + return (false); +#endif +} + +static int +z_erofs_load_zstd_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_zstd_cfgs *zstd; + + (void)dsb; + if (!erofs_zstd_available()) { + vfs_mount_error(sbi->mnt, + "erofs: ZSTD compression requires ZSTDIO support"); + return (EOPNOTSUPP); + } + if (size < sizeof(*zstd)) + return (EINTEGRITY); + zstd = data; + if (zstd->format != 0 || zstd->windowlog > 10) + return (EOPNOTSUPP); + sbi->zstd_windowlog = zstd->windowlog; + return (0); +} + +#ifdef ZSTDIO +static void * +zstd_alloc(void *opaque, size_t size) +{ + return (z_erofs_stream_ctx_alloc(opaque, size)); +} + +static void +zstd_free(void *opaque, void *address) +{ + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_zstd_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + ZSTD_customMem alloc; + size_t ret; + + ctx = (struct z_erofs_zstd_ctx *)pool; + alloc = (ZSTD_customMem) { + .customAlloc = zstd_alloc, + .customFree = zstd_free, + .opaque = pool, + }; + ctx->dctx = ZSTD_createDCtx_advanced(alloc); + if (ctx->dctx == NULL) + return (ENOMEM); + ret = ZSTD_DCtx_setParameter(ctx->dctx, ZSTD_d_windowLogMax, + pool->sbi->zstd_windowlog + 10); + if (ZSTD_isError(ret)) + return (pool->allocation_failed ? ENOMEM : EOPNOTSUPP); + return (0); +} + +static void +z_erofs_zstd_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + + ctx = (struct z_erofs_zstd_ctx *)pool; + if (ctx->dctx != NULL) { + (void)ZSTD_freeDCtx(ctx->dctx); + ctx->dctx = NULL; + } +} + +static int +z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret, + size_t input_pos, size_t input_size) +{ + + if (rq->partial_decoding) + return (0); + if (ret != 0 || input_pos != input_size) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_zstd_ctx *ctx; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t in_before, out_before, ret; + int error; + + if (rq->sbi->zstd_windowlog + 10 > 20) + return (EOPNOTSUPP); + if (rq->outputsize == 0) + return (EINTEGRITY); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_ZSTD, + sizeof(*ctx), z_erofs_zstd_ctx_init, z_erofs_zstd_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_zstd_ctx *)pool; + pool->allocation_failed = false; + + input = (ZSTD_inBuffer) { + .src = rq->in, + .size = rq->inputsize, + }; + output = (ZSTD_outBuffer) { + .dst = rq->out, + .size = rq->outputsize, + }; + error = 0; + ret = 1; + while (output.pos != output.size) { + in_before = input.pos; + out_before = output.pos; + ret = ZSTD_decompressStream(ctx->dctx, &output, &input); + if (ZSTD_isError(ret)) { + error = pool->allocation_failed ? ENOMEM : EINTEGRITY; + break; + } + if (input.pos == in_before && output.pos == out_before) { + error = EINTEGRITY; + break; + } + if (ret == 0) + break; + } + if (error == 0 && output.pos != output.size) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_zstd_finish(rq, ret, input.pos, input.size); + z_erofs_stream_ctx_put(pool, error == 0 && !rq->partial_decoding); + return (error); +} +#else +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + (void)rq; + return (EOPNOTSUPP); +} +#endif + +const struct z_erofs_decompressor z_erofs_zstd_decomp = { + .config = z_erofs_load_zstd_config, + .decompress = z_erofs_zstd_decompress, + .supports_subextent = 0, + .name = "zstd", +}; diff --git a/dir.c b/dir.c new file mode 100644 index 0000000..21d9352 --- /dev/null +++ b/dir.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* Map EROFS directory entry file type to FreeBSD dirent.d_type. */ +static unsigned char +erofs_ftype_to_dtype(uint8_t ftype) +{ + switch (ftype) { + case EROFS_FT_REG_FILE: + return (DT_REG); + case EROFS_FT_DIR: + return (DT_DIR); + case EROFS_FT_CHRDEV: + return (DT_CHR); + case EROFS_FT_BLKDEV: + return (DT_BLK); + case EROFS_FT_FIFO: + return (DT_FIFO); + case EROFS_FT_SOCK: + return (DT_SOCK); + case EROFS_FT_SYMLINK: + return (DT_LNK); + default: + return (DT_UNKNOWN); + } +} + +static int +erofs_dirname_order(const char *left, size_t leftlen, const char *right, + size_t rightlen) +{ + size_t common; + int order; + + common = MIN(leftlen, rightlen); + order = memcmp(left, right, common); + if (order != 0) + return (order); + if (leftlen == rightlen) + return (0); + return (leftlen < rightlen ? -1 : 1); +} + +/* Validate one name slot and return its Linux-visible length. */ +int +erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp) +{ + size_t namelen, span; + + if (endoff <= nameoff) + return (EINTEGRITY); + span = endoff - nameoff; + if (trailing) { + namelen = strnlen(blk + nameoff, span); + } else { + namelen = span; + if (memchr(blk + nameoff, '\0', span) != NULL) + return (EINTEGRITY); + } + if (namelen == 0 || namelen > EROFS_NAME_LEN) + return (EINTEGRITY); + for (size_t i = 0; i < namelen; i++) { + if (blk[nameoff + i] == '/') + return (EINTEGRITY); + } + *namelenp = namelen; + return (0); +} + +/* Validate the dirent array, name offsets, and names in a block. */ +int +erofs_validate_dirblock(const char *blk, uint32_t blksz, uint32_t maxsize, + uint32_t *ndirentsp) +{ + const struct erofs_dirent *de; + uint32_t endoff, first_nameoff, idx, nameoff, ndirents, prev_nameoff; + size_t namelen, prev_namelen; + int error; + + if (blksz < sizeof(struct erofs_dirent) || + maxsize < sizeof(struct erofs_dirent) || + maxsize > blksz) + return (EINTEGRITY); + de = (const struct erofs_dirent *)blk; + first_nameoff = le16toh(de[0].nameoff); + if (first_nameoff < sizeof(struct erofs_dirent) || + first_nameoff >= maxsize || + (first_nameoff % sizeof(struct erofs_dirent)) != 0) + return (EINTEGRITY); + ndirents = first_nameoff / sizeof(struct erofs_dirent); + prev_nameoff = 0; + prev_namelen = 0; + for (idx = 0; idx < ndirents; idx++) { + nameoff = le16toh(de[idx].nameoff); + if ((idx == 0 && nameoff != first_nameoff) || + (idx != 0 && nameoff <= prev_nameoff) || + nameoff < first_nameoff || nameoff >= maxsize) + return (EINTEGRITY); + endoff = idx + 1 < ndirents ? + le16toh(de[idx + 1].nameoff) : maxsize; + if (endoff <= nameoff || endoff > maxsize) + return (EINTEGRITY); + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + if (idx != 0 && erofs_dirname_order(blk + prev_nameoff, + prev_namelen, blk + nameoff, namelen) >= 0) + return (EINTEGRITY); + prev_nameoff = nameoff; + prev_namelen = namelen; + } + *ndirentsp = ndirents; + return (0); +} + +/* Extract name, nid, and type of the idx'th directory entry from a block. */ +static int +erofs_dirent_name(const char *blk, uint32_t maxsize, + uint32_t idx, uint32_t ndirents, char *name, size_t namesz, erofs_nid_t *nid, + uint8_t *ftype, size_t *namelenp) +{ + const struct erofs_dirent *de; + uint32_t nameoff, endoff; + size_t namelen; + int error; + + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[idx].nameoff); + if (idx + 1 < ndirents) + endoff = le16toh(de[idx + 1].nameoff); + else + endoff = maxsize; + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + if (namelen >= namesz) + return (EINTEGRITY); + memcpy(name, blk + nameoff, namelen); + name[namelen] = '\0'; + *nid = le64toh(de[idx].nid); + *ftype = de[idx].file_type; + *namelenp = namelen; + return (0); +} + +static int +erofs_previous_dirname(struct erofs_sb_info *sbi, struct erofs_inode *dir, + erofs_off_t block_off, char *name, size_t namesz, size_t *namelenp) +{ + erofs_nid_t nid; + uint32_t maxsize, ndirents; + uint8_t ftype; + char *blk; + int error; + + block_off -= sbi->block_size; + maxsize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, maxsize, (void **)&blk); + if (error != 0) + return (error); + error = erofs_validate_dirblock(blk, sbi->block_size, maxsize, + &ndirents); + if (error == 0) + error = erofs_dirent_name(blk, maxsize, ndirents - 1, + ndirents, name, namesz, &nid, &ftype, namelenp); + erofs_brelse(blk); + return (error); +} + +/* Per-call state for readdir dirent/cookie output. */ +struct erofs_uiodir { + struct dirent *dirent; + uint64_t *cookies; + uint64_t last_cookie; + int ncookies; + int acookies; + int eofflag; +}; + +enum erofs_uiodir_result { + EROFS_UIODIR_BUFFER_FULL = -1, + EROFS_UIODIR_OK = 0, +}; + +/* Push a dirent and its cookie to the caller, modelled after UDF. */ +static int +erofs_uiodir(struct erofs_uiodir *uiodir, int de_size, struct uio *uio, + uint64_t cookie) +{ + int error; + + if (cookie <= uiodir->last_cookie) + return (EINTEGRITY); + if (uio->uio_resid < de_size || + (uiodir->cookies != NULL && + uiodir->acookies >= uiodir->ncookies)) { + return (EROFS_UIODIR_BUFFER_FULL); + } + error = uiomove(uiodir->dirent, de_size, uio); + if (error != 0) + return (error); + uiodir->last_cookie = cookie; + if (uiodir->cookies != NULL) + uiodir->cookies[uiodir->acookies++] = cookie; + return (EROFS_UIODIR_OK); +} + +/* + * Process directory entries within a single block and output them to uio. + * (Linux equivalent: erofs_fill_dentries in Linux's dir.c) + * + * Returns 0 on success (all entries consumed), -1 if uio is full, or a + * positive error code on corruption. + */ +static int +erofs_fill_dentries(struct erofs_sb_info *sbi, struct erofs_uiodir *uiodir, + struct uio *uio, struct dirent *d, const char *blk, uint32_t maxsize, + uint32_t start_idx, uint32_t ndirents, erofs_off_t block_off, + uint64_t *logical_offp) +{ + char name[EROFS_NAME_LEN + 1]; + uint32_t idx; + uint64_t curpos, nextoff; + erofs_nid_t nid; + size_t namelen; + uint8_t ftype; + int error; + + for (idx = start_idx; idx < ndirents; idx++) { + curpos = block_off + idx * sizeof(struct erofs_dirent); + nextoff = (idx + 1 < ndirents) ? + (curpos + sizeof(struct erofs_dirent)) : + (block_off + maxsize); + error = erofs_dirent_name(blk, maxsize, idx, ndirents, name, + sizeof(name), &nid, &ftype, &namelen); + if (error != 0) + return (error); + if (!erofs_nid_is_valid(sbi, nid)) + return (EINTEGRITY); + bzero(d, sizeof(*d)); + d->d_fileno = nid; + d->d_type = erofs_ftype_to_dtype(ftype); + d->d_namlen = namelen; + d->d_reclen = GENERIC_DIRSIZ(d); + d->d_off = nextoff; + strlcpy(d->d_name, name, sizeof(d->d_name)); + error = erofs_uiodir(uiodir, d->d_reclen, uio, d->d_off); + if (error != 0) + return (error); + *logical_offp = nextoff; + uio->uio_offset = *logical_offp; + } + return (0); +} + +/* + * Read directory contents and output a FreeBSD dirent stream to uio. + * + * Key points: + * - On-disk entries use their logical file offsets as cookies; + * - A dot_omitted directory appends a synthetic "." at i_size, matching + * Linux, so existing on-disk cookies are not shifted; + * - The dirent array occupies only the front portion of a block, so after + * scanning all entries offset must jump to maxsize (the block end), + * otherwise the loop would get stuck on the same block; + * - Supports a_ncookies / a_cookies for NFS and other callers that need + * resumable iteration. + */ +int +erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies) +{ + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct erofs_uiodir uiodir; + struct dirent d; + uint64_t *cookiebuf; + size_t cookie_count; + char first_name[EROFS_NAME_LEN + 1]; + char previous_name[EROFS_NAME_LEN + 1]; + char *blk; + erofs_off_t block_off; + uint64_t logical_off; + uint32_t block_pos, blksz, ndirents, start_idx, maxsize; + erofs_nid_t edge_nid; + size_t first_namelen, previous_namelen; + uint8_t edge_ftype; + bool have_previous, sequential; + int error; + + dir = VTOE(vp); + sbi = MTOE(vp->v_mount); + blksz = sbi->block_size; + error = 0; + cookiebuf = NULL; + cookie_count = 0; + have_previous = false; + uiodir.eofflag = 0; + uiodir.acookies = 0; + uiodir.dirent = &d; + uiodir.cookies = NULL; + uiodir.ncookies = 0; + if (cookies != NULL && ncookies != NULL) { + *cookies = NULL; + *ncookies = 0; + if (uio->uio_resid > 0) { + cookie_count = (size_t)uio->uio_resid / + GENERIC_MINDIRSIZ; + cookie_count = MIN(cookie_count, (size_t)INT_MAX); + cookie_count = MIN(cookie_count, + SIZE_MAX / sizeof(*cookiebuf)); + } + uiodir.ncookies = (int)cookie_count; + if (cookie_count != 0) + cookiebuf = malloc(sizeof(*cookiebuf) * cookie_count, + M_TEMP, M_WAITOK); + uiodir.cookies = cookiebuf; + } + + if (uio->uio_offset < 0) { + error = EINVAL; + goto out; + } + if (dir->dot_omitted && dir->size == (uint64_t)OFF_MAX) { + error = EINTEGRITY; + goto out; + } + + logical_off = uio->uio_offset; + sequential = logical_off == 0; + uiodir.last_cookie = logical_off; + uio->uio_offset = logical_off; + + while (logical_off < dir->size) { + block_off = rounddown2(logical_off, (uint64_t)blksz); + maxsize = MIN((uint64_t)blksz, dir->size - block_off); + block_pos = logical_off - block_off; + if ((block_pos % sizeof(struct erofs_dirent)) != 0) { + block_pos = roundup(block_pos, sizeof(struct erofs_dirent)); + logical_off = block_off + block_pos; + uio->uio_offset = logical_off; + } + error = erofs_read_data_readahead(sbi, dir, block_off, + maxsize, sequential, (void **)&blk); + if (error != 0) + goto out; + error = erofs_validate_dirblock(blk, blksz, maxsize, &ndirents); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + if (!have_previous && block_off != 0) { + error = erofs_previous_dirname(sbi, dir, block_off, + previous_name, sizeof(previous_name), + &previous_namelen); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + have_previous = true; + } + error = erofs_dirent_name(blk, maxsize, 0, ndirents, + first_name, sizeof(first_name), &edge_nid, &edge_ftype, + &first_namelen); + if (error == 0 && have_previous && + erofs_dirname_order(previous_name, previous_namelen, + first_name, first_namelen) >= 0) + error = EINTEGRITY; + if (error == 0) + error = erofs_dirent_name(blk, maxsize, ndirents - 1, + ndirents, previous_name, sizeof(previous_name), + &edge_nid, &edge_ftype, &previous_namelen); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + have_previous = true; + start_idx = block_pos / sizeof(struct erofs_dirent); + if (start_idx >= ndirents) { + logical_off = block_off + maxsize; + uio->uio_offset = logical_off; + erofs_brelse(blk); + continue; + } + error = erofs_fill_dentries(sbi, &uiodir, uio, &d, blk, maxsize, + start_idx, ndirents, block_off, &logical_off); + erofs_brelse(blk); + if (error != 0) + goto out; + } + if (dir->dot_omitted && logical_off == dir->size) { + bzero(&d, sizeof(d)); + d.d_fileno = dir->nid; + d.d_type = DT_DIR; + d.d_namlen = 1; + d.d_reclen = GENERIC_DIRSIZ(&d); + d.d_off = dir->size + 1; + d.d_name[0] = '.'; + d.d_name[1] = '\0'; + error = erofs_uiodir(&uiodir, d.d_reclen, uio, d.d_off); + if (error != 0) + goto out; + logical_off++; + uio->uio_offset = logical_off; + } + uiodir.eofflag = 1; +out: + if (error == EROFS_UIODIR_BUFFER_FULL) + error = 0; + if (eofflag != NULL && error == 0) + *eofflag = uiodir.eofflag; + if (cookies != NULL && ncookies != NULL) { + if (error != 0) { + if (cookiebuf != NULL) + free(cookiebuf, M_TEMP); + } else { + *ncookies = uiodir.acookies; + *cookies = cookiebuf; + } + } + return (error); +} diff --git a/erofs_fs.h b/erofs_fs.h new file mode 100644 index 0000000..378af2c --- /dev/null +++ b/erofs_fs.h @@ -0,0 +1,496 @@ +/* SPDX-License-Identifier: MIT */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +#include +#include + +/* FreeBSD compatibility - Linux-style little-endian types */ +#ifndef __le16 +typedef uint16_t __le16; +typedef uint32_t __le32; +typedef uint64_t __le64; +typedef uint8_t __u8; +#endif + +/* to allow for x86 boot sectors and other oddities. */ +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 +#define EROFS_FEATURE_COMPAT_XATTR_FILTER 0x00000004 +#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX 0x00000008 +#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX 0x00000010 +#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS 0x00000020 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 +#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 +#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES 0x00000040 +#define EROFS_FEATURE_INCOMPAT_48BIT 0x00000080 +#define EROFS_FEATURE_INCOMPAT_METABOX 0x00000100 + +#define EROFS_ALL_FEATURE_INCOMPAT \ + (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | EROFS_FEATURE_INCOMPAT_48BIT | \ + EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ + EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES | \ + EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \ + EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ + EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \ + EROFS_FEATURE_INCOMPAT_FRAGMENTS | \ + EROFS_FEATURE_INCOMPAT_METABOX) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +/* Device table slot (128 bytes) */ +struct erofs_deviceslot { + uint8_t tag[64]; + __le32 blocks_lo; + __le32 uniaddr_lo; + __le16 blocks_hi; + __le16 uniaddr_hi; + uint8_t reserved[52]; +} __packed; +#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) + +/* erofs on-disk super block (currently 144 bytes at maximum) */ +struct erofs_super_block { + __le32 magic; + __le32 checksum; + __le32 feature_compat; + uint8_t blkszbits; + uint8_t sb_extslots; + union { + __le16 rootnid_2b; + __le16 blocks_hi; + } __packed rb; + __le64 inos; + __le64 epoch; + __le32 fixed_nsec; + __le32 blocks_lo; + __le32 meta_blkaddr; + __le32 xattr_blkaddr; + uint8_t uuid[16]; + uint8_t volume_name[16]; + __le32 feature_incompat; + union { + __le16 available_compr_algs; + __le16 lz4_max_distance; + } __packed u1; + __le16 extra_devices; + __le16 devt_slotoff; + uint8_t dirblkbits; + uint8_t xattr_prefix_count; + __le32 xattr_prefix_start; + __le64 packed_nid; + uint8_t xattr_filter_reserved; + uint8_t ishare_xattr_prefix_id; + uint8_t reserved[2]; + __le32 build_time; + __le64 rootnid_8b; + __le64 reserved2; + __le64 metabox_nid; + __le64 reserved3; +} __packed; + +/* EROFS inode datalayout (i_format in on-disk inode) */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_COMPRESSED_FULL = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_COMPRESSED_COMPACT = 3, + EROFS_INODE_CHUNK_BASED = 4, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool +erofs_inode_is_data_compressed(unsigned int datamode) +{ + return (datamode == EROFS_INODE_COMPRESSED_FULL || + datamode == EROFS_INODE_COMPRESSED_COMPACT); +} + +/* bit definitions of inode i_format */ +#define EROFS_I_VERSION_MASK 0x01 +#define EROFS_I_DATALAYOUT_MASK 0x07 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_NLINK_1_BIT 4 /* non-directory compact inodes only */ +#define EROFS_I_DOT_OMITTED_BIT 4 /* (directories) omit the `.` dirent */ +#define EROFS_I_ALL ((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1) + +/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS_MASK 0x001F +/* with chunk indexes or just a 4-byte block array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 +#define EROFS_CHUNK_FORMAT_48BIT 0x0040 +#define EROFS_CHUNK_FORMAT_ALL ((EROFS_CHUNK_FORMAT_48BIT << 1) - 1) + +/* 32-byte and 64-byte on-disk inode record layouts. */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +#define EROFS_INODE_LAYOUT_EXTENDED 1 +#define EROFS_INODE_LAYOUT_PLAIN EROFS_INODE_FLAT_PLAIN + +struct erofs_inode_chunk_info { + __le16 format; + __le16 reserved; +} __packed; + +union erofs_inode_i_u { + __le32 blocks_lo; + __le32 startblk_lo; + __le32 rdev; + struct erofs_inode_chunk_info c; +}; + +union erofs_inode_i_nb { + __le16 nlink; /* if EROFS_I_NLINK_1_BIT is unset */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 startblk_hi; /* starting block number MSB */ +} __packed; + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le32 i_size; + __le32 i_mtime; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved; +} __packed; + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le64 i_size; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le32 i_uid; + __le32 i_gid; + __le64 i_mtime; + __le32 i_mtime_nsec; + __le32 i_nlink; + uint8_t i_reserved2[16]; +} __packed; + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_name_filter; /* bit value 1 indicates not-present */ + uint8_t h_shared_count; + uint8_t h_reserved2[7]; + __le32 h_shared_xattrs[]; /* shared xattr id array */ +} __packed; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* + * bit 7 of e_name_index is set when it refers to a long xattr name prefix, + * while the remained lower bits represent the index of the prefix. + */ +#define EROFS_XATTR_LONG_PREFIX 0x80 +#define EROFS_XATTR_LONG_PREFIX_MASK 0x7f + +#define EROFS_XATTR_FILTER_BITS 32 +#define EROFS_XATTR_FILTER_DEFAULT UINT32_MAX +#define EROFS_XATTR_FILTER_SEED 0x25BBE08F + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + uint8_t e_name_len; + uint8_t e_name_index; + __le16 e_value_size; + char e_name[]; /* attribute name */ +} __packed; + +/* long xattr name prefix */ +struct erofs_xattr_long_prefix { + uint8_t base_index; /* short xattr name prefix index */ + char infix[]; /* infix apart from short prefix */ +} __packed; + +static inline unsigned int +erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + return (sizeof(struct erofs_xattr_ibody_header) + + sizeof(uint32_t) * (le16toh(i_xattr_icount) - 1)); +} + +#define EROFS_XATTR_ALIGN(size) \ + (((size) + sizeof(struct erofs_xattr_entry) - 1) & \ + ~(sizeof(struct erofs_xattr_entry) - 1)) + +static inline unsigned int +erofs_xattr_entry_size(const struct erofs_xattr_entry *entry) +{ + return (EROFS_XATTR_ALIGN( + sizeof(*entry) + entry->e_name_len + le16toh(entry->e_value_size))); +} + +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR ((uint64_t)-1) + +/* 4-byte block address array */ +#define EROFS_BLOCK_MAP_ENTRY_SIZE sizeof(__le32) + +/* 8-byte inode chunk index */ +struct erofs_inode_chunk_index { + __le16 startblk_hi; + __le16 device_id; + __le32 startblk_lo; +} __packed; + +#define EROFS_DIRENT_NID_METABOX_BIT 63 +#define EROFS_DIRENT_NID_METABOX \ + (1ULL << EROFS_DIRENT_NID_METABOX_BIT) +#define EROFS_DIRENT_NID_MASK \ + ((1ULL << EROFS_DIRENT_NID_METABOX_BIT) - 1) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; + __le16 nameoff; + uint8_t file_type; + uint8_t reserved; +} __packed; + +/* file type definitions in directory entries */ +#define EROFS_FT_UNKNOWN 0 +#define EROFS_FT_REG_FILE 1 +#define EROFS_FT_DIR 2 +#define EROFS_FT_CHRDEV 3 +#define EROFS_FT_BLKDEV 4 +#define EROFS_FT_FIFO 5 +#define EROFS_FT_SOCK 6 +#define EROFS_FT_SYMLINK 7 + +#define EROFS_NAME_LEN 255 + +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) +#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) + +/* compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_LZMA = 1, + Z_EROFS_COMPRESSION_DEFLATE = 2, + Z_EROFS_COMPRESSION_ZSTD = 3, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS ((1 << Z_EROFS_COMPRESSION_MAX) - 1) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + uint8_t reserved[10]; +} __packed; + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lzma_cfgs { + __le32 dict_size; + __le16 format; + uint8_t reserved[8]; +} __packed; + +#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE) + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_deflate_cfgs { + uint8_t windowbits; + uint8_t reserved[5]; +} __packed; + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_zstd_cfgs { + uint8_t format; + uint8_t windowlog; + uint8_t reserved[4]; +} __packed; + +#define Z_EROFS_ZSTD_MAX_DICT_SIZE Z_EROFS_PCLUSTER_MAX_SIZE + +/* z_advise flags */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +#define Z_EROFS_ADVISE_EXTENTS 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 +#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 + +#define Z_EROFS_FRAGMENT_INODE_BIT 7 + +struct z_erofs_map_header { + union { + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + __le16 h_idata_size; + }; + __le32 h_extents_lo; + }; + __le16 h_advise; + union { + struct { + uint8_t h_algorithmtype; + uint8_t h_clusterbits; + } __packed; + __le16 h_extents_hi; + } __packed; +} __packed; + +/* Logical cluster types */ +enum { + Z_EROFS_LCLUSTER_TYPE_PLAIN = 0, + Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1, + Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3, + Z_EROFS_LCLUSTER_TYPE_MAX +}; + +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) +#define Z_EROFS_LI_PARTIAL_REF (1 << 15) +#define Z_EROFS_LI_D0_CBLKCNT (1 << 11) + +/* Compression extent index structures */ +struct z_erofs_lcluster_index { + __le16 di_advise; + __le16 di_clusterofs; + union { + __le32 blkaddr; + __le16 delta[2]; + } di_u; +} __packed; + +#define Z_EROFS_MAP_HEADER_END(end) \ + (roundup2((end), 8) + sizeof(struct z_erofs_map_header)) +#define Z_EROFS_FULL_INDEX_START(end) (Z_EROFS_MAP_HEADER_END(end) + 8) + +#define Z_EROFS_EXTENT_PLEN_PARTIAL (1U << 27) +#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 +#define Z_EROFS_EXTENT_PLEN_MASK \ + ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) +struct z_erofs_extent { + __le32 plen; + __le32 pstart_lo; + __le32 pstart_hi; + __le32 lstart_lo; + __le32 lstart_hi; + uint8_t reserved[12]; +} __packed; + +static inline unsigned int +z_erofs_extent_recsize(unsigned int advise) +{ + return (4U << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & + Z_EROFS_ADVISE_EXTRECSZ_MASK)); +} + +_Static_assert(sizeof(struct erofs_deviceslot) == 128, + "EROFS device slot ABI size"); +_Static_assert(sizeof(struct erofs_super_block) == 144, + "EROFS super block ABI size"); +_Static_assert(sizeof(struct erofs_inode_compact) == 32, + "EROFS compact inode ABI size"); +_Static_assert(sizeof(struct erofs_inode_extended) == 64, + "EROFS extended inode ABI size"); +_Static_assert(sizeof(struct erofs_xattr_ibody_header) == 12, + "EROFS xattr ibody header ABI size"); +_Static_assert(sizeof(struct erofs_xattr_entry) == 4, + "EROFS xattr entry ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_info) == 4, + "EROFS chunk info ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_index) == 8, + "EROFS chunk index ABI size"); +_Static_assert(sizeof(struct z_erofs_map_header) == 8, + "EROFS zmap header ABI size"); +_Static_assert(sizeof(struct z_erofs_lcluster_index) == 8, + "EROFS lcluster index ABI size"); +_Static_assert(sizeof(struct z_erofs_extent) == 32, + "EROFS compression extent ABI size"); +_Static_assert(sizeof(struct erofs_dirent) == 12, + "EROFS dirent ABI size"); +_Static_assert(sizeof(struct erofs_deviceslot) == EROFS_DEVT_SLOT_SIZE, + "EROFS device slot ABI macro"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, extra_devices) == 86, + "EROFS extra device count ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, devt_slotoff) == 88, + "EROFS device table slot offset ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, rootnid_8b) == 112, + "EROFS 48-bit root nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, metabox_nid) == 128, + "EROFS metabox nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_compact, i_u) == 16, + "EROFS compact inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_u) == 16, + "EROFS extended inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_reserved2) == 48, + "EROFS extended inode reserved ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_chunk_index, device_id) == 2, + "EROFS chunk device id ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_chunk_index, startblk_lo) == 4, + "EROFS chunk start block ABI offset"); +_Static_assert(__builtin_offsetof(struct z_erofs_map_header, h_advise) == 4, + "EROFS zmap advise ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_lo) == 64, + "EROFS device blocks ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_lo) == 68, + "EROFS device unified address ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_hi) == 72, + "EROFS device blocks high ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_hi) == 74, + "EROFS device unified address high ABI offset"); + +#endif diff --git a/erofs_vnops.c b/erofs_vnops.c new file mode 100644 index 0000000..83b176d --- /dev/null +++ b/erofs_vnops.c @@ -0,0 +1,570 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "xattr.h" + +static vop_inactive_t erofs_inactive; +static vop_reclaim_t erofs_reclaim; + +static vop_readdir_t erofs_readdir; +static vop_readlink_t erofs_readlink; + +static vop_open_t erofs_open; +static vop_read_t erofs_read; +static vop_bmap_t erofs_bmap; + +static vop_getattr_t erofs_getattr; +static vop_setattr_t erofs_setattr; +static vop_access_t erofs_access; +static vop_pathconf_t erofs_pathconf; + +static vop_getextattr_t erofs_getextattr; +static vop_listextattr_t erofs_listextattr; +static vop_deleteextattr_t erofs_deleteextattr; +static vop_setextattr_t erofs_setextattr; +static vop_getacl_t erofs_vop_getacl; +static vop_aclcheck_t erofs_aclcheck; +static vop_setacl_t erofs_setacl; + +/* vop_fhtovp removed in FreeBSD 15.0 */ +static vop_vptofh_t erofs_vptofh; + +/* Access check: data nodes are read-only, but device/FIFO nodes are not denied + * writes. */ +static int +erofs_access(struct vop_access_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct acl *acl; + accmode_t accmode; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + accmode = ap->a_accmode; + if ((accmode & VMODIFY_PERMS) != 0) { + switch (vp->v_type) { + case VDIR: + case VLNK: + case VREG: + return (EROFS); + default: + break; + } + } + error = vfs_unixify_accmode(&accmode); + if (error != 0) + return (error); + if ((vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); + + acl = acl_alloc(M_WAITOK); + error = erofs_get_acl(vp, ACL_TYPE_ACCESS, acl); + if (error == 0) + error = vaccess_acl_posix1e(vp->v_type, vi->uid, vi->gid, acl, + accmode, ap->a_cred); + acl_free(acl); + return (error); +} + +/* + * Tell the generic pager that EROFS does not provide block-level bmap. + * + * Returning EOPNOTSUPP prevents the pager from assuming a bufobj/strategy is + * available, avoiding “No strategy for buffer” errors. VM will correctly + * fall back to the VOP_READ-based page-in path. + */ +static int +erofs_bmap(struct vop_bmap_args *ap) +{ + (void)ap; + return (EOPNOTSUPP); +} + +/* No dirty writeback on last ref release, so inactive is a no-op. */ +static int +erofs_inactive(struct vop_inactive_args *ap) +{ + (void)ap; + return (0); +} + +/* + * Create VM object when opening a regular vnode. + * + * The FreeBSD local vnode pager services synchronous and asynchronous faults + * through VOP_READ without requiring a block strategy method. + */ +static int +erofs_open(struct vop_open_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (VN_ISDEV(vp)) + return (EOPNOTSUPP); + if (vp->v_type == VREG) + vnode_create_vobject(vp, vi->size, ap->a_td); + return (0); +} + +static int +erofs_getattr(struct vop_getattr_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct erofs_sb_info *sbi; + struct vattr *vap; + + vp = ap->a_vp; + vi = VTOE(vp); + sbi = MTOE(vp->v_mount); + vap = ap->a_vap; + VATTR_NULL(vap); + vap->va_type = vp->v_type; + vap->va_mode = vi->mode & ALLPERMS; + vap->va_nlink = vi->nlink; + vap->va_uid = vi->uid; + vap->va_gid = vi->gid; + vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; + vap->va_fileid = vi->nid; + vap->va_size = vi->size; + vap->va_blocksize = sbi->block_size; + vap->va_atime.tv_sec = vi->mtime; + vap->va_mtime.tv_sec = vi->mtime; + vap->va_ctime.tv_sec = vi->mtime; + vap->va_atime.tv_nsec = vi->mtime_nsec; + vap->va_mtime.tv_nsec = vi->mtime_nsec; + vap->va_ctime.tv_nsec = vi->mtime_nsec; + vap->va_gen = vi->generation; + vap->va_flags = 0; + vap->va_rdev = VN_ISDEV(vp) ? vi->rdev : NODEV; + if (vi->data_blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + vap->va_bytes = vi->data_blocks << sbi->blkszbits; + vap->va_filerev = 0; + return (0); +} + +/* + * Read-only xattr get entry point. + * + * Delegates to erofs_getxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.* / security.*) + */ +static int +erofs_getextattr(struct vop_getextattr_args *ap) +{ + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + if (ap->a_name == NULL || ap->a_name[0] == '\0') + return (EINVAL); + if (strlen(ap->a_name) > EXTATTR_MAXNAMELEN) + return (EINVAL); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_getxattr(ap->a_vp, ap->a_attrnamespace, ap->a_name, + ap->a_uio, ap->a_size)); +} + +/* + * Read-only xattr list entry point. + * + * Delegates to erofs_listxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.* / security.*) + */ +static int +erofs_listextattr(struct vop_listextattr_args *ap) +{ + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_listxattr(ap->a_vp, ap->a_attrnamespace, ap->a_uio, + ap->a_size)); +} + +static int +erofs_deleteextattr(struct vop_deleteextattr_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +static int +erofs_setextattr(struct vop_setextattr_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +/* EROFS is read-only; mutations on regular files/dirs/symlinks are denied, size + * changes on special vnodes are treated as no-ops per read-only convention. */ +static int +erofs_setattr(struct vop_setattr_args *ap) +{ + struct vnode *vp; + struct vattr *vap; + + vp = ap->a_vp; + vap = ap->a_vap; + if (vap->va_mode != (mode_t)VNOVAL || vap->va_uid != (uid_t)VNOVAL || + vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL || + vap->va_atime.tv_nsec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || + vap->va_mtime.tv_nsec != VNOVAL || vap->va_flags != VNOVAL) + return (EROFS); + if (vap->va_size != VNOVAL) { + switch (vp->v_type) { + case VDIR: + return (EISDIR); + case VLNK: + case VREG: + return (EROFS); + case VCHR: + case VBLK: + case VSOCK: + case VFIFO: + case VNON: + case VBAD: + case VMARKER: + return (0); + } + } + return (0); +} + +static int +erofs_read(struct vop_read_args *ap) +{ + switch (ap->a_vp->v_type) { + case VREG: + return (erofs_read_file(ap->a_vp, ap->a_uio, ap->a_ioflag)); + case VDIR: + return (EISDIR); + default: + return (EINVAL); + } +} + +static int +erofs_readdir(struct vop_readdir_args *ap) +{ + if (ap->a_vp->v_type != VDIR) + return (ENOTDIR); + return (erofs_readdir_block(ap->a_vp, ap->a_uio, ap->a_eofflag, + ap->a_ncookies, ap->a_cookies)); +} + +static int +erofs_readlink(struct vop_readlink_args *ap) +{ + if (ap->a_vp->v_type != VLNK) + return (EINVAL); + return (erofs_readlink_target(ap->a_vp, ap->a_uio)); +} + +static int +erofs_pathconf(struct vop_pathconf_args *ap) +{ + switch (ap->a_name) { + case _PC_NAME_MAX: + *ap->a_retval = EROFS_NAME_LEN; + return (0); + case _PC_PATH_MAX: + *ap->a_retval = PATH_MAX; + return (0); + case _PC_FILESIZEBITS: + *ap->a_retval = 64; + return (0); + case _PC_LINK_MAX: + *ap->a_retval = INT_MAX; + return (0); + case _PC_CHOWN_RESTRICTED: + case _PC_NO_TRUNC: + *ap->a_retval = 1; + return (0); + case _PC_ACL_EXTENDED: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? 1 : 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? + ACL_MAX_ENTRIES : 3; + return (0); + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); + default: + return (vop_stdpathconf(ap)); + } +} + +static int +erofs_vop_getacl(struct vop_getacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + return (erofs_get_acl(ap->a_vp, ap->a_type, ap->a_aclp)); +} + +static int +erofs_aclcheck(struct vop_aclcheck_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + if (ap->a_aclp == NULL) + return (EINVAL); + switch (ap->a_type) { + case ACL_TYPE_ACCESS: + break; + case ACL_TYPE_DEFAULT: + if (ap->a_vp->v_type != VDIR) + return (EINVAL); + break; + default: + return (EINVAL); + } + return (acl_posix1e_check(ap->a_aclp)); +} + +static int +erofs_setacl(struct vop_setacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +static u_int +erofs_vfs_hash(erofs_nid_t nid) +{ + + return (fnv_32_buf(&nid, sizeof(nid), FNV1_32_INIT)); +} + +static int +erofs_vfs_hash_cmp(struct vnode *vp, void *pnid) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + return (vi == NULL || vi->nid != *(erofs_nid_t *)pnid); +} + +static void +erofs_fill_vnode(struct erofs_sb_info *sbi, struct vnode *vp, + const struct erofs_inode *vi) +{ + vp->v_type = vi->vtype; + if (vp->v_type == VFIFO) + vp->v_op = &erofs_fifoops; + if (vi->nid == sbi->root_nid) + vp->v_vflag |= VV_ROOT; +} + +/* + * Get vnode by raw on-disk nid. The raw nid is also the FreeBSD fileid and + * hash identity, so the metabox selector bit remains collision-free. + * Uses the standard FreeBSD vfs_hash API. + * (Linux equivalent: erofs_iget in Linux's inode.c) + */ +int +erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct thread *td; + struct vnode *vp; + erofs_nid_t nid; + u_int hash; + bool shared; + int error; + + td = curthread; + nid = (uint64_t)ino; + shared = (flags & LK_TYPE_MASK) == LK_SHARED; + hash = erofs_vfs_hash(nid); + error = vfs_hash_get(mp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + sbi = MTOE(mp); + vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO); + error = getnewvnode("erofs", mp, &erofs_vnodeops, &vp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + vp->v_data = vi; + vi->nid = nid; + lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); + error = insmntque(vp, mp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + error = vfs_hash_insert(vp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + error = erofs_read_inode(sbi, nid, vi); + if (error != 0) { + *vpp = NULL; + vgone(vp); + vput(vp); + return (error); + } + erofs_xattr_cache_init(vi); + erofs_fill_vnode(sbi, vp, vi); + vn_set_state(vp, VSTATE_CONSTRUCTED); + if (shared) + VOP_LOCK(vp, LK_DOWNGRADE); + *vpp = vp; + return (0); +} + +static int +erofs_reclaim(struct vop_reclaim_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (vi != NULL) { + vfs_hash_remove(vp); + erofs_xattr_cache_fini(MTOE(vp->v_mount), vi); + free(vi, M_EROFS); + vp->v_data = NULL; + } + return (0); +} + +/* Vnode pointer to persistent EROFS file handle. */ +static int +erofs_vptofh(struct vop_vptofh_args *ap) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + + vi = VTOE(ap->a_vp); + bzero(&efid, sizeof(efid)); + efid.len = sizeof(efid); + efid.nid_hi = vi->nid >> 32; + efid.nid_lo = vi->nid; + efid.gen = vi->generation; + memcpy(ap->a_fhp, &efid, sizeof(efid)); + return (0); +} + +struct vop_vector erofs_vnodeops = { + .vop_default = &default_vnodeops, + .vop_inactive = erofs_inactive, + .vop_reclaim = erofs_reclaim, + + .vop_lookup = vfs_cache_lookup, + .vop_cachedlookup = erofs_lookup, + .vop_readdir = erofs_readdir, + .vop_readlink = erofs_readlink, + + .vop_open = erofs_open, + .vop_read = erofs_read, + .vop_bmap = erofs_bmap, + .vop_getpages = vnode_pager_local_getpages, + .vop_getpages_async = vnode_pager_local_getpages_async, + + .vop_getattr = erofs_getattr, + .vop_setattr = erofs_setattr, + .vop_access = erofs_access, + .vop_pathconf = erofs_pathconf, + + .vop_getextattr = erofs_getextattr, + .vop_listextattr = erofs_listextattr, + .vop_deleteextattr = erofs_deleteextattr, + .vop_setextattr = erofs_setextattr, + .vop_getacl = erofs_vop_getacl, + .vop_aclcheck = erofs_aclcheck, + .vop_setacl = erofs_setacl, + + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_vnodeops); + +struct vop_vector erofs_fifoops = { + .vop_default = &fifo_specops, + .vop_access = erofs_access, + .vop_aclcheck = erofs_aclcheck, + .vop_deleteextattr = erofs_deleteextattr, + .vop_getacl = erofs_vop_getacl, + .vop_getextattr = erofs_getextattr, + .vop_getattr = erofs_getattr, + .vop_listextattr = erofs_listextattr, + .vop_pathconf = erofs_pathconf, + .vop_reclaim = erofs_reclaim, + .vop_setacl = erofs_setacl, + .vop_setattr = erofs_setattr, + .vop_setextattr = erofs_setextattr, + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_fifoops); diff --git a/inode.c b/inode.c new file mode 100644 index 0000000..36ba7da --- /dev/null +++ b/inode.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static uint64_t +erofs_addrmask(const struct erofs_sb_info *sbi) +{ + if (erofs_sb_has_48bit(sbi)) + return ((1ULL << 48) - 1); + return (UINT32_MAX); +} + +static dev_t +erofs_decode_dev(uint32_t dev) +{ + unsigned int major, minor; + + major = (dev & 0xfff00) >> 8; + minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); + return (makedev(major, minor)); +} + +static uint32_t +erofs_inode_generation(const struct erofs_sb_info *sbi, erofs_nid_t nid, + const void *inode, size_t inode_size) +{ + uint8_t encoded_nid[sizeof(nid)]; + uint32_t generation; + + le64enc(encoded_nid, nid); + generation = fnv_32_buf(encoded_nid, sizeof(encoded_nid), + sbi->generation_seed); + generation = fnv_32_buf(inode, inode_size, generation); + return (generation != 0 ? generation : 1); +} + +static int +erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds, + uint32_t nanoseconds) +{ + time_t mtime; + + if (nanoseconds >= 1000000000 || + __builtin_add_overflow(seconds, 0, &mtime)) + return (EINTEGRITY); + vi->mtime = mtime; + vi->mtime_nsec = nanoseconds; + return (0); +} + +static int +erofs_set_data_blocks(const struct erofs_sb_info *sbi, struct erofs_inode *vi, + uint64_t compressed_blocks) +{ + if (erofs_inode_is_data_compressed(vi->datalayout)) { + vi->data_blocks = compressed_blocks; + return (0); + } + if (vi->size == 0) { + vi->data_blocks = 0; + return (0); + } + if (vi->size > UINT64_MAX - (sbi->block_size - 1)) + return (EINTEGRITY); + vi->data_blocks = roundup2(vi->size, (uint64_t)sbi->block_size) >> + sbi->blkszbits; + return (0); +} + +static int +erofs_validate_inline_data(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + uint64_t image_size, inline_end, inline_off, inline_size, tail_start; + + if (vi->datalayout != EROFS_INODE_FLAT_INLINE || vi->size == 0) + return (0); + tail_start = roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size; + inline_size = vi->size - tail_start; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, &inline_off) || + __builtin_add_overflow(inline_off, vi->xattr_isize, &inline_off) || + __builtin_add_overflow(inline_off, inline_size, &inline_end)) + return (EINTEGRITY); + if ((inline_off & (sbi->block_size - 1)) + inline_size > sbi->block_size) + return (EINTEGRITY); + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || inline_end > sbi->metabox_en->size) + return (EINTEGRITY); + return (0); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + image_size = sbi->blocks << sbi->blkszbits; + if (inline_end > image_size || inline_end > sbi->dif0.mediasize) + return (EINTEGRITY); + return (0); +} + +/* + * Convert a logical nid to its inode-table byte offset. Normal NIDs are + * relative to the primary metadata area. For metabox NIDs, bit 63 selects + * the metabox backing inode and the remaining bits are relative to its data. + * EROFS_NULL_ADDR is returned when the address cannot be represented. + */ +static erofs_off_t +erofs_iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t meta_offset; + erofs_nid_t nid_lo; + erofs_off_t result; + bool in_metabox; + + in_metabox = erofs_nid_in_metabox(nid); + if (in_metabox && !erofs_sb_has_metabox(sbi)) + return (EROFS_NULL_ADDR); + nid_lo = nid & EROFS_DIRENT_NID_MASK; + if (nid_lo > (UINT64_MAX >> 5)) + return (EROFS_NULL_ADDR); + result = nid_lo << 5; + if (in_metabox) + return (result); + + if (sbi->blkszbits > 58) + return (EROFS_NULL_ADDR); + meta_offset = (uint64_t)sbi->meta_blkaddr << sbi->blkszbits; + if (result > UINT64_MAX - meta_offset) + return (EROFS_NULL_ADDR); + + return (meta_offset + result); +} + +/* + * Check that a NID can address at least one compact inode slot without + * crossing the declared primary image or metabox backing-file boundary. + */ +bool +erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t image_size, off; + + off = erofs_iloc(sbi, nid); + if (off == EROFS_NULL_ADDR) + return (false); + if (erofs_nid_in_metabox(nid)) { + if (sbi->metabox_en == NULL || off > sbi->metabox_en->size) + return (false); + return (sizeof(struct erofs_inode_compact) <= + sbi->metabox_en->size - off); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (false); + image_size = sbi->blocks << sbi->blkszbits; + if (off > image_size || sizeof(struct erofs_inode_compact) > + image_size - off) + return (false); + if (off > sbi->dif0.mediasize || sizeof(struct erofs_inode_compact) > + sbi->dif0.mediasize - off) + return (false); + return (true); +} + +bool +erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype) +{ + switch (file_type) { + case EROFS_FT_REG_FILE: + return (vtype == VREG); + case EROFS_FT_DIR: + return (vtype == VDIR); + case EROFS_FT_CHRDEV: + return (vtype == VCHR); + case EROFS_FT_BLKDEV: + return (vtype == VBLK); + case EROFS_FT_FIFO: + return (vtype == VFIFO); + case EROFS_FT_SOCK: + return (vtype == VSOCK); + case EROFS_FT_SYMLINK: + return (vtype == VLNK); + default: + return (true); + } +} + +/* + * Read and decode a disk inode. + * + * Currently supports: + * - compact / extended inode; + * - plain / inline uncompressed layouts; + * - basic 48-bit address parsing; + * - compact inode epoch/fixed_nsec timestamp semantics; + * - dot_omitted / nlink==1 i_format details. + */ +int +erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, struct erofs_inode *vi) +{ + struct erofs_inode_compact *dic; + struct erofs_inode_extended *die; + struct erofs_inode_chunk_info chunk_info; + union erofs_inode_i_nb inode_nb; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t addrmask; + int64_t mtime; + erofs_off_t off; + erofs_blk_t startblk; + uint64_t compressed_blocks; + uint32_t raw_rdev, startblk_lo; + uint16_t ifmt, startblk_hi; + int error; + + if (!erofs_nid_is_valid(sbi, nid)) + return (EINTEGRITY); + off = erofs_iloc(sbi, nid); + error = erofs_read_metadata(sbi, nid, off, + sizeof(struct erofs_inode_compact), &buf); + if (error != 0) + return (error); + + bzero(&vi->size, sizeof(*vi) - offsetof(struct erofs_inode, size)); + vi->nid = nid; + vi->inode_off = off; + ifmt = le16dec(buf.data); + if ((ifmt & ~EROFS_I_ALL) != 0) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + vi->compact_inode = (erofs_inode_version(ifmt) == 0); + if (!vi->compact_inode) { + erofs_put_metabuf(&buf); + error = erofs_read_metadata(sbi, nid, off, + sizeof(struct erofs_inode_extended), &buf); + if (error != 0) + return (error); + } + addrmask = erofs_addrmask(sbi); + startblk = EROFS_NULL_ADDR; + startblk_lo = 0; + startblk_hi = 0; + compressed_blocks = 0; + raw_rdev = 0; + bzero(&inode_nb, sizeof(inode_nb)); + dic = buf.data; + if (vi->compact_inode) { + vi->inode_isize = sizeof(struct erofs_inode_compact); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(dic->i_mode); + vi->size = le32toh(dic->i_size); + vi->uid = le16toh(dic->i_uid); + vi->gid = le16toh(dic->i_gid); + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + if (__builtin_add_overflow(sbi->epoch, + (int64_t)le32toh(dic->i_mtime), &mtime)) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + error = erofs_set_timestamp(vi, mtime, sbi->fixed_nsec); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + startblk_lo = le32toh(dic->i_u.startblk_lo); + compressed_blocks = le32toh(dic->i_u.blocks_lo); + raw_rdev = le32toh(dic->i_u.rdev); + if (!S_ISDIR(vi->mode) && + ((ifmt >> EROFS_I_NLINK_1_BIT) & 0x1) != 0) { + vi->nlink = 1; + inode_nb = dic->i_nb; + } else { + vi->nlink = le16toh(dic->i_nb.nlink); + addrmask = UINT32_MAX; + } + } else { + die = buf.data; + vi->inode_isize = sizeof(struct erofs_inode_extended); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(die->i_mode); + vi->size = le64toh(die->i_size); + vi->uid = le32toh(die->i_uid); + vi->gid = le32toh(die->i_gid); + vi->nlink = le32toh(die->i_nlink); + inode_nb = die->i_nb; + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + error = erofs_set_timestamp(vi, + (int64_t)le64toh(die->i_mtime), + le32toh(die->i_mtime_nsec)); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + startblk_lo = le32toh(die->i_u.startblk_lo); + compressed_blocks = le32toh(die->i_u.blocks_lo); + raw_rdev = le32toh(die->i_u.rdev); + } + startblk_hi = le16toh(inode_nb.startblk_hi); + compressed_blocks |= (uint64_t)le16toh(inode_nb.blocks_hi) << 32; + if (vi->size > (uint64_t)OFF_MAX) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + + vi->vtype = IFTOVT(vi->mode); + if (vi->mode != 0 && vi->vtype == VNON) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->vtype == VLNK) { + if (vi->size == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->size > MAXPATHLEN) { + erofs_put_metabuf(&buf); + return (ENAMETOOLONG); + } + } + vi->dot_omitted = (vi->vtype == VDIR) && + (((ifmt >> EROFS_I_DOT_OMITTED_BIT) & 0x1) != 0); + + if (erofs_inode_is_data_compressed(vi->datalayout)) { + error = z_erofs_fill_inode(sbi, vi); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + } else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + if (!erofs_sb_has_chunked_file(sbi)) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->compact_inode) + chunk_info = dic->i_u.c; + else + chunk_info = die->i_u.c; + if (le16toh(chunk_info.reserved) != 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->chunkformat = le16toh(chunk_info.format); + if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 && + (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->chunkbits = sbi->blkszbits + + (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); + if (vi->chunkbits >= 64) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + } else if (vi->datalayout != EROFS_INODE_FLAT_PLAIN && + vi->datalayout != EROFS_INODE_FLAT_INLINE) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + + switch (vi->vtype) { + case VREG: + case VDIR: + case VLNK: + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + } + startblk = startblk_lo | ((uint64_t)startblk_hi << 32); + if (vi->datalayout == EROFS_INODE_FLAT_PLAIN && + ((startblk ^ EROFS_NULL_ADDR) & addrmask) == 0) + startblk = EROFS_NULL_ADDR; + vi->startblk = startblk; + vi->rdev = NODEV; + break; + case VCHR: + case VBLK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = erofs_decode_dev(raw_rdev); + break; + case VFIFO: + case VSOCK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + default: + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + error = erofs_set_data_blocks(sbi, vi, compressed_blocks); + if (error == 0) + error = erofs_validate_inline_data(sbi, vi); + if (error == 0 && vi->vtype == VLNK) + error = erofs_validate_symlink_target(sbi, vi); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + + erofs_put_metabuf(&buf); + return (0); +} diff --git a/internal.h b/internal.h new file mode 100644 index 0000000..4ba3f29 --- /dev/null +++ b/internal.h @@ -0,0 +1,423 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#ifndef __EROFS_INTERNAL_H +#define __EROFS_INTERNAL_H + +#include +#include // MUST FIRST +#include +#include +#include +#include +#include +#include +#include + +#include "erofs_fs.h" + +MALLOC_DECLARE(M_EROFS); + +struct cdev; +struct g_consumer; +struct erofs_device_info; + +/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ +#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 + +typedef uint64_t erofs_nid_t; +typedef uint64_t erofs_off_t; +typedef uint64_t erofs_blk_t; + +#define EROFS_MOUNT_POSIX_ACL 0x00000020 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +struct erofs_sb_info_opts { + unsigned int mount_opt; +}; + +struct erofs_device_info { + struct vnode *devvp; + struct cdev *dev; + struct g_consumer *cp; + uint64_t mediasize; + uint32_t sectorsize; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +struct erofs_xattr_prefix_item { + uint8_t base_index; + uint8_t infix_len; + char *infix; +}; + +struct erofs_map_blocks { + erofs_off_t m_pa, m_la; + uint64_t m_plen, m_llen; + + unsigned short m_deviceid; + uint8_t m_algorithmformat; + unsigned int m_flags; +}; + +enum erofs_zextent_cache_state { + EROFS_ZCACHE_EMPTY, + EROFS_ZCACHE_INFLIGHT, + EROFS_ZCACHE_READY, + EROFS_ZCACHE_FAILED, +}; + +struct erofs_zextent_cache_metrics { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + size_t resident_bytes; +}; + +struct erofs_zextent_cache { + struct erofs_map_blocks map; + void *data; + erofs_nid_t nid; + size_t decoded_size; + size_t charged_bytes; + size_t budget_bytes; + uint64_t minimum_decode_work; + struct erofs_zextent_cache_metrics metrics[Z_EROFS_COMPRESSION_MAX]; + unsigned int waiters; + int error; + enum erofs_zextent_cache_state state; + struct cv cv; + bool closing; +}; + +enum erofs_xattr_cache_state { + EROFS_XATTR_CACHE_EMPTY, + EROFS_XATTR_CACHE_INFLIGHT, + EROFS_XATTR_CACHE_READY, + EROFS_XATTR_CACHE_FAILED, +}; + +struct erofs_xattr_cache { + struct mtx lock; + struct cv cv; + void *data; + size_t size; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_xattr_cache_state state; + bool closing; + bool initialized; +}; + +#define EROFS_STREAM_CTX_WRAPPER_SIZE 256 + +struct erofs_stream_ctx; +typedef int erofs_stream_ctx_init_t(struct erofs_stream_ctx *ctx); +typedef void erofs_stream_ctx_fini_t(struct erofs_stream_ctx *ctx); + +struct erofs_stream_ctx { + STAILQ_ENTRY(erofs_stream_ctx) link; + struct erofs_sb_info *sbi; + erofs_stream_ctx_fini_t *fini; + size_t charged_bytes; + uint8_t algorithm; + bool cached; + bool allocation_failed; +}; + +STAILQ_HEAD(erofs_stream_ctx_head, erofs_stream_ctx); + +struct erofs_stream_pool_codec { + struct erofs_stream_ctx_head idle; + unsigned int contexts; + unsigned int cached; + unsigned int borrowed; + unsigned int idle_count; +}; + +struct erofs_stream_pool { + struct erofs_stream_pool_codec codec[Z_EROFS_COMPRESSION_MAX]; + size_t resident_bytes; + struct cv cv; + bool closing; +}; + +struct erofs_sb_info { + struct mount *mnt; + struct erofs_device_info dif0; + + uint32_t block_size; + uint32_t sb_size; + uint8_t blkszbits; + uint32_t meta_blkaddr; + uint32_t xattr_blkaddr; + uint32_t xattr_prefix_start; + uint8_t xattr_prefix_count; + uint8_t xattr_filter_reserved; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + struct erofs_inode *metabox_en; + struct erofs_inode *packed_inode; + struct erofs_xattr_prefix_item *xattr_prefixes; + erofs_blk_t blocks; + uint64_t inos; + erofs_nid_t root_nid; + int64_t epoch; + uint32_t fixed_nsec; + uint32_t generation_seed; + uint32_t feature_compat; + uint32_t feature_incompat; + char volume_name[17]; + + struct erofs_sb_info_opts opt; + uint16_t available_compr_algs; + uint32_t lzma_dict_size; + uint8_t deflate_windowbits; + uint8_t zstd_windowlog; + + /* Device table */ + uint16_t extra_devices; + uint16_t device_id_mask; + bool flatdev; + erofs_blk_t total_blocks; + erofs_blk_t flatdev_blocks; + struct erofs_device_info *devs; + struct mtx z_extent_cache_lock; + struct erofs_zextent_cache z_extent_cache; + LIST_ENTRY(erofs_sb_info) z_extent_cache_link; + bool z_extent_cache_initialized; + struct erofs_stream_pool stream_pool; + LIST_ENTRY(erofs_sb_info) stream_pool_link; + bool stream_pool_initialized; + volatile u_long xattr_cache_resident; +}; + +#define MTOE(mp) ((struct erofs_sb_info *)(mp)->mnt_data) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ + static inline bool erofs_sb_has_##name(const struct erofs_sb_info *sbi) \ + { \ + return ( \ + (sbi->feature_##compat & EROFS_FEATURE_##feature) != 0); \ + } + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) +EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS) + +static inline bool +erofs_sb_has_xattr_filter_v1(const struct erofs_sb_info *sbi) +{ + return (erofs_sb_has_xattr_filter(sbi) && + sbi->xattr_filter_reserved == 0); +} + +struct erofs_inode { + erofs_nid_t nid; + uint64_t size; + uint64_t data_blocks; + /* Absolute device offset, or metabox-file offset when bit 63 is set. */ + erofs_off_t inode_off; + erofs_blk_t startblk; + uint32_t generation; + uint32_t nlink; + uid_t uid; + gid_t gid; + mode_t mode; + __enum_uint8(vtype) vtype; + dev_t rdev; + time_t mtime; + uint32_t mtime_nsec; + uint8_t datalayout; + uint8_t inode_isize; + uint32_t xattr_isize; + bool compact_inode; + bool dot_omitted; + /* Compression fields */ + uint16_t z_advise; + uint8_t z_algorithmtype[2]; + uint8_t z_lclusterbits; + uint16_t z_idata_size; + erofs_off_t z_fragmentoff; + uint64_t z_tailextent_headlcn; + uint64_t z_extents; + bool z_initialized; + /* Chunk-based fields */ + uint16_t chunkformat; + uint8_t chunkbits; + /* Fragment fields */ + bool fragment; + struct erofs_xattr_cache xattr_cache; +}; + +struct erofs_fid { + uint16_t len; + uint16_t pad; + uint32_t nid_hi; + uint32_t nid_lo; + uint32_t gen; +}; + +_Static_assert(sizeof(struct erofs_fid) == 16, + "EROFS file handle ABI must be 16 bytes"); +_Static_assert(sizeof(struct erofs_fid) <= sizeof(struct fid), + "struct erofs_fid must fit within struct fid"); + +#define VTOE(vp) ((struct erofs_inode *)(vp)->v_data) + +static inline unsigned int +erofs_inode_version(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK); +} + +static inline unsigned int +erofs_inode_datalayout(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK); +} + +static inline bool +erofs_nid_in_metabox(erofs_nid_t nid) +{ + return ((nid & EROFS_DIRENT_NID_METABOX) != 0); +} + +/* Allocated on disk at m_pa. */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata. */ +#define EROFS_MAP_META 0x0002 +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +#define EROFS_MAP_PARTIAL_REF 0x0008 +#define EROFS_MAP_FRAGMENT 0x0010 +#define EROFS_MAP_FULL(f) \ + (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) + +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +#define EROFS_GET_BLOCKS_READMORE 0x0002 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +_Static_assert(Z_EROFS_COMPRESSION_RUNTIME_MAX <= UINT8_MAX, + "algorithm format must fit in uint8_t"); + +struct erofs_map_dev { + struct erofs_device_info *m_dif; + + erofs_off_t m_pa; + unsigned int m_deviceid; + uint64_t m_plen; +}; + +struct erofs_buf { + void *data; + void (*release)(void *); +}; + +#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } + +/* Buffer and device I/O. */ +int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp); +int erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp); +void erofs_brelse(void *buf); +void erofs_put_metabuf(struct erofs_buf *buf); +int erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf); +/* Logical mapping and file data. */ +int erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int erofs_read_data_readahead(struct erofs_sb_info *sbi, + struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential, + void **bufp); +int erofs_read_file(struct vnode *vp, struct uio *uio, int ioflag); +int erofs_validate_symlink_target(struct erofs_sb_info *sbi, + struct erofs_inode *vi); +int erofs_readlink_target(struct vnode *vp, struct uio *uio); + +/* Inode and vnode lifecycle. */ +bool erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid); +bool erofs_dirent_type_matches(uint8_t file_type, + __enum_uint8(vtype) vtype); +int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi); +int erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp); +void erofs_xattr_cache_init(struct erofs_inode *vi); +void erofs_xattr_cache_fini(struct erofs_sb_info *sbi, + struct erofs_inode *vi); + +/* Directory operations. */ +int erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies); +int erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp); +int erofs_validate_dirblock(const char *blk, uint32_t blksz, uint32_t maxsize, + uint32_t *ndirentsp); +int erofs_lookup(struct vop_cachedlookup_args *ap); + +/* Compressed mapping and data. */ +int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi); +int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +void z_erofs_extent_cache_init(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_fini(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_init(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_fini(struct erofs_sb_info *sbi); +int z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp); +void z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable); +int z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes); +void *z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address); +int z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio); +int z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src, size_t srclen, + void *dst, size_t dstlen, bool partial); + +/* Compression configuration. */ +int z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb); + +/* VOP vectors. */ +extern struct vop_vector erofs_vnodeops; +extern struct vop_vector erofs_fifoops; + +#endif /* __EROFS_INTERNAL_H */ diff --git a/namei.c b/namei.c new file mode 100644 index 0000000..3079d7e --- /dev/null +++ b/namei.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct erofs_qstr { + const unsigned char *name; + const unsigned char *end; +}; + +/* + * Compare two directory entry names using an already-matched prefix. + * (Linux equivalent: erofs_dirnamecmp in Linux's namei.c) + * + * qn: search key (not necessarily null-terminated). + * qd: on-disk name range (may not be null-terminated). + * matched: in/out count of prefix characters already known to match. + * + * Returns 0 if equal, 1 if qn > qd, -1 if qn < qd. + */ +static int +erofs_dirnamecmp(const struct erofs_qstr *qn, const struct erofs_qstr *qd, + unsigned int *matched) +{ + size_t dname_span, qname_span; + unsigned int i; + + qname_span = qn->end - qn->name; + dname_span = qd->end - qd->name; + i = MIN(*matched, qname_span); + i = MIN(i, dname_span); + while (i < qname_span && i < dname_span && qd->name[i] != '\0') { + if (qn->name[i] != qd->name[i]) { + *matched = i; + return (qn->name[i] > qd->name[i] ? 1 : -1); + } + ++i; + } + *matched = i; + if (i == qname_span) + return (i == dname_span || qd->name[i] == '\0' ? 0 : -1); + return (1); +} + +static int +erofs_dirent_qstr(const char *data, uint32_t datasize, uint32_t index, + uint32_t ndirents, struct erofs_qstr *name) +{ + const struct erofs_dirent *de; + uint32_t endoff, nameoff; + size_t namelen; + int error; + + de = (const struct erofs_dirent *)data; + nameoff = le16toh(de[index].nameoff); + endoff = index + 1 < ndirents ? + le16toh(de[index + 1].nameoff) : datasize; + error = erofs_dirent_namelen(data, nameoff, endoff, + index + 1 == ndirents, &namelen); + if (error != 0) + return (error); + name->name = (const unsigned char *)data + nameoff; + name->end = name->name + namelen; + return (0); +} + +static int +erofs_read_dirblock(struct erofs_sb_info *sbi, struct erofs_inode *dir, + uint64_t block, char **datap, uint32_t *datasizep, uint32_t *ndirentsp) +{ + erofs_off_t block_off; + uint32_t datasize; + char *data; + int error; + + *datap = NULL; + if (__builtin_mul_overflow(block, (uint64_t)sbi->block_size, + &block_off) || block_off >= dir->size) + return (EINTEGRITY); + datasize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, datasize, (void **)&data); + if (error != 0) + return (error); + error = erofs_validate_dirblock(data, sbi->block_size, datasize, + ndirentsp); + if (error != 0) { + erofs_brelse(data); + return (error); + } + *datap = data; + *datasizep = datasize; + return (0); +} + +static int +erofs_dirblock_order(const char *left, uint32_t leftsize, + uint32_t leftents, const char *right, uint32_t rightsize, + uint32_t rightents) +{ + struct erofs_qstr leftname, rightname; + unsigned int matched; + int error; + + error = erofs_dirent_qstr(left, leftsize, leftents - 1, leftents, + &leftname); + if (error != 0) + return (error); + error = erofs_dirent_qstr(right, rightsize, 0, rightents, &rightname); + if (error != 0) + return (error); + matched = 0; + if (erofs_dirnamecmp(&leftname, &rightname, &matched) >= 0) + return (EINTEGRITY); + return (0); +} + +static int +erofs_validate_dirblock_neighbors(struct erofs_sb_info *sbi, + struct erofs_inode *dir, uint64_t block, const char *data, + uint32_t datasize, uint32_t ndirents) +{ + uint64_t lastblock; + uint32_t neighborsize, neighborents; + char *neighbor; + int error; + + lastblock = (dir->size - 1) / sbi->block_size; + if (block > 0) { + error = erofs_read_dirblock(sbi, dir, block - 1, &neighbor, + &neighborsize, &neighborents); + if (error != 0) + return (error); + error = erofs_dirblock_order(neighbor, neighborsize, + neighborents, data, datasize, ndirents); + erofs_brelse(neighbor); + if (error != 0) + return (error); + } + if (block < lastblock) { + error = erofs_read_dirblock(sbi, dir, block + 1, &neighbor, + &neighborsize, &neighborents); + if (error != 0) + return (error); + error = erofs_dirblock_order(data, datasize, ndirents, + neighbor, neighborsize, neighborents); + erofs_brelse(neighbor); + if (error != 0) + return (error); + } + return (0); +} + +/* + * Binary search within a directory block for the target name. + * + * Returns a pointer to the matching dirent, or NULL on miss. + */ +static struct erofs_dirent * +find_target_dirent(const struct erofs_qstr *name, char *data, + uint32_t datasize, uint32_t ndirents) +{ + uint32_t head, back; + unsigned int startprfx, endprfx; + struct erofs_dirent *const de = (struct erofs_dirent *)data; + + /* The 1st dirent has already been evaluated by the caller. */ + head = 1; + back = ndirents - 1; + startprfx = endprfx = 0; + + while (head <= back) { + const uint32_t mid = head + (back - head) / 2; + const uint32_t nameoff = le16toh(de[mid].nameoff); + unsigned int matched = MIN(startprfx, endprfx); + struct erofs_qstr dname = { + .name = (const unsigned char *)data + nameoff, + }; + + if (mid >= ndirents - 1) + dname.end = (const unsigned char *)data + datasize; + else + dname.end = (const unsigned char *)data + + le16toh(de[mid + 1].nameoff); + + /* String comparison without already matched prefix */ + int ret = erofs_dirnamecmp(name, &dname, &matched); + + if (ret == 0) + return (de + mid); + else if (ret > 0) { + head = mid + 1; + startprfx = matched; + } else { + back = mid - 1; + endprfx = matched; + } + } + + return (NULL); +} + +/* + * Find the directory block most likely to contain the target name. + * + * Uses two-level binary search: first across blocks, then within the + * candidate block via find_target_dirent(). + * + * Returns the block buffer on success (caller must erofs_brelse), + * or NULL on error. *_ndirents is set to the number of dirents in + * the returned block (0 means the first entry is the match). + * On error, *errorp is set to a positive errno. + */ +static char * +erofs_find_target_block(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, uint32_t *_ndirents, uint32_t *_datasize, + int *errorp) +{ + uint32_t bsz = sbi->block_size; + uint64_t head, back; + unsigned int startprfx = 0, endprfx = 0; + char *candidate = NULL; + int error; + + *errorp = 0; + *_ndirents = 0; + *_datasize = 0; + + if (dir->size == 0) + return (NULL); + + head = 0; + back = (dir->size - 1) / bsz; + + while (head <= back) { + const uint64_t mid = head + (back - head) / 2; + uint32_t maxsize; + const struct erofs_dirent *de; + char *blk; + int diff; + uint32_t ndirents; + uint32_t nameoff; + unsigned int matched; + struct erofs_qstr dname; + + error = erofs_read_dirblock(sbi, dir, mid, &blk, &maxsize, + &ndirents); + if (error != 0) { + *errorp = error; + goto out; + } + error = erofs_validate_dirblock_neighbors(sbi, dir, mid, blk, + maxsize, ndirents); + if (error != 0) { + erofs_brelse(blk); + *errorp = error; + goto out; + } + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[0].nameoff); + + matched = MIN(startprfx, endprfx); + dname.name = (const unsigned char *)blk + nameoff; + if (ndirents == 1) + dname.end = (const unsigned char *)blk + maxsize; + else + dname.end = (const unsigned char *)blk + + le16toh(de[1].nameoff); + + /* String comparison without already matched prefix */ + diff = erofs_dirnamecmp(name, &dname, &matched); + + if (diff < 0) { + erofs_brelse(blk); + if (mid == 0) + break; + back = mid - 1; + endprfx = matched; + continue; + } + + /* diff >= 0: this block is a candidate. */ + if (candidate != NULL) + erofs_brelse(candidate); + candidate = blk; + if (diff == 0) { + *_ndirents = 0; + *_datasize = maxsize; + return (candidate); + } + head = mid + 1; + startprfx = matched; + *_ndirents = ndirents; + *_datasize = maxsize; + } + return (candidate); +out: + if (candidate != NULL) + erofs_brelse(candidate); + return (NULL); +} + +/* + * Look up a name in a directory and return its nid and d_type. + * (Linux equivalent: erofs_namei in Linux's namei.c) + */ +static int +erofs_namei(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, erofs_nid_t *nid, uint8_t *d_type) +{ + int error; + uint32_t ndirents; + uint32_t datasize; + char *blk; + struct erofs_dirent *de; + + if (dir->size == 0) + return (ENOENT); + + blk = erofs_find_target_block(sbi, dir, name, &ndirents, + &datasize, &error); + if (blk == NULL) + return (error != 0 ? error : ENOENT); + + de = (struct erofs_dirent *)blk; + if (ndirents > 0) + de = find_target_dirent(name, blk, datasize, + ndirents); + + if (de != NULL) { + erofs_nid_t found_nid; + + found_nid = le64toh(de->nid); + if (!erofs_nid_is_valid(sbi, found_nid)) { + error = EINTEGRITY; + } else { + *nid = found_nid; + *d_type = de->file_type; + } + } + erofs_brelse(blk); + if (error != 0) + return (error); + return (de != NULL ? 0 : ENOENT); +} + +/* + * Directory name lookup (VOP_CACHEDLOOKUP entry point). + * + * FreeBSD-side API requirements: + * - "." must be returned under the caller's requested lock mode; + * - ".." must go through vn_vget_ino() to avoid holding a child lock while + * acquiring the parent directory lock in reverse; + * - Both hit and miss must correctly update the namecache. + */ +int +erofs_lookup(struct vop_cachedlookup_args *ap) +{ + struct erofs_qstr qname; + struct vnode *dvp, *vp; + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct componentname *cnp; + erofs_nid_t nid; + uint8_t dtype; + int error, ltype; + + dvp = ap->a_dvp; + cnp = ap->a_cnp; + *ap->a_vpp = NULL; + if ((cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) + return (EROFS); + if (cnp->cn_namelen < 0) + return (EINVAL); + if (cnp->cn_namelen > EROFS_NAME_LEN) + return (ENAMETOOLONG); + if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + + dir = VTOE(dvp); + sbi = MTOE(dvp->v_mount); + qname.name = (const unsigned char *)cnp->cn_nameptr; + qname.end = qname.name + cnp->cn_namelen; + error = erofs_namei(sbi, dir, &qname, &nid, &dtype); + if (error != 0) { + if (error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, NULL, cnp); + if (error == ENOENT && (cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == CREATE || cnp->cn_nameiop == RENAME)) + return (EROFS); + return (error); + } + if ((cnp->cn_flags & ISDOTDOT) == 0 && nid == dir->nid) + return (EINTEGRITY); + + if ((cnp->cn_flags & ISDOTDOT) != 0) + error = vn_vget_ino(dvp, nid, cnp->cn_lkflags, &vp); + else + error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp); + if (error != 0) + return (error); + if ((cnp->cn_flags & ISDOTDOT) == 0 && + !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) { + vput(vp); + return (EINTEGRITY); + } + *ap->a_vpp = vp; + if ((cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, vp, cnp); + return (0); +} diff --git a/super.c b/super.c new file mode 100644 index 0000000..8eaec97 --- /dev/null +++ b/super.c @@ -0,0 +1,1387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "xattr.h" + +MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem"); + +static const uint32_t erofs_crc32c_seed = 0x5045b54aU; + +#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024) +#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024) +#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024) +#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U) +#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U) +#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U) +#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U) + +static unsigned long erofs_stream_mount_budget = + EROFS_STREAM_MOUNT_HARD_BUDGET; +static unsigned long erofs_stream_global_budget = + EROFS_STREAM_GLOBAL_HARD_BUDGET; +static int erofs_stream_mount_contexts = + EROFS_STREAM_MOUNT_HARD_CONTEXTS; +static int erofs_stream_global_contexts_limit = + EROFS_STREAM_GLOBAL_HARD_CONTEXTS; +static int erofs_stream_mount_cached = + EROFS_STREAM_MOUNT_CACHED_PER_CODEC; +static int erofs_stream_global_cached = + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC; + +TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget", + &erofs_stream_mount_budget); +TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget", + &erofs_stream_global_budget); +TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts", + &erofs_stream_mount_contexts); +TUNABLE_INT("vfs.erofs.stream_pool.global_contexts", + &erofs_stream_global_contexts_limit); +TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec", + &erofs_stream_mount_cached); +TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec", + &erofs_stream_global_cached); + +static struct mtx erofs_stream_lock; +static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts = + LIST_HEAD_INITIALIZER(erofs_stream_mounts); +static uma_zone_t erofs_stream_zone; +static eventhandler_tag erofs_stream_lowmem_tag; +static size_t erofs_stream_global_resident; +static unsigned int erofs_stream_global_contexts; +static unsigned int erofs_stream_global_cached_by_codec[ + Z_EROFS_COMPRESSION_MAX]; +static unsigned long erofs_stream_cached_contexts; +static unsigned long erofs_stream_borrowed_contexts; +static unsigned long erofs_stream_idle_contexts; +static unsigned long erofs_stream_creations; +static unsigned long erofs_stream_reuses; +static unsigned long erofs_stream_fallbacks; +static unsigned long erofs_stream_exhaustions; +static unsigned long erofs_stream_allocation_failures; +static unsigned long erofs_stream_destroys; +static unsigned long erofs_stream_reclaims; + +MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool", + MTX_DEF); + +SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "EROFS filesystem"); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool, + CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD, + &erofs_stream_mount_budget, 0, "Effective per-mount byte budget"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD, + &erofs_stream_global_budget, 0, "Effective global byte budget"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD, + &erofs_stream_mount_contexts, 0, "Effective per-mount context limit"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit, + CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0, + "Effective global context limit"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD, + &erofs_stream_global_resident, 0, "Currently charged global bytes"); +SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD, + &erofs_stream_global_contexts, 0, "Currently allocated contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD, + &erofs_stream_cached_contexts, 0, "Currently cacheable contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD, + &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD, + &erofs_stream_idle_contexts, 0, "Currently idle contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD, + &erofs_stream_creations, 0, "Successful context creations"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD, + &erofs_stream_reuses, 0, "Idle context acquisitions"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD, + &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD, + &erofs_stream_exhaustions, 0, "Hard-limit allocation rejections"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, allocation_failures, + CTLFLAG_RD, &erofs_stream_allocation_failures, 0, + "UMA or malloc allocation failures"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD, + &erofs_stream_destroys, 0, "Destroyed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD, + &erofs_stream_reclaims, 0, "Low-memory idle context reclaims"); + +struct erofs_stream_alloc_header { + struct erofs_stream_ctx *ctx; + size_t bytes; +}; + +static bool +erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + if (bytes > erofs_stream_mount_budget || + pool->resident_bytes > erofs_stream_mount_budget - bytes || + bytes > erofs_stream_global_budget || + erofs_stream_global_resident > erofs_stream_global_budget - bytes) + return (false); + pool->resident_bytes += bytes; + erofs_stream_global_resident += bytes; + return (true); +} + +static void +erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + KASSERT(bytes <= sbi->stream_pool.resident_bytes && + bytes <= erofs_stream_global_resident, + ("erofs stream pool byte accounting underflow")); + sbi->stream_pool.resident_bytes -= bytes; + erofs_stream_global_resident -= bytes; +} + +int +z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + bool reserved; + + if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) { + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mtx_lock(&erofs_stream_lock); + reserved = ctx->charged_bytes <= SIZE_MAX - bytes && + erofs_stream_reserve_locked(ctx->sbi, bytes); + if (reserved) + ctx->charged_bytes += bytes; + else + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + if (!reserved) { + ctx->allocation_failed = true; + return (ENOMEM); + } + return (0); +} + +void +z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + + mtx_lock(&erofs_stream_lock); + KASSERT(bytes <= ctx->charged_bytes, + ("erofs stream context byte accounting underflow")); + ctx->charged_bytes -= bytes; + erofs_stream_release_locked(ctx->sbi, bytes); + mtx_unlock(&erofs_stream_lock); +} + +void * +z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_alloc_header *header; + size_t total; + + if (__builtin_add_overflow(bytes, sizeof(*header), &total) || + z_erofs_stream_ctx_charge(ctx, total) != 0) + return (NULL); + header = malloc(total, M_EROFS, M_NOWAIT | M_ZERO); + if (header == NULL) { + z_erofs_stream_ctx_uncharge(ctx, total); + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_allocation_failures; + mtx_unlock(&erofs_stream_lock); + return (NULL); + } + header->ctx = ctx; + header->bytes = total; + return (header + 1); +} + +void +z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address) +{ + struct erofs_stream_alloc_header *header; + size_t bytes; + + if (address == NULL) + return; + header = (struct erofs_stream_alloc_header *)address - 1; + KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch")); + bytes = header->bytes; + free(header, M_EROFS); + z_erofs_stream_ctx_uncharge(ctx, bytes); +} + +static void +erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_sb_info *sbi; + bool cached; + uint8_t algorithm; + + sbi = ctx->sbi; + algorithm = ctx->algorithm; + cached = ctx->cached; + ctx->fini(ctx); + KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE, + ("erofs stream backend allocation leaked")); + uma_zfree(erofs_stream_zone, ctx); + + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + KASSERT(codec->contexts != 0 && codec->borrowed != 0 && + erofs_stream_global_contexts != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream context accounting underflow")); + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + KASSERT(codec->cached != 0 && + erofs_stream_global_cached_by_codec[algorithm] != 0 && + erofs_stream_cached_contexts != 0, + ("erofs stream cache accounting underflow")); + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_destroys; + cv_broadcast(&sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); +} + +int +z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + bool cached; + int error; + + *ctxp = NULL; + if (algorithm == Z_EROFS_COMPRESSION_LZ4 || + algorithm >= Z_EROFS_COMPRESSION_MAX || + context_size < sizeof(*ctx) || + context_size > EROFS_STREAM_CTX_WRAPPER_SIZE) + return (EINVAL); + mtx_lock(&erofs_stream_lock); + if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) { + mtx_unlock(&erofs_stream_lock); + return (ENXIO); + } + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx != NULL) { + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + ++erofs_stream_reuses; + ctx->allocation_failed = false; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); + } + cached = codec->cached < (unsigned int)erofs_stream_mount_cached && + erofs_stream_global_cached_by_codec[algorithm] < + (unsigned int)erofs_stream_global_cached; + if (codec->contexts >= (unsigned int)erofs_stream_mount_contexts || + erofs_stream_global_contexts >= + (unsigned int)erofs_stream_global_contexts_limit || + !erofs_stream_reserve_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE)) { + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + ++codec->contexts; + ++codec->borrowed; + ++erofs_stream_global_contexts; + ++erofs_stream_borrowed_contexts; + if (cached) { + ++codec->cached; + ++erofs_stream_global_cached_by_codec[algorithm]; + ++erofs_stream_cached_contexts; + } else { + ++erofs_stream_fallbacks; + } + mtx_unlock(&erofs_stream_lock); + + ctx = uma_zalloc(erofs_stream_zone, M_NOWAIT | M_ZERO); + if (ctx == NULL) { + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_allocation_failures; + cv_broadcast(&sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + ctx->sbi = sbi; + ctx->fini = fini; + ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE; + ctx->algorithm = algorithm; + ctx->cached = cached; + error = init(ctx); + if (error != 0) { + erofs_stream_ctx_destroy(ctx); + return (error > 0 ? error : EIO); + } + mtx_lock(&erofs_stream_lock); + ++erofs_stream_creations; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); +} + +void +z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable) +{ + struct erofs_stream_pool_codec *codec; + + mtx_lock(&erofs_stream_lock); + codec = &ctx->sbi->stream_pool.codec[ctx->algorithm]; + if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) { + KASSERT(codec->borrowed != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream borrowed accounting underflow")); + --codec->borrowed; + --erofs_stream_borrowed_contexts; + STAILQ_INSERT_HEAD(&codec->idle, ctx, link); + ++codec->idle_count; + ++erofs_stream_idle_contexts; + cv_broadcast(&ctx->sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); + return; + } + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); +} + +static struct erofs_stream_ctx * +erofs_stream_take_idle_locked(struct erofs_sb_info *sbi) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx == NULL) + continue; + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + return (ctx); + } + return (NULL); +} + +static bool +erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool) +{ + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + if (pool->codec[algorithm].contexts != 0) + return (false); + } + return (true); +} + +void +z_erofs_stream_pool_init(struct erofs_sb_info *sbi) +{ + unsigned int algorithm; + + bzero(&sbi->stream_pool, sizeof(sbi->stream_pool)); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) + STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle); + cv_init(&sbi->stream_pool.cv, "erofs stream drain"); + mtx_lock(&erofs_stream_lock); + sbi->stream_pool_initialized = true; + LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link); + mtx_unlock(&erofs_stream_lock); +} + +void +z_erofs_stream_pool_fini(struct erofs_sb_info *sbi) +{ + struct erofs_stream_ctx *ctx; + + if (!sbi->stream_pool_initialized) + return; + mtx_lock(&erofs_stream_lock); + sbi->stream_pool.closing = true; + for (;;) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + } + while (!erofs_stream_pool_empty_locked(&sbi->stream_pool)) + cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock); + KASSERT(sbi->stream_pool.resident_bytes == 0, + ("erofs stream mount bytes remain at unmount")); + LIST_REMOVE(sbi, stream_pool_link); + sbi->stream_pool_initialized = false; + mtx_unlock(&erofs_stream_lock); + cv_destroy(&sbi->stream_pool.cv); +} + +static void +erofs_stream_pool_lowmem(void *arg, int howto) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *sbi; + + (void)arg; + (void)howto; + for (;;) { + ctx = NULL; + mtx_lock(&erofs_stream_lock); + LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL) { + ++erofs_stream_reclaims; + break; + } + } + mtx_unlock(&erofs_stream_lock); + if (ctx == NULL) + break; + erofs_stream_ctx_destroy(ctx); + } +} + +static void +erofs_stream_pool_global_init(void *arg) +{ + + (void)arg; + erofs_stream_mount_budget = MIN(erofs_stream_mount_budget, + EROFS_STREAM_MOUNT_HARD_BUDGET); + erofs_stream_global_budget = MIN(erofs_stream_global_budget, + EROFS_STREAM_GLOBAL_HARD_BUDGET); + erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts, + (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS)); + erofs_stream_global_contexts_limit = MAX(0, MIN( + erofs_stream_global_contexts_limit, + (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS)); + erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached, + (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC)); + erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached, + (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC)); + erofs_stream_zone = uma_zcreate("erofs stream ctx", + EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + KASSERT(erofs_stream_zone != NULL, ("cannot create erofs stream UMA zone")); + (void)uma_zone_set_max(erofs_stream_zone, + EROFS_STREAM_GLOBAL_HARD_CONTEXTS); + uma_zone_set_maxcache(erofs_stream_zone, + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3); + erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +erofs_stream_pool_global_fini(void *arg) +{ + + (void)arg; + if (erofs_stream_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag); + KASSERT(LIST_EMPTY(&erofs_stream_mounts), + ("erofs stream mounts remain at unload")); + KASSERT(erofs_stream_global_contexts == 0 && + erofs_stream_global_resident == 0 && + erofs_stream_borrowed_contexts == 0 && + erofs_stream_idle_contexts == 0 && + erofs_stream_cached_contexts == 0, + ("erofs stream resources remain at unload")); + uma_zdestroy(erofs_stream_zone); +} + +SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY, + erofs_stream_pool_global_init, NULL); +SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY, + erofs_stream_pool_global_fini, NULL); + +static const char *erofs_opts[] = { + "export", + "from", + NULL, +}; + +static vfs_mount_t erofs_mount; +static vfs_root_t erofs_root; +static vfs_statfs_t erofs_statfs; +static vfs_unmount_t erofs_unmount; +static vfs_fhtovp_t erofs_fhtovp; + +#define EROFS_DEVICE_OPT_PREFIX "device." + +struct erofs_device_arg { + uint16_t slot; + char *path; +}; + +static int +erofs_load_generation_seed(struct erofs_sb_info *sbi, uint32_t sb_size, + uint32_t *seedp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t seed; + int error; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sb_size, &buf); + if (error != 0) + return (error); + seed = fnv_32_buf(buf.data, sb_size, FNV1_32_INIT); + erofs_put_metabuf(&buf); + *seedp = seed != 0 ? seed : 1; + return (0); +} + +static void +erofs_free_device_args(struct erofs_device_arg *args, unsigned int count) +{ + unsigned int i; + + if (args == NULL) + return; + for (i = 0; i < count; ++i) + free(args[i].path, M_EROFS); + free(args, M_EROFS); +} + +static int +erofs_parse_device_slot(const char *name, uint16_t *slotp) +{ + const char *p; + unsigned int slot; + + if (strncmp(name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + return (ENOENT); + p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1; + if (*p < '1' || *p > '9') + return (EINVAL); + slot = 0; + for (; *p != '\0'; ++p) { + if (*p < '0' || *p > '9' || slot > (UINT16_MAX - (*p - '0')) / 10) + return (EINVAL); + slot = slot * 10 + (*p - '0'); + } + if (slot == 0 || slot > UINT16_MAX) + return (EINVAL); + *slotp = slot; + return (0); +} + +static int +erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp, + unsigned int *countp) +{ + struct erofs_device_arg *args; + struct vfsopt *opt; + char name[32]; + unsigned int count, i; + uint16_t slot; + int error; + + *argsp = NULL; + *countp = 0; + count = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + if (error != 0 || opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (count == UINT16_MAX) + return (E2BIG); + ++count; + } + if (count == 0) + return (0); + + args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO); + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + KASSERT(error == 0, ("validated EROFS device option changed")); + args[i].slot = slot; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + for (i = 0; i < count; ++i) { + snprintf(name, sizeof(name), EROFS_DEVICE_OPT_PREFIX "%u", + args[i].slot); + vfs_deleteopt(mp->mnt_optnew, name); + } + *argsp = args; + *countp = count; + return (0); +} + +static void +erofs_release_device_info(struct erofs_device_info *dif) +{ + if (dif->cp != NULL) { + g_topology_lock(); + g_vfs_close(dif->cp); + g_topology_unlock(); + dif->cp = NULL; + } + if (dif->devvp != NULL) { + vrele(dif->devvp); + dif->devvp = NULL; + } + if (dif->dev != NULL) { + dev_rel(dif->dev); + dif->dev = NULL; + } +} + +static bool +erofs_provider_is_duplicate(struct erofs_sb_info *sbi, struct g_provider *pp) +{ + unsigned int i; + + if (sbi == NULL) + return (false); + if (sbi->dif0.cp != NULL && sbi->dif0.cp->provider == pp) + return (true); + for (i = 0; i < sbi->extra_devices; ++i) { + if (sbi->devs[i].cp != NULL && sbi->devs[i].cp->provider == pp) + return (true); + } + return (false); +} + +static int +erofs_open_device(struct erofs_sb_info *sbi, const char *path, + struct erofs_device_info *dif) +{ + struct g_provider *pp; + struct nameidata nd; + struct vnode *devvp; + struct cdev *dev; + int error; + + bzero(dif, sizeof(*dif)); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path); + error = namei(&nd); + if (error != 0) + return (error); + devvp = nd.ni_vp; + NDFREE_PNBUF(&nd); + if (!vn_isdisk_error(devvp, &error)) { + vput(devvp); + return (error); + } + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread); + if (error != 0) + error = priv_check(curthread, PRIV_VFS_MOUNT_PERM); + if (error != 0) { + vput(devvp); + return (error); + } + dev = devvp->v_rdev; + dev_ref(dev); + g_topology_lock(); + pp = g_dev_getprovider(dev); + if (pp == NULL) + error = ENXIO; + else if (erofs_provider_is_duplicate(sbi, pp)) + error = EINVAL; + else + error = g_vfs_open(devvp, &dif->cp, "erofs", 0); + if (error == 0) { + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + } + g_topology_unlock(); + VOP_UNLOCK(devvp); + if (error != 0) { + dev_rel(dev); + vrele(devvp); + return (error); + } + dif->devvp = devvp; + dif->dev = dev; + if (dif->sectorsize == 0 || + (dif->sectorsize & (dif->sectorsize - 1)) != 0) { + erofs_release_device_info(dif); + return (EINVAL); + } + return (0); +} + +static void +erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif) +{ + u_long iosize; + + iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ? + dif->dev->si_iosize_max : MAXPHYS; + mp->mnt_iosize_max = MIN(mp->mnt_iosize_max, MIN(iosize, (u_long)MAXPHYS)); +} + +static void +erofs_free_dev_context(struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (sbi->devs != NULL) { + for (i = sbi->extra_devices; i > 0; --i) + erofs_release_device_info(&sbi->devs[i - 1]); + free(sbi->devs, M_EROFS); + } +} + +static void +erofs_drop_internal_inodes(struct erofs_sb_info *sbi) +{ + if (sbi->metabox_en != NULL) + free(sbi->metabox_en, M_EROFS); + if (sbi->packed_inode != NULL) + free(sbi->packed_inode, M_EROFS); +} + +static void +erofs_sb_free(struct erofs_sb_info *sbi) +{ + if (sbi == NULL) + return; + z_erofs_extent_cache_fini(sbi); + z_erofs_stream_pool_fini(sbi); + erofs_xattr_prefixes_cleanup(sbi); + erofs_drop_internal_inodes(sbi); + erofs_free_dev_context(sbi); + erofs_release_device_info(&sbi->dif0); + free(sbi, M_EROFS); +} + +static int +erofs_superblock_csum_verify(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t expected, crc; + size_t len; + int error; + + if (!erofs_sb_has_sb_chksum(sbi)) + return (0); + + len = 1u << dsb->blkszbits; + if (len > EROFS_SUPER_OFFSET) + len -= EROFS_SUPER_OFFSET; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, len, &buf); + if (error != 0) + return (error); + + crc = calculate_crc32c(erofs_crc32c_seed, + (const uint8_t *)buf.data + + offsetof(struct erofs_super_block, checksum) + + sizeof(dsb->checksum), + len - offsetof(struct erofs_super_block, checksum) - + sizeof(dsb->checksum)); + expected = le32toh(dsb->checksum); + erofs_put_metabuf(&buf); + + if (crc != expected) { + vfs_mount_error(sbi->mnt, + "erofs: invalid superblock checksum 0x%08x, " + "0x%08x expected", crc, expected); + return (EINTEGRITY); + } + return (0); +} + +static void +erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat, + uint64_t *blocks, erofs_nid_t *root_nid) +{ + *blocks = le32toh(dsb->blocks_lo); + if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 && + dsb->rootnid_8b != 0) { + *blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32; + *root_nid = le64toh(dsb->rootnid_8b); + } else { + *root_nid = le16toh(dsb->rb.rootnid_2b); + } +} + +static int +erofs_validate_device_size(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_blk_t blocks) +{ + uint64_t bytes; + + if (blocks == 0) + return (EINTEGRITY); + if (sbi->block_size < dif->sectorsize || + sbi->block_size % dif->sectorsize != 0) + return (EINVAL); + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + bytes = blocks << sbi->blkszbits; + if (bytes > dif->mediasize) + return (ENXIO); + return (0); +} + +static int +erofs_init_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + const char *path) +{ + struct erofs_device_info opened; + erofs_blk_t blocks, uniaddr; + int error; + + blocks = dif->blocks; + uniaddr = dif->uniaddr; + error = erofs_open_device(sbi, path, &opened); + if (error != 0) + return (error); + erofs_update_iosize_max(sbi->mnt, &opened); + opened.blocks = blocks; + opened.uniaddr = uniaddr; + *dif = opened; + return (erofs_validate_device_size(sbi, dif, dif->blocks)); +} + +static const char * +erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count, + unsigned int slot) +{ + unsigned int i; + + for (i = 0; i < count; ++i) { + if (args[i].slot == slot) + return (args[i].path); + } + return (NULL); +} + +static int +erofs_scan_devices(struct erofs_sb_info *sbi, const struct erofs_super_block *dsb, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_deviceslot *slots; + struct erofs_device_info *dif; + const char *path; + erofs_off_t devt_off; + uint64_t devt_size, image_size, end, other_end; + erofs_blk_t maxend; + unsigned int i, j, mask; + int error; + + sbi->total_blocks = sbi->dif0.blocks; + sbi->flatdev_blocks = sbi->dif0.blocks; + if (sbi->extra_devices == 0) { + if (arg_count != 0) { + vfs_mount_error(sbi->mnt, + "erofs: external devices given without a device table"); + return (EINVAL); + } + return (0); + } + devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE; + devt_size = (uint64_t)sbi->extra_devices * EROFS_DEVT_SLOT_SIZE; + if (sbi->dif0.blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + image_size = sbi->dif0.blocks << sbi->blkszbits; + if (devt_off > image_size || devt_size > image_size - devt_off || + devt_size > SIZE_MAX) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, 0, devt_off, (size_t)devt_size, &buf); + if (error != 0) + return (error); + slots = buf.data; + sbi->devs = mallocarray(sbi->extra_devices, sizeof(*sbi->devs), M_EROFS, + M_WAITOK | M_ZERO); + maxend = sbi->dif0.blocks; + for (i = 0; i < sbi->extra_devices; ++i) { + dif = &sbi->devs[i]; + dif->blocks = le32toh(slots[i].blocks_lo); + dif->uniaddr = le32toh(slots[i].uniaddr_lo); + if (erofs_sb_has_48bit(sbi)) { + dif->blocks |= (uint64_t)le16toh(slots[i].blocks_hi) << 32; + dif->uniaddr |= (uint64_t)le16toh(slots[i].uniaddr_hi) << 32; + } + if (dif->blocks == 0 || + __builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) { + error = EINTEGRITY; + goto out; + } + if (end > + (erofs_sb_has_48bit(sbi) ? (1ULL << 48) : (1ULL << 32))) { + error = EINTEGRITY; + goto out; + } + if (dif->uniaddr != 0 && dif->uniaddr < sbi->dif0.blocks) { + error = EINTEGRITY; + goto out; + } + for (j = 0; j < i; ++j) { + if (dif->uniaddr == 0 || sbi->devs[j].uniaddr == 0) + continue; + if (__builtin_add_overflow(sbi->devs[j].uniaddr, + sbi->devs[j].blocks, &other_end)) { + error = EINTEGRITY; + goto out; + } + if (dif->uniaddr < other_end && sbi->devs[j].uniaddr < end) { + error = EINTEGRITY; + goto out; + } + } + if (__builtin_add_overflow(sbi->total_blocks, dif->blocks, + &sbi->total_blocks)) { + error = EOVERFLOW; + goto out; + } + maxend = MAX(maxend, (erofs_blk_t)end); + } + erofs_put_metabuf(&buf); + sbi->flatdev_blocks = maxend; + mask = 1; + while (mask < (unsigned int)sbi->extra_devices + 1) + mask <<= 1; + sbi->device_id_mask = mask - 1; + sbi->flatdev = arg_count == 0; + if (sbi->flatdev) + return (erofs_validate_device_size(sbi, &sbi->dif0, + sbi->flatdev_blocks)); + if (arg_count != sbi->extra_devices) { + vfs_mount_error(sbi->mnt, + "erofs: external devices don't match (ondisk %u, given %u)", + sbi->extra_devices, arg_count); + return (arg_count < sbi->extra_devices ? ENXIO : EINVAL); + } + for (i = 0; i < arg_count; ++i) { + if (args[i].slot == 0 || args[i].slot > sbi->extra_devices) + return (EINVAL); + } + for (i = 0; i < sbi->extra_devices; ++i) { + path = erofs_device_arg_path(args, arg_count, i + 1); + if (path == NULL) + return (ENXIO); + error = erofs_init_device(sbi, &sbi->devs[i], path); + if (error != 0) + return (error); + } + return (0); +out: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_init_packed_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* Load the packed carrier before any fragment-backed metabox inode. */ + if (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) { + sbi->packed_inode = malloc(sizeof(*sbi->packed_inode), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->packed_nid, sbi->packed_inode); + if (error != 0) { + free(sbi->packed_inode, M_EROFS); + sbi->packed_inode = NULL; + return (error); + } + if (sbi->packed_inode->vtype != VREG || sbi->packed_inode->fragment) { + vfs_mount_error(sbi->mnt, + "erofs: packed inode nid=%ju is not a non-recursive regular file", + (uintmax_t)sbi->packed_nid); + return (EINTEGRITY); + } + } + return (0); +} + +static int +erofs_init_metabox_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* + * METABOX NIDs address inode slots in this backing inode's data. The + * packed carrier is ready first so a compressed metabox may legally end in + * a fragment pcluster without reading an uninitialized dependency. + */ + if (erofs_sb_has_metabox(sbi)) { + struct erofs_map_blocks map; + + sbi->metabox_en = malloc(sizeof(*sbi->metabox_en), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->metabox_nid, sbi->metabox_en); + if (error != 0) + return (error); + if (sbi->metabox_en->vtype != VREG) { + vfs_mount_error(sbi->mnt, + "erofs: metabox inode nid=%ju is not a regular file", + (uintmax_t)sbi->metabox_nid); + return (EINTEGRITY); + } + if (sbi->metabox_en->fragment) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == sbi->metabox_en->nid || + sbi->metabox_en->size == 0) + return (EINTEGRITY); + map = (struct erofs_map_blocks) { + .m_la = sbi->metabox_en->size - 1, + }; + error = erofs_map_blocks(sbi, sbi->metabox_en, &map); + if (error != 0 || (map.m_flags & EROFS_MAP_FRAGMENT) == 0) + return (error != 0 ? error : EINTEGRITY); + } + } + return (0); +} + +static int +erofs_read_superblock(struct erofs_sb_info *sbi, struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t unsupported; + int error; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sizeof(*dsb), + &buf); + if (error != 0) + return (error); + memcpy(dsb, buf.data, sizeof(*dsb)); + erofs_put_metabuf(&buf); + + if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1) + return (EINVAL); + if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT) + return (EINVAL); + sbi->blkszbits = dsb->blkszbits; + sbi->block_size = 1u << sbi->blkszbits; + sbi->feature_compat = le32toh(dsb->feature_compat); + error = erofs_superblock_csum_verify(sbi, dsb); + if (error != 0) + return (error); + + if (dsb->dirblkbits != 0) + return (EOPNOTSUPP); + sbi->feature_incompat = le32toh(dsb->feature_incompat); + sbi->packed_nid = le64toh(dsb->packed_nid); + sbi->extra_devices = erofs_sb_has_device_table(sbi) ? + le16toh(dsb->extra_devices) : 0; + unsupported = sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT; + if (unsupported != 0) + return (EOPNOTSUPP); + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) + return (EINVAL); + sbi->meta_blkaddr = le32toh(dsb->meta_blkaddr); + sbi->xattr_blkaddr = le32toh(dsb->xattr_blkaddr); + sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start); + sbi->xattr_prefix_count = dsb->xattr_prefix_count; + if (erofs_sb_has_ishare_xattrs(sbi) && + dsb->ishare_xattr_prefix_id >= sbi->xattr_prefix_count) + return (EINTEGRITY); + /* Preserve the raw feature declaration and gate its format at use sites. */ + sbi->xattr_filter_reserved = dsb->xattr_filter_reserved; + erofs_sb_blocks_root(dsb, sbi->feature_incompat, &sbi->blocks, + &sbi->root_nid); + sbi->dif0.blocks = sbi->blocks; + error = erofs_validate_device_size(sbi, &sbi->dif0, sbi->dif0.blocks); + if (error != 0) + return (error); + sbi->inos = le64toh(dsb->inos); + sbi->epoch = (int64_t)le64toh(dsb->epoch); + sbi->fixed_nsec = le32toh(dsb->fixed_nsec); + if (sbi->fixed_nsec >= 1000000000) + return (EINTEGRITY); + error = erofs_load_generation_seed(sbi, sbi->sb_size, + &sbi->generation_seed); + if (error != 0) + return (error); + if (sbi->packed_nid != 0 && erofs_nid_in_metabox(sbi->packed_nid)) + return (EINTEGRITY); + if (erofs_sb_has_metabox(sbi)) { + if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) + return (EINTEGRITY); + sbi->metabox_nid = le64toh(dsb->metabox_nid); + if (erofs_nid_in_metabox(sbi->metabox_nid)) + return (EINTEGRITY); + } + + return (z_erofs_parse_cfgs(sbi, dsb)); +} + +static int +erofs_mountfs(struct erofs_device_info *primary, struct mount *mp, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_sb_info *sbi; + struct erofs_inode root; + struct erofs_super_block dsb; + int error; + + sbi = malloc(sizeof(*sbi), M_EROFS, M_WAITOK | M_ZERO); + sbi->mnt = mp; + z_erofs_extent_cache_init(sbi); + z_erofs_stream_pool_init(sbi); + sbi->dif0 = *primary; + bzero(primary, sizeof(*primary)); + + error = erofs_read_superblock(sbi, &dsb); + if (error != 0) + goto fail; + error = erofs_scan_devices(sbi, &dsb, args, arg_count); + if (error != 0) + goto fail; + + if (erofs_sb_has_shared_ea_in_metabox(sbi) && + !erofs_sb_has_metabox(sbi)) { + error = EINTEGRITY; + goto fail; + } + + error = erofs_init_packed_inode(sbi); + if (error != 0) + goto fail; + error = erofs_init_metabox_inode(sbi); + if (error != 0) + goto fail; + error = erofs_read_inode(sbi, sbi->root_nid, &root); + if (error != 0) + goto fail; + if (root.vtype != VDIR) { + vfs_mount_error(mp, + "erofs: root inode nid=%ju is not a directory", + (uintmax_t)sbi->root_nid); + error = EINTEGRITY; + goto fail; + } + error = erofs_xattr_prefixes_init(sbi); + if (error != 0) + goto fail; + set_opt(&sbi->opt, POSIX_ACL); + memcpy(sbi->volume_name, dsb.volume_name, 16); + sbi->volume_name[16] = '\0'; + + mp->mnt_data = sbi; + mp->mnt_stat.f_fsid.val[0] = dev2udev(sbi->dif0.devvp->v_rdev); + mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum; + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY | MNT_ACLS; + mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED | + MNTK_USES_BCACHE; + MNT_IUNLOCK(mp); + return (0); +fail: + erofs_sb_free(sbi); + return (error); +} + +static int +erofs_mount(struct mount *mp) +{ + struct erofs_device_arg *args; + struct erofs_device_info primary; + char *fspec; + unsigned int arg_count; + int error, len; + + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_RDONLY; + MNT_IUNLOCK(mp); + if (mp->mnt_flag & MNT_UPDATE) { + if (vfs_flagopt(mp->mnt_optnew, "export", NULL, 0)) + return (0); + return (EOPNOTSUPP); + } + args = NULL; + arg_count = 0; + error = erofs_parse_device_options(mp, &args, &arg_count); + if (error != 0) + return (error); + if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) { + error = EINVAL; + goto out_args; + } + fspec = NULL; + error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len); + if (error != 0 || fspec == NULL || len == 0 || + fspec[len - 1] != '\0') { + error = EINVAL; + goto out_args; + } + mp->mnt_iosize_max = MAXPHYS; + error = erofs_open_device(NULL, fspec, &primary); + if (error != 0) + goto out_args; + erofs_update_iosize_max(mp, &primary); + error = erofs_mountfs(&primary, mp, args, arg_count); +out_args: + erofs_free_device_args(args, arg_count); + if (error != 0) + return (error); + vfs_mountedfrom(mp, fspec); + return (erofs_statfs(mp, &mp->mnt_stat)); +} + +static int +erofs_root(struct mount *mp, int flags, struct vnode **vpp) +{ + int error; + + error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp); + if (error != 0) + vfs_mount_error(mp, "erofs: failed to load root nid %ju: error %d", + (uintmax_t)MTOE(mp)->root_nid, error); + return (error); +} + +static int +erofs_statfs(struct mount *mp, struct statfs *sbp) +{ + struct erofs_sb_info *sbi; + + sbi = MTOE(mp); + sbp->f_bsize = sbi->block_size; + sbp->f_iosize = sbi->block_size; + sbp->f_blocks = sbi->total_blocks; + sbp->f_bfree = 0; + sbp->f_bavail = 0; + sbp->f_files = sbi->inos; + sbp->f_ffree = 0; + return (0); +} + +static int +erofs_unmount(struct mount *mp, int mntflags) +{ + struct erofs_sb_info *sbi; + int error, flags; + + flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0; + error = vflush(mp, 0, flags, curthread); + if (error != 0) + return (error); + sbi = MTOE(mp); + mp->mnt_data = NULL; + erofs_sb_free(sbi); + return (0); +} + +/* Persistent EROFS file handle to locked vnode. */ +static int +erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + struct vnode *vp; + erofs_nid_t nid; + int error; + + *vpp = NULL; + bzero(&efid, sizeof(efid)); + memcpy(&efid, fhp, sizeof(efid)); + if (efid.len != sizeof(efid) || efid.pad != 0) + return (EINVAL); + nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo; + if (!erofs_nid_is_valid(MTOE(mp), nid)) + return (ESTALE); + error = VFS_VGET(mp, (ino_t)nid, flags, &vp); + if (error != 0) + return (error); + vi = VTOE(vp); + if (vi->mode == 0 || vi->nlink == 0 || vi->nid != nid || + vi->generation != efid.gen) { + vput(vp); + return (ESTALE); + } + *vpp = vp; + return (0); +} + +static struct vfsops erofs_vfsops = { + .vfs_fhtovp = erofs_fhtovp, + .vfs_mount = erofs_mount, + .vfs_root = erofs_root, + .vfs_statfs = erofs_statfs, + .vfs_unmount = erofs_unmount, + .vfs_vget = erofs_vget, +}; +VFS_SET(erofs_vfsops, erofs, VFCF_READONLY); +MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1); +MODULE_DEPEND(erofs, zlib, 1, 1, 1); +MODULE_VERSION(erofs, 1); diff --git a/xattr.c b/xattr.c new file mode 100644 index 0000000..37aca63 --- /dev/null +++ b/xattr.c @@ -0,0 +1,1272 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021-2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "xattr.h" + +struct posix_acl_xattr_entry { + uint16_t e_tag; + uint16_t e_perm; + uint32_t e_id; +}; + +struct posix_acl_xattr_header { + uint32_t a_version; +}; + +#define POSIX_ACL_XATTR_VERSION 0x0002 +#define EROFS_XATTR_FILTER_POSIX_ACL \ + ((1U << 21) | (1U << 30)) +#define EROFS_XATTR_CACHE_BODY_LIMIT (64UL * 1024) +#define EROFS_XATTR_CACHE_MOUNT_BUDGET (1024UL * 1024) + +struct erofs_xattr_iter { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + int attrnamespace; + const char *name; + size_t name_len; + struct uio *uio; + size_t *sizep; +}; + +static int erofs_xattr_load_body(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep); + +static uint32_t +erofs_xxh32_rotl(uint32_t value, unsigned int count) +{ + return ((value << count) | (value >> (32 - count))); +} + +static uint32_t +erofs_xxh32_round(uint32_t seed, uint32_t input) +{ + seed += input * UINT32_C(2246822519); + seed = erofs_xxh32_rotl(seed, 13); + return (seed * UINT32_C(2654435761)); +} + +static uint32_t +erofs_xxh32(const void *input, size_t length, uint32_t seed) +{ + const uint8_t *cursor, *end; + uint32_t hash; + + cursor = input; + end = cursor + length; + if (length >= 16) { + const uint8_t *limit; + uint32_t v1, v2, v3, v4; + + limit = end - 16; + v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519); + v2 = seed + UINT32_C(2246822519); + v3 = seed; + v4 = seed - UINT32_C(2654435761); + do { + v1 = erofs_xxh32_round(v1, le32dec(cursor)); + cursor += 4; + v2 = erofs_xxh32_round(v2, le32dec(cursor)); + cursor += 4; + v3 = erofs_xxh32_round(v3, le32dec(cursor)); + cursor += 4; + v4 = erofs_xxh32_round(v4, le32dec(cursor)); + cursor += 4; + } while (cursor <= limit); + hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) + + erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18); + } else { + hash = seed + UINT32_C(374761393); + } + hash += (uint32_t)length; + while (cursor + 4 <= end) { + hash += le32dec(cursor) * UINT32_C(3266489917); + hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263); + cursor += 4; + } + while (cursor < end) { + hash += *cursor++ * UINT32_C(374761393); + hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761); + } + hash ^= hash >> 15; + hash *= UINT32_C(2246822519); + hash ^= hash >> 13; + hash *= UINT32_C(3266489917); + hash ^= hash >> 16; + return (hash); +} + +static bool +erofs_xattr_filter_name(int attrnamespace, const char *name, size_t name_len, + uint8_t *indexp, const char **filter_namep, size_t *filter_name_lenp) +{ + static const char acl_access[] = "posix_acl_access"; + static const char acl_default[] = "posix_acl_default"; + static const char security[] = "security."; + static const char trusted[] = "trusted."; + + if (attrnamespace == EXTATTR_NAMESPACE_USER) { + *indexp = EROFS_XATTR_INDEX_USER; + *filter_namep = name; + *filter_name_lenp = name_len; + return (true); + } + if (attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (false); + if (name_len == sizeof(acl_access) - 1 && + memcmp(name, acl_access, sizeof(acl_access) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len == sizeof(acl_default) - 1 && + memcmp(name, acl_default, sizeof(acl_default) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len >= sizeof(trusted) - 1 && + memcmp(name, trusted, sizeof(trusted) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_TRUSTED; + *filter_namep = name + sizeof(trusted) - 1; + *filter_name_lenp = name_len - (sizeof(trusted) - 1); + return (true); + } + if (name_len >= sizeof(security) - 1 && + memcmp(name, security, sizeof(security) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_SECURITY; + *filter_namep = name + sizeof(security) - 1; + *filter_name_lenp = name_len - (sizeof(security) - 1); + return (true); + } + return (false); +} + +static int +erofs_xattr_backing_size(struct erofs_sb_info *sbi, struct erofs_inode *backing_en, + erofs_off_t *sizep) +{ + if (backing_en != NULL) { + *sizep = backing_en->size; + return (0); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + *sizep = sbi->blocks << sbi->blkszbits; + return (0); +} + +static int +erofs_xattr_read_backing(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, erofs_off_t off, size_t len, + struct erofs_buf *buf) +{ + void *data; + erofs_off_t backing_size; + int error; + + error = erofs_xattr_backing_size(sbi, backing_en, &backing_size); + if (error != 0) + return (error); + if (off > backing_size || (uint64_t)len > backing_size - off) + return (EINTEGRITY); + if (backing_en != NULL) + error = erofs_read_data(sbi, backing_en, off, len, &data); + else { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_xattr_filter_negative(struct erofs_sb_info *sbi, struct erofs_inode *vi, + int attrnamespace, const char *name, size_t name_len, bool *negativep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + const char *filter_name; + size_t filter_name_len, header_size; + uint32_t hashbit, name_filter; + uint8_t index; + int error; + + *negativep = false; + if (!erofs_sb_has_xattr_filter_v1(sbi) || + !erofs_xattr_filter_name(attrnamespace, name, name_len, &index, + &filter_name, &filter_name_len)) + return (0); + if (vi->xattr_isize < sizeof(*ih)) + return (EINTEGRITY); + if (vi->xattr_isize == sizeof(*ih)) + return (EOPNOTSUPP); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + name_filter = le32toh(ih->h_name_filter); + hashbit = erofs_xxh32(filter_name, filter_name_len, + EROFS_XATTR_FILTER_SEED + index); + hashbit &= EROFS_XATTR_FILTER_BITS - 1; + *negativep = (name_filter & (1U << hashbit)) != 0; + error = 0; + (void)header_size; + erofs_put_metabuf(&body); + return (error); +} + +/* + * Read one prefix table metadata record. + * + * When backing_en == NULL the record is in the physical metadata area; + * otherwise it lives in the selected metadata carrier's logical data stream. + */ +static int +erofs_xattr_read_metadata(struct erofs_sb_info *sbi, struct erofs_inode *backing_en, + erofs_off_t *offp, struct erofs_buf *buf, size_t *lenp) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + uint16_t raw_len; + erofs_off_t off; + size_t len; + int error; + + if (*offp > UINT64_MAX - (sizeof(struct erofs_xattr_entry) - 1)) + return (EOVERFLOW); + off = roundup2(*offp, sizeof(struct erofs_xattr_entry)); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(raw_len), + &hdrbuf); + if (error != 0) + return (error); + raw_len = le16dec(hdrbuf.data); + erofs_put_metabuf(&hdrbuf); + len = (raw_len == 0) ? (size_t)UINT16_MAX + 1 : raw_len; + if (len < sizeof(struct erofs_xattr_long_prefix) || + len > EROFS_NAME_LEN + sizeof(struct erofs_xattr_long_prefix)) + return (EINTEGRITY); + if (off > UINT64_MAX - sizeof(raw_len)) + return (EOVERFLOW); + error = erofs_xattr_read_backing(sbi, backing_en, + off + sizeof(raw_len), len, buf); + if (error != 0) + return (error); + *offp = off + sizeof(raw_len) + len; + *lenp = len; + return (0); +} + +static int +erofs_xattr_move(void *value, size_t value_size, struct uio *uio, size_t *sizep) +{ + if (sizep != NULL) + *sizep = value_size; + if (uio == NULL || value_size == 0) + return (0); + return (uiomove(value, value_size, uio)); +} + +static int +erofs_xattr_load_body_uncached(struct erofs_sb_info *sbi, + struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + char *body; + uint64_t body_off; + size_t header_size; + int error; + + if (vi->xattr_isize < sizeof(*ih)) + return (EINTEGRITY); + if (vi->inode_off > UINT64_MAX - vi->inode_isize) + return (EINTEGRITY); + body_off = vi->inode_off + vi->inode_isize; + error = erofs_xattr_read_backing(sbi, + erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL, body_off, + vi->xattr_isize, &buf); + if (error != 0) + return (error); + body = buf.data; + ih = (struct erofs_xattr_ibody_header *)body; + if (vi->xattr_isize == sizeof(*ih)) { + error = EOPNOTSUPP; + goto fail; + } + header_size = sizeof(*ih) + sizeof(uint32_t) * ih->h_shared_count; + if (header_size > vi->xattr_isize) { + error = EINTEGRITY; + goto fail; + } + *bodybuf = buf; + *ihp = ih; + *header_sizep = header_size; + return (0); +fail: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_xattr_validate_entry(struct erofs_xattr_entry *entry, size_t remaining, + size_t *entry_sizep, size_t *value_sizep) +{ + size_t entry_size, min_size, value_size; + + if (remaining < sizeof(*entry)) + return (EINTEGRITY); + value_size = le16toh(entry->e_value_size); + min_size = sizeof(*entry) + entry->e_name_len + value_size; + if (min_size > remaining) + return (EINTEGRITY); + entry_size = erofs_xattr_entry_size(entry); + if (entry_size > remaining) + return (EINTEGRITY); + if (memchr(entry->e_name, '\0', entry->e_name_len) != NULL) + return (EINTEGRITY); + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static bool +erofs_xattr_prefix(uint8_t base_index, int *namespacep, + const char **prefixp, size_t *prefix_lenp) +{ + switch (base_index) { + case EROFS_XATTR_INDEX_USER: + *namespacep = EXTATTR_NAMESPACE_USER; + *prefixp = NULL; + *prefix_lenp = 0; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_ACCESS: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_access"; + *prefix_lenp = sizeof("posix_acl_access") - 1; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_default"; + *prefix_lenp = sizeof("posix_acl_default") - 1; + return (true); + case EROFS_XATTR_INDEX_TRUSTED: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "trusted."; + *prefix_lenp = sizeof("trusted.") - 1; + return (true); + case EROFS_XATTR_INDEX_SECURITY: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "security."; + *prefix_lenp = sizeof("security.") - 1; + return (true); + case EROFS_XATTR_INDEX_LUSTRE: + default: + return (false); + } +} + +static int +erofs_xattr_namespace_prefix(int attrnamespace, uint8_t base_index, + const char **prefixp, size_t *prefix_lenp) +{ + int mapped_namespace; + + if (attrnamespace != EXTATTR_NAMESPACE_USER && + attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (EOPNOTSUPP); + if (!erofs_xattr_prefix(base_index, &mapped_namespace, prefixp, + prefix_lenp)) + return (ENOATTR); + if (mapped_namespace != attrnamespace) + return (ENOATTR); + return (0); +} + +static int +erofs_xattr_list_move(const char *namespace_prefix, size_t namespace_prefix_len, + const char *infix, size_t infix_len, const char *name, uint8_t name_len, + struct uio *uio, size_t *sizep) +{ + uint8_t total_name_len; + int error; + + if (namespace_prefix_len + infix_len + name_len > EROFS_NAME_LEN) + return (EINTEGRITY); + total_name_len = namespace_prefix_len + infix_len + name_len; + if (sizep != NULL) { + *sizep += total_name_len + 1; + return (0); + } + if (uio == NULL) + return (0); + error = uiomove(__DECONST(void *, &total_name_len), 1, uio); + if (error != 0) + return (error); + if (namespace_prefix_len != 0) { + error = uiomove(__DECONST(void *, namespace_prefix), + namespace_prefix_len, uio); + if (error != 0) + return (error); + } + if (infix_len != 0) { + error = uiomove(__DECONST(void *, infix), infix_len, uio); + if (error != 0) + return (error); + } + return (uiomove(__DECONST(void *, name), name_len, uio)); +} + +static int +erofs_xattr_resolve_name(struct erofs_sb_info *sbi, + const struct erofs_xattr_entry *entry, uint8_t *base_indexp, + const char **infixp, size_t *infix_lenp) +{ + struct erofs_xattr_prefix_item *prefix; + uint8_t prefix_id; + + if ((entry->e_name_index & EROFS_XATTR_LONG_PREFIX) == 0) { + *base_indexp = entry->e_name_index; + *infixp = NULL; + *infix_lenp = 0; + return (0); + } + if (sbi->xattr_prefixes == NULL) + return (ENOATTR); + prefix_id = entry->e_name_index & EROFS_XATTR_LONG_PREFIX_MASK; + if (prefix_id >= sbi->xattr_prefix_count) + return (ENOATTR); + prefix = &sbi->xattr_prefixes[prefix_id]; + *base_indexp = prefix->base_index; + *infixp = prefix->infix; + *infix_lenp = prefix->infix_len; + return (0); +} + +static bool +erofs_xattr_name_match(const char *namespace_prefix, + size_t namespace_prefix_len, const char *infix, size_t infix_len, + const struct erofs_xattr_entry *entry, const char *name, size_t name_len) +{ + if (name_len != namespace_prefix_len + infix_len + entry->e_name_len) + return (false); + if (namespace_prefix_len != 0 && + memcmp(name, namespace_prefix, namespace_prefix_len) != 0) + return (false); + if (infix_len != 0 && + memcmp(name + namespace_prefix_len, infix, infix_len) != 0) + return (false); + return (memcmp(name + namespace_prefix_len + infix_len, entry->e_name, + entry->e_name_len) == 0); +} + +static int +erofs_xattr_shared_entry_offset(struct erofs_sb_info *sbi, uint32_t shared_id, + erofs_off_t *phys_offp) +{ + erofs_off_t base, relative; + + if (sbi->xattr_blkaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + base = (uint64_t)sbi->xattr_blkaddr << sbi->blkszbits; + relative = (uint64_t)shared_id * sizeof(uint32_t); + if (relative > UINT64_MAX - base) + return (EOVERFLOW); + *phys_offp = base + relative; + return (0); +} + +static int +erofs_xattr_load_shared_entry(struct erofs_sb_info *sbi, uint32_t shared_id, + struct erofs_buf *entrybuf, size_t *entry_sizep, size_t *value_sizep) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + struct erofs_inode *backing_en; + erofs_off_t off; + size_t entry_size, value_size; + int error; + + backing_en = erofs_sb_has_shared_ea_in_metabox(sbi) ? sbi->metabox_en : NULL; + if (erofs_sb_has_shared_ea_in_metabox(sbi) && backing_en == NULL) + return (EINTEGRITY); + + error = erofs_xattr_shared_entry_offset(sbi, shared_id, &off); + if (error != 0) + return (error); + + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(*entry), + &hdrbuf); + if (error != 0) + return (error); + entry = hdrbuf.data; + value_size = le16toh(entry->e_value_size); + entry_size = erofs_xattr_entry_size(entry); + + erofs_put_metabuf(&hdrbuf); + + error = erofs_xattr_read_backing(sbi, backing_en, off, entry_size, + entrybuf); + if (error != 0) + return (error); + error = erofs_xattr_validate_entry(entrybuf->data, entry_size, NULL, + value_sizep); + if (error != 0) { + erofs_put_metabuf(entrybuf); + return (error); + } + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static int +erofs_xattr_validate_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + void *body, struct erofs_xattr_ibody_header *ih, size_t header_size) +{ + struct erofs_buf entrybuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + char *cursor; + uint32_t shared_id; + size_t entry_size, remaining; + int error; + + if (header_size > vi->xattr_isize || + memchr(ih->h_reserved2, '\0', sizeof(ih->h_reserved2)) == NULL) + return (EINTEGRITY); + remaining = vi->xattr_isize - header_size; + cursor = (char *)body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + error = erofs_xattr_validate_entry(entry, remaining, &entry_size, + NULL); + if (error != 0) + return (error); + cursor += entry_size; + remaining -= entry_size; + } + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + error = erofs_xattr_load_shared_entry(sbi, shared_id, &entrybuf, + NULL, NULL); + if (error != 0) + return (error); + erofs_put_metabuf(&entrybuf); + } + return (0); +} + +static int +erofs_xattr_load_body_checked(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep) +{ + int error; + + error = erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep); + if (error != 0) + return (error); + error = erofs_xattr_validate_body(sbi, vi, bodybuf->data, *ihp, + *header_sizep); + if (error != 0) + erofs_put_metabuf(bodybuf); + return (error); +} + +static bool +erofs_xattr_cache_reserve(struct erofs_sb_info *sbi, size_t size) +{ + u_long resident; + + for (;;) { + resident = atomic_load_acq_long(&sbi->xattr_cache_resident); + if (resident > EROFS_XATTR_CACHE_MOUNT_BUDGET || + size > EROFS_XATTR_CACHE_MOUNT_BUDGET - resident) + return (false); + if (atomic_cmpset_acq_long(&sbi->xattr_cache_resident, resident, + resident + size)) + return (true); + } +} + +static void +erofs_xattr_cache_release(struct erofs_sb_info *sbi, size_t size) +{ + KASSERT(size <= EROFS_XATTR_CACHE_MOUNT_BUDGET, + ("erofs xattr cache release exceeds mount budget")); + atomic_subtract_rel_long(&sbi->xattr_cache_resident, size); +} + +static void * +erofs_xattr_cache_invalidate_locked(struct erofs_sb_info *sbi, + struct erofs_xattr_cache *cache) +{ + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + erofs_xattr_cache_release(sbi, cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + cache->size = 0; + cache->error = 0; + cache->state = EROFS_XATTR_CACHE_EMPTY; + return (data); +} + +void +erofs_xattr_cache_init(struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + + cache = &vi->xattr_cache; + bzero(cache, sizeof(*cache)); + mtx_init(&cache->lock, "erofs xattr", NULL, MTX_DEF); + cv_init(&cache->cv, "erofs xattr"); + cache->state = EROFS_XATTR_CACHE_EMPTY; + cache->initialized = true; +} + +void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + void *data; + + cache = &vi->xattr_cache; + if (!cache->initialized) + return; + mtx_lock(&cache->lock); + cache->closing = true; + while (cache->state == EROFS_XATTR_CACHE_INFLIGHT || + cache->waiters != 0) + cv_wait(&cache->cv, &cache->lock); + data = erofs_xattr_cache_invalidate_locked(sbi, cache); + cache->initialized = false; + mtx_unlock(&cache->lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&cache->lock); +} + +static int +erofs_xattr_load_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_xattr_cache *cache; + bool reserved; + int error; + + cache = &vi->xattr_cache; + if (!cache->initialized || vi->xattr_isize > EROFS_XATTR_CACHE_BODY_LIMIT) + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + mtx_lock(&cache->lock); + if (cache->closing) { + mtx_unlock(&cache->lock); + return (ENXIO); + } + if (cache->state == EROFS_XATTR_CACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs xattr ready cache has no body")); + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + mtx_unlock(&cache->lock); + return (0); + } + if (cache->state == EROFS_XATTR_CACHE_INFLIGHT) { + cache->waiters++; + do { + cv_wait(&cache->cv, &cache->lock); + } while (cache->state == EROFS_XATTR_CACHE_INFLIGHT); + if (cache->state == EROFS_XATTR_CACHE_READY) { + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + error = 0; + } else { + KASSERT(cache->state == EROFS_XATTR_CACHE_FAILED && + cache->error > 0, + ("erofs xattr failed cache has no typed error")); + error = cache->error; + } + cache->waiters--; + if (cache->state == EROFS_XATTR_CACHE_FAILED && + cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (error); + } + if (cache->state == EROFS_XATTR_CACHE_FAILED) { + if (cache->waiters != 0) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + } + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + } + reserved = erofs_xattr_cache_reserve(sbi, vi->xattr_isize); + if (!reserved) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + } + cache->state = EROFS_XATTR_CACHE_INFLIGHT; + cache->error = 0; + mtx_unlock(&cache->lock); + + error = erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep); + mtx_lock(&cache->lock); + if (error == 0 && !cache->closing) { + cache->data = bodybuf->data; + cache->size = vi->xattr_isize; + cache->charged_bytes = vi->xattr_isize; + cache->state = EROFS_XATTR_CACHE_READY; + bodybuf->release = NULL; + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (0); + } + if (error == 0) + error = ENXIO; + erofs_xattr_cache_release(sbi, vi->xattr_isize); + cache->error = error; + cache->state = EROFS_XATTR_CACHE_FAILED; + if (cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + erofs_put_metabuf(bodybuf); + return (error); +} + +static int +erofs_listxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error == ENOATTR) + return (0); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error == ENOATTR) + return (0); + if (error != 0) + return (error); + return (erofs_xattr_list_move(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry->e_name, entry->e_name_len, it->uio, + it->sizep)); +} + +static int +erofs_getxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry, size_t value_size) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error != 0) + return (error); + if (!erofs_xattr_name_match(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry, it->name, it->name_len)) + return (ENOATTR); + return (erofs_xattr_move(entry->e_name + entry->e_name_len, value_size, + it->uio, it->sizep)); +} + +static int +erofs_xattr_iter_inline(struct erofs_xattr_iter *it, char *body, + size_t header_size, bool get) +{ + struct erofs_xattr_entry *entry; + char *cursor; + size_t entry_size, remaining, value_size; + int error; + + remaining = it->vi->xattr_isize - header_size; + cursor = body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + error = erofs_xattr_validate_entry(entry, remaining, + &entry_size, get ? &value_size : NULL); + if (error != 0) + return (error); + if (get) + error = erofs_getxattr_foreach(it, entry, value_size); + else + error = erofs_listxattr_foreach(it, entry); + if (get) { + if (error != ENOATTR) + return (error); + } else if (error != 0) { + return (error); + } + cursor += entry_size; + remaining -= entry_size; + } + return (get ? ENOATTR : 0); +} + +static int +erofs_xattr_iter_shared(struct erofs_xattr_iter *it, + struct erofs_xattr_ibody_header *ih, bool get) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + uint32_t shared_id; + size_t value_size; + int error; + + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + error = erofs_xattr_load_shared_entry(it->sbi, shared_id, &buf, + NULL, get ? &value_size : NULL); + if (error != 0) + return (error); + entry = buf.data; + if (get) + error = erofs_getxattr_foreach(it, entry, value_size); + else + error = erofs_listxattr_foreach(it, entry); + erofs_put_metabuf(&buf); + if (get) { + if (error != ENOATTR) + return (error); + } else if (error != 0) { + return (error); + } + } + return (get ? ENOATTR : 0); +} + +/* + * Look up one inline/shared xattr by name. + * + * Name exposure rules: + * - user namespace: bare name, no "user." prefix; + * - system namespace: exposes full "trusted.*" / "security.*" names. + */ +int +erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size, name_len; + bool filter_negative; + int error; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (name == NULL || name[0] == '\0') + return (EINVAL); + name_len = strlen(name); + if (name_len > EROFS_NAME_LEN) + return (EINVAL); + if (vi->xattr_isize == 0) + return (ENOATTR); + error = erofs_xattr_filter_negative(sbi, vi, attrnamespace, name, name_len, + &filter_negative); + if (error != 0) + return (error); + if (filter_negative) + return (ENOATTR); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = name; + it.name_len = name_len; + it.uio = uio; + it.sizep = sizep; + error = erofs_xattr_iter_inline(&it, body.data, header_size, true); + if (error == ENOATTR) + error = erofs_xattr_iter_shared(&it, ih, true); + erofs_put_metabuf(&body); + return (error); +} + +/* + * Enumerate inline/shared xattr names for a given namespace. + * + * Return format: 1-byte name length followed by non-NUL-terminated name bytes. + */ +int +erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size; + int error; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (sizep != NULL) + *sizep = 0; + if (vi->xattr_isize == 0) + return (0); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = NULL; + it.name_len = 0; + it.uio = uio; + it.sizep = sizep; + error = erofs_xattr_iter_inline(&it, body.data, header_size, false); + if (error == 0) + error = erofs_xattr_iter_shared(&it, ih, false); + erofs_put_metabuf(&body); + return (error); +} + +void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi) +{ + if (sbi->xattr_prefixes == NULL) + return; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) + free(sbi->xattr_prefixes[i].infix, M_EROFS); + free(sbi->xattr_prefixes, M_EROFS); + sbi->xattr_prefixes = NULL; +} + +int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_long_prefix *prefix = NULL; + struct erofs_inode packed_en, *prefix_en; + erofs_off_t off; + size_t infix_len, len; + int error; + + if (!erofs_sb_has_xattr_prefixes(sbi) || sbi->xattr_prefix_count == 0) + return (0); + prefix_en = NULL; + if (!erofs_sb_has_plain_xattr_pfx(sbi)) { + if (erofs_sb_has_metabox(sbi)) { + if (sbi->metabox_en == NULL) + return (EINTEGRITY); + prefix_en = sbi->metabox_en; + } else if (sbi->packed_inode != NULL) { + prefix_en = sbi->packed_inode; + } else if (sbi->packed_nid != 0) { + error = erofs_read_inode(sbi, sbi->packed_nid, &packed_en); + if (error != 0) + return (error); + if (packed_en.vtype != VREG) + return (EINTEGRITY); + prefix_en = &packed_en; + } + } + sbi->xattr_prefixes = malloc(sizeof(*sbi->xattr_prefixes) * + sbi->xattr_prefix_count, + M_EROFS, M_WAITOK | M_ZERO); + off = (uint64_t)sbi->xattr_prefix_start << 2; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) { + error = erofs_xattr_read_metadata(sbi, prefix_en, &off, + &buf, &len); + if (error != 0) + goto fail; + prefix = buf.data; + infix_len = len - sizeof(*prefix); + if (memchr(prefix->infix, '\0', infix_len) != NULL) { + error = EINTEGRITY; + goto fail; + } + sbi->xattr_prefixes[i].base_index = prefix->base_index; + sbi->xattr_prefixes[i].infix_len = infix_len; + sbi->xattr_prefixes[i].infix = malloc(infix_len + 1, M_EROFS, + M_WAITOK); + memcpy(sbi->xattr_prefixes[i].infix, prefix->infix, infix_len); + sbi->xattr_prefixes[i].infix[infix_len] = '\0'; + erofs_put_metabuf(&buf); + } + return (0); +fail: + erofs_put_metabuf(&buf); + erofs_xattr_prefixes_cleanup(sbi); + return (error); +} + +static int +erofs_inode_has_noacl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + bool *noaclp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + struct erofs_inode *backing_en; + erofs_off_t body_off; + uint32_t name_filter; + int error; + + *noaclp = false; + if (vi->xattr_isize < sizeof(*ih)) { + *noaclp = true; + return (0); + } + if (!erofs_sb_has_xattr_filter(sbi) || + sbi->xattr_filter_reserved != 0) + return (0); + if (vi->inode_off > UINT64_MAX - vi->inode_isize) + return (EINTEGRITY); + body_off = vi->inode_off + vi->inode_isize; + backing_en = erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL; + error = erofs_xattr_read_backing(sbi, backing_en, body_off, sizeof(*ih), + &buf); + if (error != 0) + return (error); + ih = buf.data; + name_filter = le32toh(ih->h_name_filter); + erofs_put_metabuf(&buf); + *noaclp = (name_filter & EROFS_XATTR_FILTER_POSIX_ACL) == + EROFS_XATTR_FILTER_POSIX_ACL; + return (0); +} + +static void +erofs_acl_from_mode(mode_t mode, acl_type_t type, struct acl *aclp) +{ + if (type == ACL_TYPE_DEFAULT) { + aclp->acl_cnt = 0; + return; + } + aclp->acl_cnt = 3; + aclp->acl_entry[0].ae_tag = ACL_USER_OBJ; + aclp->acl_entry[0].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[0].ae_perm = acl_posix1e_mode_to_perm(ACL_USER_OBJ, mode); + aclp->acl_entry[1].ae_tag = ACL_GROUP_OBJ; + aclp->acl_entry[1].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[1].ae_perm = acl_posix1e_mode_to_perm(ACL_GROUP_OBJ, mode); + aclp->acl_entry[2].ae_tag = ACL_OTHER; + aclp->acl_entry[2].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[2].ae_perm = acl_posix1e_mode_to_perm(ACL_OTHER, mode); +} + +static int +erofs_posix_acl_from_xattr(const void *value, size_t size, mode_t mode, + acl_type_t type, struct acl *aclp) +{ + const uint8_t *buf; + struct posix_acl_xattr_header hdr; + struct posix_acl_xattr_entry entry; + uint32_t id; + int count, i, j, phase; + + buf = value; + if (size < sizeof(hdr) || (size - sizeof(hdr)) % sizeof(entry) != 0) + return (EINTEGRITY); + + memcpy(&hdr, buf, sizeof(hdr)); + if (le32toh(hdr.a_version) != POSIX_ACL_XATTR_VERSION) + return (EINTEGRITY); + + count = (size - sizeof(hdr)) / sizeof(entry); + if (count > ACL_MAX_ENTRIES) + return (EINTEGRITY); + if (count == 0) { + erofs_acl_from_mode(mode, type, aclp); + return (0); + } + + aclp->acl_cnt = count; + phase = 0; + for (i = 0; i < count; i++) { + uint16_t tag, perm; + + memcpy(&entry, buf + sizeof(hdr) + i * sizeof(entry), + sizeof(entry)); + tag = le16toh(entry.e_tag); + perm = le16toh(entry.e_perm); + + id = le32toh(entry.e_id); + if ((perm & ~ACL_PERM_BITS) != 0) + return (EINTEGRITY); + switch (tag) { + case ACL_USER_OBJ: + if (phase != 0 || id != UINT32_MAX) + return (EINTEGRITY); + phase = 1; + break; + case ACL_USER: + if ((phase != 1 && phase != 2) || id == UINT32_MAX) + return (EINTEGRITY); + phase = 2; + break; + case ACL_GROUP_OBJ: + if ((phase != 1 && phase != 2) || id != UINT32_MAX) + return (EINTEGRITY); + phase = 3; + break; + case ACL_GROUP: + if ((phase != 3 && phase != 4) || id == UINT32_MAX) + return (EINTEGRITY); + phase = 4; + break; + case ACL_MASK: + if ((phase != 3 && phase != 4) || id != UINT32_MAX) + return (EINTEGRITY); + phase = 5; + break; + case ACL_OTHER: + if ((phase != 3 && phase != 4 && phase != 5) || + id != UINT32_MAX) + return (EINTEGRITY); + phase = 6; + break; + default: + return (EINTEGRITY); + } + if (tag == ACL_USER || tag == ACL_GROUP) { + for (j = 0; j < i; j++) { + if (aclp->acl_entry[j].ae_tag == tag && + aclp->acl_entry[j].ae_id == id) + return (EINTEGRITY); + } + } + + aclp->acl_entry[i].ae_tag = tag; + aclp->acl_entry[i].ae_perm = perm; + aclp->acl_entry[i].ae_id = (id == UINT32_MAX) ? ACL_UNDEFINED_ID : id; + } + if (phase != 6 || acl_posix1e_check(aclp) != 0) + return (EINTEGRITY); + + return (0); +} + +int +erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + const char *xattr_name; + struct uio auio; + struct iovec aiov; + uint8_t buf[sizeof(struct posix_acl_xattr_header) + + sizeof(struct posix_acl_xattr_entry) * ACL_MAX_ENTRIES]; + size_t size; + bool noacl; + int error; + + sbi = MTOE(vp->v_mount); + if (!test_opt(&sbi->opt, POSIX_ACL)) + return (EOPNOTSUPP); + + vi = VTOE(vp); + + switch (type) { + case ACL_TYPE_ACCESS: + xattr_name = "posix_acl_access"; + break; + case ACL_TYPE_DEFAULT: + if (vp->v_type != VDIR) + return (EINVAL); + xattr_name = "posix_acl_default"; + break; + default: + return (EINVAL); + } + error = erofs_inode_has_noacl(sbi, vi, &noacl); + if (error != 0) + return (error); + if (noacl) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, NULL, + &size); + if (error == ENOATTR) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + if (error != 0) + return (error); + if (size > sizeof(buf)) + return (EINTEGRITY); + + aiov.iov_base = buf; + aiov.iov_len = size; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_offset = 0; + auio.uio_resid = size; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_rw = UIO_READ; + auio.uio_td = curthread; + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, &auio, + NULL); + if (error != 0) + return (error); + if (auio.uio_resid != 0) + return (EINTEGRITY); + + return (erofs_posix_acl_from_xattr(buf, size, vi->mode, type, aclp)); +} diff --git a/xattr.h b/xattr.h new file mode 100644 index 0000000..f60090a --- /dev/null +++ b/xattr.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_XATTR_H +#define __EROFS_XATTR_H + +#include "internal.h" + +int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi); +void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi); +int erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep); +int erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep); +int erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp); +#endif diff --git a/zdata.c b/zdata.c new file mode 100644 index 0000000..366ae0f --- /dev/null +++ b/zdata.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" + +enum z_erofs_cache_lookup { + Z_EROFS_CACHE_BYPASS, + Z_EROFS_CACHE_HIT, + Z_EROFS_CACHE_OWNER, + Z_EROFS_CACHE_ERROR, +}; + +#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024) +#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024) +#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024) + +static int z_erofs_cache_enabled = 1; +static unsigned long z_erofs_cache_mount_budget = + EROFS_ZCACHE_MOUNT_HARD_BUDGET; +static unsigned long z_erofs_cache_global_budget = + EROFS_ZCACHE_GLOBAL_HARD_BUDGET; +static unsigned long z_erofs_cache_minimum_decode_work = + EROFS_ZCACHE_MIN_DECODE_WORK; + +TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled); +TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget", + &z_erofs_cache_mount_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget", + &z_erofs_cache_global_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work", + &z_erofs_cache_minimum_decode_work); + +static struct mtx z_erofs_cache_list_lock; +static struct mtx z_erofs_cache_budget_lock; +static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts = + LIST_HEAD_INITIALIZER(z_erofs_cache_mounts); +static size_t z_erofs_cache_global_charged; +static eventhandler_tag z_erofs_cache_lowmem_tag; + +MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock, + "erofs zcache list", MTX_DEF); +MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock, + "erofs zcache budget", MTX_DEF); + +static size_t +z_erofs_extent_cache_global_limit(void) +{ + + return (MIN(z_erofs_cache_global_budget, + EROFS_ZCACHE_GLOBAL_HARD_BUDGET)); +} + +static bool +z_erofs_extent_cache_reserve(size_t bytes) +{ + size_t limit; + bool reserved; + + reserved = false; + mtx_lock(&z_erofs_cache_budget_lock); + limit = z_erofs_extent_cache_global_limit(); + if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) { + z_erofs_cache_global_charged += bytes; + reserved = true; + } + mtx_unlock(&z_erofs_cache_budget_lock); + return (reserved); +} + +static void +z_erofs_extent_cache_release(size_t bytes) +{ + + mtx_lock(&z_erofs_cache_budget_lock); + KASSERT(bytes <= z_erofs_cache_global_charged, + ("erofs decoded cache budget underflow")); + z_erofs_cache_global_charged -= bytes; + mtx_unlock(&z_erofs_cache_budget_lock); +} + +static void * +z_erofs_extent_cache_drop_locked(struct erofs_zextent_cache *cache, + bool evicted, bool reclaimed) +{ + struct erofs_zextent_cache_metrics *metrics; + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + KASSERT(cache->map.m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache charged unknown codec")); + metrics = &cache->metrics[cache->map.m_algorithmformat]; + if (cache->state == EROFS_ZCACHE_READY) { + KASSERT(metrics->resident_bytes == cache->charged_bytes, + ("erofs decoded cache codec accounting mismatch")); + metrics->resident_bytes = 0; + if (evicted) + ++metrics->evictions; + if (reclaimed) + ++metrics->reclaims; + } + z_erofs_extent_cache_release(cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + return (data); +} + +static bool +z_erofs_extent_cache_match(const struct erofs_zextent_cache *cache, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size) +{ + + return (cache->state != EROFS_ZCACHE_EMPTY && cache->nid == vi->nid && + cache->decoded_size == decoded_size && + cache->map.m_pa == map->m_pa && cache->map.m_la == map->m_la && + cache->map.m_plen == map->m_plen && + cache->map.m_llen == map->m_llen && + cache->map.m_deviceid == map->m_deviceid && + cache->map.m_algorithmformat == map->m_algorithmformat && + cache->map.m_flags == map->m_flags); +} + +static enum z_erofs_cache_lookup +z_erofs_extent_cache_claim(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + void *old; + enum z_erofs_cache_lookup result; + + KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS")); + KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff, + ("erofs extent cache copy exceeds decoded extent")); + *errorp = 0; + if (!sbi->z_extent_cache_initialized) + return (Z_EROFS_CACHE_BYPASS); + cache = &sbi->z_extent_cache; + KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache claim has unknown codec")); + old = NULL; + mtx_lock(&sbi->z_extent_cache_lock); + metrics = &cache->metrics[map->m_algorithmformat]; + if (cache->closing) { + ++metrics->bypasses; + goto bypass; + } + if (z_erofs_extent_cache_match(cache, vi, map, decoded_size)) { + switch (cache->state) { + case EROFS_ZCACHE_READY: + KASSERT(cache->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)cache->data + (size_t)mapoff, len); + ++metrics->hits; + mtx_unlock(&sbi->z_extent_cache_lock); + return (Z_EROFS_CACHE_HIT); + case EROFS_ZCACHE_INFLIGHT: + ++cache->waiters; + do { + cv_wait(&cache->cv, &sbi->z_extent_cache_lock); + } while (cache->state == EROFS_ZCACHE_INFLIGHT); + KASSERT(z_erofs_extent_cache_match(cache, vi, map, + decoded_size), ("erofs inflight cache key changed")); + if (cache->state == EROFS_ZCACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)cache->data + (size_t)mapoff, + len); + ++metrics->hits; + result = Z_EROFS_CACHE_HIT; + } else { + KASSERT(cache->state == EROFS_ZCACHE_FAILED && + cache->error > 0, + ("erofs inflight cache has no typed result")); + *errorp = cache->error; + result = Z_EROFS_CACHE_ERROR; + } + --cache->waiters; + if (cache->state == EROFS_ZCACHE_FAILED && + cache->waiters == 0) { + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + } + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&sbi->z_extent_cache_lock); + return (result); + case EROFS_ZCACHE_FAILED: + if (cache->waiters != 0) + goto bypass; + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + break; + case EROFS_ZCACHE_EMPTY: + break; + } + } + if (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) { + ++metrics->bypasses; + goto bypass; + } + if (cache->state == EROFS_ZCACHE_READY) { + old = z_erofs_extent_cache_drop_locked(cache, true, false); + cache->state = EROFS_ZCACHE_EMPTY; + } + if (!z_erofs_extent_cache_reserve(decoded_size)) { + ++metrics->bypasses; + goto bypass; + } + cache->map = *map; + cache->nid = vi->nid; + cache->decoded_size = decoded_size; + cache->charged_bytes = decoded_size; + cache->error = 0; + cache->state = EROFS_ZCACHE_INFLIGHT; + ++metrics->misses; + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_OWNER); + +bypass: + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_BYPASS); +} + +static void +z_erofs_extent_cache_complete(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, void *decoded, int error) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + + KASSERT((error == 0) == (decoded != NULL), + ("erofs extent cache completion is untyped")); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + KASSERT(cache->state == EROFS_ZCACHE_INFLIGHT && + z_erofs_extent_cache_match(cache, vi, map, decoded_size), + ("erofs extent cache owner lost its key")); + metrics = &cache->metrics[map->m_algorithmformat]; + if (error == 0) { + KASSERT(cache->charged_bytes == decoded_size && + metrics->resident_bytes == 0, + ("erofs decoded cache success accounting mismatch")); + cache->data = decoded; + metrics->resident_bytes = decoded_size; + cache->error = 0; + cache->state = EROFS_ZCACHE_READY; + } else { + KASSERT(error > 0, ("erofs extent cache published negative errno")); + z_erofs_extent_cache_release(cache->charged_bytes); + cache->charged_bytes = 0; + cache->error = error; + cache->state = EROFS_ZCACHE_FAILED; + if (cache->waiters == 0) { + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + } + } + cv_broadcast(&cache->cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_extent_cache_lowmem(void *arg, int howto) +{ + struct erofs_zextent_cache *cache; + struct erofs_sb_info *sbi; + void *data; + + (void)arg; + (void)howto; + mtx_lock(&z_erofs_cache_list_lock); + LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) { + data = NULL; + if (!mtx_trylock(&sbi->z_extent_cache_lock)) + continue; + cache = &sbi->z_extent_cache; + if (cache->state == EROFS_ZCACHE_READY && cache->waiters == 0) { + data = z_erofs_extent_cache_drop_locked(cache, true, true); + cache->state = EROFS_ZCACHE_EMPTY; + } + mtx_unlock(&sbi->z_extent_cache_lock); + free(data, M_EROFS); + } + mtx_unlock(&z_erofs_cache_list_lock); +} + +static void +z_erofs_extent_cache_global_init(void *arg) +{ + + (void)arg; + z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +z_erofs_extent_cache_global_fini(void *arg) +{ + + (void)arg; + if (z_erofs_cache_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag); + KASSERT(LIST_EMPTY(&z_erofs_cache_mounts), + ("erofs decoded cache mounts remain at unload")); + KASSERT(z_erofs_cache_global_charged == 0, + ("erofs decoded cache bytes remain at unload")); +} + +SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY, + z_erofs_extent_cache_global_init, NULL); +SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY, + z_erofs_extent_cache_global_fini, NULL); + +void +z_erofs_extent_cache_init(struct erofs_sb_info *sbi) +{ + + bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache)); + sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget, + EROFS_ZCACHE_MOUNT_HARD_BUDGET); + sbi->z_extent_cache.minimum_decode_work = + z_erofs_cache_minimum_decode_work; + mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF); + cv_init(&sbi->z_extent_cache.cv, "erofs zextent"); + sbi->z_extent_cache_initialized = true; + mtx_lock(&z_erofs_cache_list_lock); + LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); +} + +void +z_erofs_extent_cache_fini(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + void *data; + + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&z_erofs_cache_list_lock); + LIST_REMOVE(sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); + mtx_lock(&sbi->z_extent_cache_lock); + cache->closing = true; + while (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) + cv_wait(&cache->cv, &sbi->z_extent_cache_lock); + data = z_erofs_extent_cache_drop_locked(cache, false, false); + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + sbi->z_extent_cache_initialized = false; + mtx_unlock(&sbi->z_extent_cache_lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&sbi->z_extent_cache_lock); +} + +static bool +z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t len) +{ + const struct erofs_zextent_cache *cache; + uint64_t decode_work; + + cache = &sbi->z_extent_cache; + if (map->m_plen > UINT64_MAX - len) + decode_work = UINT64_MAX; + else + decode_work = map->m_plen + len; + return (z_erofs_cache_enabled != 0 && len <= MAXPHYS && + len <= cache->budget_bytes && + decode_work >= cache->minimum_decode_work && + (map->m_flags & (EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT)) == 0 && + map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX && + sbi->z_extent_cache_initialized && vi != sbi->packed_inode && + vi != sbi->metabox_en); +} + +static void +z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map) +{ + struct erofs_zextent_cache *cache; + + if (!sbi->z_extent_cache_initialized || + map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + ++cache->metrics[map->m_algorithmformat].bypasses; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static int +z_erofs_decode_length(const struct erofs_map_blocks *map, + erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial) +{ + size_t end; + + if (mapoff > map->m_llen || want > map->m_llen - mapoff) + return (EINTEGRITY); +#if SIZE_MAX < UINT64_MAX + if (map->m_llen > SIZE_MAX || mapoff > SIZE_MAX) + return (EOVERFLOW); +#endif + if ((size_t)mapoff > SIZE_MAX - want) + return (EOVERFLOW); + end = (size_t)mapoff + want; + *partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0; + if (!*partial && (!z_erofs_decompress_supports_subextent(map) || + end == map->m_llen)) { + *decoded_len = (size_t)map->m_llen; + return (0); + } + *decoded_len = end; + *partial = true; + return (0); +} + +static int +z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + const void *input; + void *compressed, *decoded; + int error; + + *bufp = NULL; + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) + return (EINTEGRITY); + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (EINTEGRITY); +#if SIZE_MAX < UINT64_MAX + if (map->m_plen > SIZE_MAX || map->m_llen > SIZE_MAX) + return (EOVERFLOW); +#endif + if (decoded_len == 0 || decoded_len > map->m_llen) + return (EINTEGRITY); + if (!partial && decoded_len != map->m_llen) + return (EINTEGRITY); + + if ((map->m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map->m_pa, + (size_t)map->m_plen, &buf); + } else { + error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa, + (size_t)map->m_plen, &compressed); + } + if (error != 0) + return (error); + input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed; + + decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO); + error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen, + decoded, decoded_len, partial); + if ((map->m_flags & EROFS_MAP_META) != 0) + erofs_put_metabuf(&buf); + else + erofs_brelse(compressed); + if (error != 0) { + free(decoded, M_EROFS); + return (error); + } + *bufp = decoded; + return (0); +} + +static int +z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, char *out) +{ + struct erofs_map_blocks map; + void *decoded, *fragment; + erofs_off_t mapoff; + size_t decoded_len, done, want; + bool cache_eligible, partial; + enum z_erofs_cache_lookup cache_lookup; + int error; + + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) + return (EINTEGRITY); + mapoff = loff + done - map.m_la; +#if SIZE_MAX < UINT64_MAX + if (mapoff > SIZE_MAX) + return (EOVERFLOW); + if (map.m_llen - mapoff > SIZE_MAX) + return (EOVERFLOW); +#endif + want = MIN((size_t)(map.m_llen - mapoff), len - done); + if (want == 0) + return (EINTEGRITY); + + if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > UINT64_MAX - mapoff) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->packed_inode, + vi->z_fragmentoff + mapoff, want, &fragment); + if (error != 0) + return (error); + memcpy(out + done, fragment, want); + erofs_brelse(fragment); + } else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + error = z_erofs_decode_length(&map, mapoff, want, + &decoded_len, &partial); + if (error != 0) + return (error); + cache_eligible = !partial && + z_erofs_extent_cache_eligible(sbi, vi, &map, decoded_len); + cache_lookup = Z_EROFS_CACHE_BYPASS; + if (cache_eligible) { + cache_lookup = z_erofs_extent_cache_claim(sbi, vi, &map, + decoded_len, mapoff, want, out + done, &error); + if (cache_lookup == Z_EROFS_CACHE_HIT) { + done += want; + continue; + } + if (cache_lookup == Z_EROFS_CACHE_ERROR) + return (error); + } else + z_erofs_extent_cache_record_bypass(sbi, &map); + error = z_erofs_decode_extent(sbi, vi, &map, decoded_len, + partial, &decoded); + if (error != 0) { + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, + decoded_len, NULL, error); + return (error); + } + memcpy(out + done, (char *)decoded + (size_t)mapoff, want); + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, decoded_len, + decoded, 0); + else + free(decoded, M_EROFS); + } + done += want; + } + return (0); +} + +int +z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) + return (0); + if (loff > UINT64_MAX - len) + return (EOVERFLOW); + if (loff > vi->size || len > vi->size - loff) + return (EINTEGRITY); + + out = malloc(len, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, loff, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); +} + +int +z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + char *buf; + size_t want; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + want = MIN((size_t)uio->uio_resid, + (size_t)MIN((uint64_t)MAXPHYS, + vi->size - (uint64_t)uio->uio_offset)); + buf = malloc(want, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset, + want, buf); + if (error == 0) + error = uiomove(buf, want, uio); + free(buf, M_EROFS); + if (error != 0) + return (error); + } + return (0); +} diff --git a/zmap.c b/zmap.c new file mode 100644 index 0000000..48732f2 --- /dev/null +++ b/zmap.c @@ -0,0 +1,1108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include + +#include "internal.h" + +#define Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE (64 * 1024) + +struct z_erofs_maprecorder { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn; + uint8_t type; + uint8_t headtype; + unsigned int clusterofs; + uint16_t delta[2]; + erofs_blk_t pblk; + erofs_blk_t compressedblks; + erofs_off_t nextpackoff; + bool partialref; +}; + +_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag"); + +static int +z_erofs_index_base(const struct erofs_inode *vi, erofs_off_t *base) +{ + erofs_off_t end; + + if (__builtin_add_overflow(vi->inode_off, + (erofs_off_t)vi->inode_isize, &end) || + __builtin_add_overflow(end, (erofs_off_t)vi->xattr_isize, &end) || + __builtin_add_overflow(end, (erofs_off_t)7, &end)) + return (EINTEGRITY); + end = rounddown2(end, 8); + if (__builtin_add_overflow(end, + (erofs_off_t)sizeof(struct z_erofs_map_header), base)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_index_advance(erofs_off_t *pos, uint64_t count, uint64_t unit) +{ + erofs_off_t delta, next; + + if (__builtin_mul_overflow(count, unit, &delta) || + __builtin_add_overflow(*pos, delta, &next)) + return (EINTEGRITY); + *pos = next; + return (0); +} + +static int +z_erofs_lcluster_count(uint64_t size, unsigned int lclusterbits, + uint64_t *count) +{ + uint64_t mask; + + if (lclusterbits >= 64) + return (EINTEGRITY); + mask = (1ULL << lclusterbits) - 1; + *count = size >> lclusterbits; + if ((size & mask) != 0 && __builtin_add_overflow(*count, 1, count)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_lcluster_pos(uint64_t lcn, unsigned int lclusterbits, + uint64_t clusterofs, uint64_t *pos) +{ + uint64_t base, clustersize; + + if (lclusterbits >= 64) + return (EINTEGRITY); + clustersize = 1ULL << lclusterbits; + if (__builtin_mul_overflow(lcn, clustersize, &base) || + __builtin_add_overflow(base, clusterofs, pos)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_lcn_advance(uint64_t *lcn, uint64_t delta) +{ + uint64_t next; + + if (__builtin_add_overflow(*lcn, delta, &next)) + return (EINTEGRITY); + *lcn = next; + return (0); +} + +static int +z_erofs_compact_pblk(uint32_t base, unsigned int nblk, erofs_blk_t *pblk) +{ + if (__builtin_add_overflow((erofs_blk_t)base, (erofs_blk_t)nblk, + pblk)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_fragment_offset(uint64_t low, erofs_blk_t high, erofs_off_t *offset) +{ + if (low > UINT32_MAX || high > UINT32_MAX) + return (EINTEGRITY); + *offset = low | (high << 32); + return (0); +} + +static int +z_erofs_post_eof_len(uint64_t la, uint64_t size, uint64_t *len) +{ + if (la < size) + return (EINTEGRITY); + *len = la - size; + if (__builtin_add_overflow(*len, 1, len)) + *len = UINT64_MAX; + return (0); +} + +static int +z_erofs_physical_end(uint64_t pa, uint64_t plen, uint64_t block_size) +{ + uint64_t limit, pend; + + if (__builtin_add_overflow(pa, plen, &pend)) + return (EINTEGRITY); + if (__builtin_mul_overflow(1ULL << 48, block_size, &limit)) + return (0); + if (pend > limit) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len, + struct erofs_buf *buf) +{ + return (erofs_read_metadata(m->sbi, m->vi->nid, pos, len, buf)); +} + +static int +z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + struct z_erofs_lcluster_index *di; + erofs_off_t base, pos; + unsigned int advise; + int error; + + vi = m->vi; + if (z_erofs_index_base(vi, &base) != 0 || + z_erofs_index_advance(&base, 1, 8) != 0) + return (EINTEGRITY); + pos = base; + if (z_erofs_index_advance(&pos, lcn, sizeof(*di)) != 0) + return (EINTEGRITY); + m->nextpackoff = pos; + if (z_erofs_index_advance(&m->nextpackoff, 1, sizeof(*di)) != 0) + return (EINTEGRITY); + error = z_erofs_read_index(m, pos, sizeof(*di), &buf); + if (error != 0) + return (error); + + di = buf.data; + m->lcn = lcn; + advise = le16toh(di->di_advise); + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + m->delta[0] = le16toh(di->di_u.delta[0]); + if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->compressedblks = + m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16toh(di->di_u.delta[1]); + } else { + m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0; + m->clusterofs = le16toh(di->di_clusterofs); + m->pblk = le32toh(di->di_u.blkaddr); + } + erofs_put_metabuf(&buf); + return (0); +} + +static unsigned int +decode_compactedbits(unsigned int lobits, const uint8_t *in, + unsigned int pos, uint8_t *type) +{ + uint32_t value; + unsigned int lo; + + value = le32dec(in + pos / 8) >> (pos & 7); + lo = value & ((1U << lobits) - 1); + *type = (value >> lobits) & 3; + return (lo); +} + +static int +get_compacted_la_distance(unsigned int lobits, unsigned int encodebits, + unsigned int vcnt, const uint8_t *in, int i) +{ + unsigned int lo, distance; + uint8_t type; + + distance = 0; + do { + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + return (distance); + ++distance; + } while (++i < (int)vcnt); + + if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0) { + if (lo == 0) + return (-1); + distance += lo - 1; + } + return ((int)distance); +} + +static int +z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + erofs_off_t ebase, pos; + uint64_t compacted_2b, totalidx, original_lcn; + unsigned int compacted_4b_initial, amortizedshift; + unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize; + bool big_pcluster; + uint8_t *in, type; + int distance, error, i; + + vi = m->vi; + if (vi->z_lclusterbits > 14 || + z_erofs_index_base(vi, &ebase) != 0 || + z_erofs_lcluster_count(vi->size, vi->z_lclusterbits, + &totalidx) != 0) + return (EINTEGRITY); + if (lcn >= totalidx) + return (EINVAL); + + original_lcn = lcn; + m->lcn = lcn; + compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; + compacted_2b = 0; + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 && + compacted_4b_initial < totalidx) + compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16); + + pos = ebase; + amortizedshift = 2; + if (lcn >= compacted_4b_initial) { + if (z_erofs_index_advance(&pos, compacted_4b_initial, 4) != 0) + return (EINTEGRITY); + lcn -= compacted_4b_initial; + if (lcn < compacted_2b) { + amortizedshift = 1; + } else { + if (z_erofs_index_advance(&pos, compacted_2b, 2) != 0) + return (EINTEGRITY); + lcn -= compacted_2b; + } + } + if (z_erofs_index_advance(&pos, lcn, 1U << amortizedshift) != 0) + return (EINTEGRITY); + + if (amortizedshift == 2 && vi->z_lclusterbits <= 14) + vcnt = 2; + else if (amortizedshift == 1 && vi->z_lclusterbits <= 12) + vcnt = 16; + else + return (EOPNOTSUPP); + + packsize = vcnt << amortizedshift; + bytes = pos & (packsize - 1); + pos -= bytes; + m->nextpackoff = pos; + if (z_erofs_index_advance(&m->nextpackoff, 1, packsize) != 0) + return (EINTEGRITY); + error = z_erofs_read_index(m, pos, packsize, &buf); + if (error != 0) + return (error); + in = buf.data; + lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT)); + encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt; + i = bytes >> amortizedshift; + + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + if (lookahead) { + distance = get_compacted_la_distance(lobits, encodebits, + vcnt, in, i); + if (distance < 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->delta[1] = distance; + } + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if (!big_pcluster) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + } else { + if (i == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + lo = decode_compactedbits(lobits, in, + encodebits * (i - 1), &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + lo = 0; + else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) + lo = 1; + m->delta[0] = lo + 1; + } + erofs_put_metabuf(&buf); + return (m->delta[0] == 0 ? EINTEGRITY : 0); + } + + m->clusterofs = lo; + m->delta[0] = 0; + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) + i -= lo; + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + --i; + nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; + continue; + } + if (lo <= 1) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + error = z_erofs_compact_pblk( + le32dec(in + packsize - sizeof(uint32_t)), nblk, &m->pblk); + erofs_put_metabuf(&buf); + if (error != 0) + return (error); + m->lcn = original_lcn; + return (0); +} + +static int +z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + int error; + + if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) + error = z_erofs_load_compact_lcluster(m, lcn, lookahead); + else if (m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL) + error = z_erofs_load_full_lcluster(m, lcn); + else + return (EINTEGRITY); + if (error != 0) + return (error); + if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX) + return (EOPNOTSUPP); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->clusterofs >= (1U << m->vi->z_lclusterbits)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + uint64_t lcn; + int error; + + while (lookback_distance != 0 && m->lcn >= lookback_distance) { + lcn = m->lcn - lookback_distance; + error = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + lookback_distance = m->delta[0]; + continue; + } + m->headtype = m->type; + error = z_erofs_lcluster_pos(lcn, m->vi->z_lclusterbits, + m->clusterofs, &m->map->m_la); + if (error != 0) + return (error); + return (0); + } + return (EINTEGRITY); +} + +static int +z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, + uint64_t initial_lcn) +{ + struct erofs_inode *vi; + bool bigpcl1, bigpcl2; + uint64_t lcn, lcnpos; + int error; + + vi = m->vi; + bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0; + lcn = m->lcn; + if (z_erofs_lcn_advance(&lcn, 1) != 0 || + z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0, &lcnpos) != 0) + return (EINTEGRITY); + if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || + ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || + m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || + lcnpos >= vi->size) + m->compressedblks = 1; + if (m->compressedblks == 0) { + error = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->delta[0] != 1) + return (EINTEGRITY); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD || + m->compressedblks == 0) + m->compressedblks = 1; + } + if (__builtin_mul_overflow(m->compressedblks, + (uint64_t)m->sbi->block_size, &m->map->m_plen)) + return (EINTEGRITY); + (void)initial_lcn; + return (0); +} + +static int +z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn, headlcn, lend, lcnpos; + int error; + + vi = m->vi; + map = m->map; + lcn = m->lcn; + headlcn = map->m_la >> vi->z_lclusterbits; + for (;;) { + if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0, + &lcnpos) != 0) + return (EINTEGRITY); + if (lcnpos >= vi->size) { + if (map->m_la > vi->size) + return (EINTEGRITY); + map->m_llen = vi->size - map->m_la; + return (0); + } + error = z_erofs_load_lcluster_from_disk(m, lcn, true); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (m->delta[1] == 0) + m->delta[1] = 1; + } else { + if (lcn != headlcn) + break; + m->delta[1] = 1; + } + if (z_erofs_lcn_advance(&lcn, m->delta[1]) != 0) + return (EINTEGRITY); + } + if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, m->clusterofs, + &lend) != 0 || lend < map->m_la) + return (EINTEGRITY); + map->m_llen = lend - map->m_la; + return (0); +} + +static int +z_erofs_extent_add(uint64_t left, uint64_t right, uint64_t *result) +{ + if (__builtin_add_overflow(left, right, result)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_roundup(uint64_t value, unsigned int alignment, + uint64_t *result) +{ + uint64_t rounded; + + if (z_erofs_extent_add(value, alignment - 1, &rounded) != 0) + return (EINTEGRITY); + *result = rounddown2(rounded, alignment); + return (0); +} + +static int +z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz, + erofs_off_t *result) +{ + erofs_off_t pos; + + if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 || + z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 || + z_erofs_extent_roundup(pos, 8, &pos) != 0 || + z_erofs_extent_add(pos, sizeof(struct z_erofs_map_header), &pos) != 0 || + z_erofs_extent_roundup(pos, recsz, result) != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_record_pos(const struct erofs_inode *vi, erofs_off_t table_pos, + unsigned int recsz, uint64_t index, erofs_off_t *result) +{ + uint64_t offset; + + if (index >= vi->z_extents || + __builtin_mul_overflow(index, recsz, &offset) || + z_erofs_extent_add(table_pos, offset, result) != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + int error; + + bzero(ext, sizeof(*ext)); + error = erofs_read_metadata(sbi, vi->nid, pos, recsz, &buf); + if (error != 0) + return (error); + memcpy(ext, buf.data, recsz); + erofs_put_metabuf(&buf); + return (0); +} + +static uint64_t +z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz) +{ + uint64_t lstart; + + lstart = le32toh(ext->lstart_lo); + if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) + lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32; + return (lstart); +} + +static int +z_erofs_validate_extent_table(struct erofs_sb_info *sbi, struct erofs_inode *vi, + unsigned int recsz) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + erofs_off_t extent_pos, record_pos, scan_pos, table_end; + uint64_t index, lstart, previous; + size_t chunk_len, offset; + int error; + + if (vi->z_extents == 0) + return (vi->size == 0 ? 0 : EINTEGRITY); + error = z_erofs_extent_table_pos(vi, recsz, &extent_pos); + if (error != 0) + return (error); + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo) && + z_erofs_extent_add(extent_pos, sizeof(uint64_t), &extent_pos) != 0) + return (EINTEGRITY); + error = z_erofs_extent_record_pos(vi, extent_pos, recsz, + vi->z_extents - 1, &record_pos); + if (error != 0 || + z_erofs_extent_add(record_pos, recsz, &table_end) != 0) + return (EINTEGRITY); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) + return (z_erofs_read_extent(sbi, vi, record_pos, recsz, &ext)); + if (vi->size == 0) + return (EINTEGRITY); + + scan_pos = extent_pos; + index = 0; + previous = 0; + while (scan_pos < table_end) { + chunk_len = (size_t)MIN(table_end - scan_pos, + (uint64_t)Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE); + error = erofs_read_metadata(sbi, vi->nid, scan_pos, chunk_len, &buf); + if (error != 0) + return (error); + for (offset = 0; offset < chunk_len; offset += recsz, index++) { + bzero(&ext, sizeof(ext)); + memcpy(&ext, (const char *)buf.data + offset, recsz); + lstart = z_erofs_extent_lstart(&ext, recsz); + if (lstart >= vi->size || + (index != 0 && lstart <= previous)) { + error = EINTEGRITY; + goto out; + } + previous = lstart; + } + erofs_put_metabuf(&buf); + scan_pos += chunk_len; + } + return (index == vi->z_extents ? 0 : EINTEGRITY); +out: + erofs_put_metabuf(&buf); + return (error); +} + +static int +z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + bool fragment, ztailpacking; + struct z_erofs_maprecorder m; + uint64_t initial_lcn, ofs, end, end_lcn; + erofs_off_t fragmentoff; + unsigned int endoff; + int error; + + fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + ztailpacking = vi->z_idata_size != 0; + bzero(&m, sizeof(m)); + m.sbi = sbi; + m.vi = vi; + m.map = map; + + if (vi->size == 0) { + map->m_la = 0; + map->m_llen = 0; + map->m_flags = 0; + return (0); + } + ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ? + vi->size - 1 : map->m_la; + if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 && + vi->z_tailextent_headlcn == 0) { + map->m_la = 0; + map->m_llen = vi->size; + map->m_flags = EROFS_MAP_FRAGMENT; + return (0); + } + initial_lcn = ofs >> vi->z_lclusterbits; + endoff = ofs & ((1U << vi->z_lclusterbits) - 1); + error = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); + if (error != 0) + return (error); + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking) + vi->z_fragmentoff = m.nextpackoff; + + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; + end_lcn = m.lcn; + if (z_erofs_lcn_advance(&end_lcn, 1) != 0 || + z_erofs_lcluster_pos(end_lcn, vi->z_lclusterbits, 0, &end) != 0) + return (EINTEGRITY); + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + endoff >= m.clusterofs) { + m.headtype = m.type; + if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits, + m.clusterofs, &map->m_la) != 0) + return (EINTEGRITY); + if (ztailpacking && end > vi->size) + end = vi->size; + } else { + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits, + m.clusterofs, &end) != 0) + return (EINTEGRITY); + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + m.delta[0] = 1; + } + error = z_erofs_extent_lookback(&m, m.delta[0]); + if (error != 0) + return (error); + } + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + if (end < map->m_la) + return (EINTEGRITY); + map->m_llen = end - map->m_la; + + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) { + vi->z_tailextent_headlcn = m.lcn; + if (fragment && + vi->datalayout == EROFS_INODE_COMPRESSED_FULL) { + error = z_erofs_fragment_offset(vi->z_fragmentoff, m.pblk, + &fragmentoff); + if (error != 0) + return (error); + vi->z_fragmentoff = fragmentoff; + } + } + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_fragmentoff; + map->m_plen = vi->z_idata_size; + if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen > + sbi->block_size) + return (EINTEGRITY); + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags = EROFS_MAP_FRAGMENT; + } else { + if (__builtin_mul_overflow(m.pblk, (erofs_blk_t)sbi->block_size, + &map->m_pa)) + return (EINTEGRITY); + error = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (error != 0) + return (error); + } + + if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { + map->m_algorithmformat = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 ? + Z_EROFS_COMPRESSION_INTERLACED : + Z_EROFS_COMPRESSION_SHIFTED; + } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { + map->m_algorithmformat = vi->z_algorithmtype[1]; + } else { + map->m_algorithmformat = vi->z_algorithmtype[0]; + } + + if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0 || + ((flags & EROFS_GET_BLOCKS_READMORE) != 0 && + (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA || + map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE || + map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) && + map->m_llen >= sbi->block_size)) { + error = z_erofs_get_extent_decompressedlen(&m); + if (error == 0) + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + return (error); + } + return (0); +} + +static int +z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + unsigned int recsz, bmask; + uint8_t fmt; + uint64_t cluster_size, extent_idx, next, rounded_lend; + erofs_off_t extent_pos, pos, table_pos; + uint64_t fragmentoff, lend, l, r, mid, pa, la, lstart; + bool interlaced, last; + int error; + + (void)flags; + interlaced = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0; + recsz = z_erofs_extent_recsize(vi->z_advise); + error = z_erofs_extent_table_pos(vi, recsz, &table_pos); + if (error != 0) + return (error); + pos = table_pos; + bmask = sbi->block_size - 1; + lend = vi->size; + cluster_size = 1ULL << vi->z_lclusterbits; + map->m_flags = 0; + + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { + error = erofs_read_metadata(sbi, vi->nid, pos, + sizeof(uint64_t), &buf); + if (error != 0) + return (error); + pa = le64dec(buf.data); + erofs_put_metabuf(&buf); + if (z_erofs_extent_add(pos, sizeof(uint64_t), &pos) != 0) + return (EINTEGRITY); + lstart = 0; + extent_idx = 0; + } else { + lstart = rounddown2(map->m_la, cluster_size); + extent_idx = lstart >> vi->z_lclusterbits; + pa = EROFS_NULL_ADDR; + } + for (;;) { + error = z_erofs_extent_record_pos(vi, pos, recsz, + extent_idx, &extent_pos); + if (error != 0) + return (error); + error = z_erofs_read_extent(sbi, vi, extent_pos, recsz, &ext); + if (error != 0) + return (error); + map->m_plen = le32toh(ext.plen); + if (pa != EROFS_NULL_ADDR) { + map->m_pa = pa; + if (z_erofs_extent_add(pa, + map->m_plen & Z_EROFS_EXTENT_PLEN_MASK, + &next) != 0) + return (EINTEGRITY); + pa = next; + } else { + map->m_pa = le32toh(ext.pstart_lo); + } + if (extent_idx == UINT64_MAX) + return (EINTEGRITY); + extent_idx++; + if (z_erofs_extent_add(lstart, cluster_size, &next) != 0) + return (EINTEGRITY); + lstart = next; + if (lstart > map->m_la) + break; + } + if (z_erofs_extent_roundup(lend, cluster_size, &rounded_lend) != 0) + return (EINTEGRITY); + last = lstart >= rounded_lend; + lend = MIN(lstart, lend); + lstart -= cluster_size; + } else { + lstart = lend; + for (l = 0, r = vi->z_extents; l < r;) { + mid = l + (r - l) / 2; + error = z_erofs_extent_record_pos(vi, table_pos, recsz, mid, + &extent_pos); + if (error != 0) + return (error); + error = z_erofs_read_extent(sbi, vi, extent_pos, + recsz, &ext); + if (error != 0) + return (error); + la = z_erofs_extent_lstart(&ext, recsz); + pa = le32toh(ext.pstart_lo) | + ((uint64_t)le32toh(ext.pstart_hi) << 32); + if (la > map->m_la) { + r = mid; + if (la > lend) + return (EINTEGRITY); + lend = la; + } else { + l = mid + 1; + if (map->m_la == la) + r = MIN(l + 1, r); + lstart = la; + map->m_plen = le32toh(ext.plen); + map->m_pa = pa; + } + } + last = l >= vi->z_extents; + } + + if (lstart < lend) { + map->m_la = lstart; + if (last && + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + map->m_flags = EROFS_MAP_FRAGMENT; + if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) { + error = z_erofs_fragment_offset(map->m_plen, map->m_pa, + &fragmentoff); + if (error != 0) + return (error); + vi->z_fragmentoff = fragmentoff; + } else { + vi->z_fragmentoff = map->m_plen; + } + } else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) { + map->m_flags = EROFS_MAP_MAPPED; + if ((map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT) > + UINT8_MAX) + return (EINTEGRITY); + fmt = (uint8_t)(map->m_plen >> + Z_EROFS_EXTENT_PLEN_FMT_BIT); + if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; + if (fmt != 0) + map->m_algorithmformat = (uint8_t)(fmt - 1); + else if (interlaced && + ((map->m_pa | map->m_plen) & bmask) == 0) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } + } + map->m_llen = lend - map->m_la; + return (0); +} + +int +z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_map_header *h; + struct erofs_map_blocks map; + uint64_t cluster_size, raw, rounded_size; + erofs_off_t pos; + unsigned int recsz; + int error; + + if (vi->z_initialized) + return (0); + if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 || + z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 || + z_erofs_extent_roundup(pos, 8, &pos) != 0) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, vi->nid, pos, sizeof(*h), &buf); + if (error != 0) + return (error); + h = buf.data; + if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) { + if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + raw = le64dec(h); + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = raw ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + vi->fragment = true; + erofs_put_metabuf(&buf); + vi->z_initialized = true; + return (0); + } + + vi->z_advise = le16toh(h->h_advise); + vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15); + if (vi->z_lclusterbits >= 31) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) { + recsz = z_erofs_extent_recsize(vi->z_advise); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + cluster_size = 1ULL << vi->z_lclusterbits; + if (z_erofs_extent_roundup(vi->size, cluster_size, + &rounded_size) != 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->z_extents = rounded_size >> vi->z_lclusterbits; + } else { + vi->z_extents = le32toh(h->h_extents_lo) | + ((uint64_t)le16toh(h->h_extents_hi) << 32); + } + vi->fragment = + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + erofs_put_metabuf(&buf); + if (vi->fragment && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (EINTEGRITY); + if (recsz > offsetof(struct z_erofs_extent, pstart_hi) && + vi->z_extents == 0 && vi->size != 0) + return (EINTEGRITY); + error = z_erofs_validate_extent_table(sbi, vi, recsz); + if (error != 0) + return (error); + vi->z_initialized = true; + return (0); + } + vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15); + vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4); + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) + vi->z_fragmentoff = le32toh(h->h_fragmentoff); + else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0) + vi->z_idata_size = le16toh(h->h_idata_size); + erofs_put_metabuf(&buf); + + if (!erofs_sb_has_big_pcluster(sbi) && + (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0) + return (EINTEGRITY); + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && + (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) != + ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0))) + return (EINTEGRITY); + if (vi->z_idata_size != 0 || + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (EINTEGRITY); + bzero(&map, sizeof(map)); + error = z_erofs_map_blocks_fo(sbi, vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (error != 0) + return (error); + } + vi->fragment = + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + vi->z_initialized = true; + return (0); +} + +static int +z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) { + if ((map->m_flags & (EROFS_MAP_MAPPED | EROFS_MAP_META)) != 0 || + sbi->packed_inode == NULL || sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > sbi->packed_inode->size || + map->m_llen > sbi->packed_inode->size - vi->z_fragmentoff) + return (EINTEGRITY); + return (0); + } + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (0); + if (map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX) + return (EOPNOTSUPP); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if ((sbi->available_compr_algs & + (1U << map->m_algorithmformat)) == 0) + return (EINTEGRITY); + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen) + return (EINTEGRITY); + } else if (map->m_llen > map->m_plen) { + return (EINTEGRITY); + } + if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || + map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE) + return (EOPNOTSUPP); + if ((map->m_flags & EROFS_MAP_META) != 0) + return (0); + if (z_erofs_physical_end(map->m_pa, map->m_plen, + sbi->block_size) != 0) + return (EINTEGRITY); + (void)vi; + return (0); +} + +int +z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + int error; + + if (map->m_la >= vi->size) { + error = z_erofs_post_eof_len(map->m_la, vi->size, + &map->m_llen); + if (error != 0) + return (error); + map->m_la = vi->size; + map->m_flags = 0; + return (0); + } + error = z_erofs_fill_inode(sbi, vi); + if (error == 0) { + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) + error = z_erofs_map_blocks_ext(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + else + error = z_erofs_map_blocks_fo(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + } + if (error == 0) + error = z_erofs_map_sanity_check(sbi, vi, map); + if (error != 0) + map->m_llen = 0; + return (error); +}