commit 770d83819bfbca9d76833041e27e48ee2b411d6a Author: imrcpan Date: Tue Aug 18 09:38:06 2026 +0200 update diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..3d436fa --- /dev/null +++ b/.clang-format @@ -0,0 +1,199 @@ +# Basic .clang-format +--- +BasedOnStyle: WebKit +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: AcrossEmptyLines +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: TopLevelDefinitions +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeTernaryOperators: false +# TODO: BreakStringLiterals can cause very strange formatting so turn it off? +BreakStringLiterals: false +# Prefer: +# some_var = function(arg1, +# arg2) +# over: +# some_var = +# function(arg1, arg2) +PenaltyBreakAssignment: 100 +# Prefer: +# some_long_function(arg1, arg2 +# arg3) +# over: +# some_long_function( +# arg1, arg2, arg3) +PenaltyBreakBeforeFirstCallParameter: 100 +CompactNamespaces: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: + - ARB_ARRFOREACH + - ARB_ARRFOREACH_REVWCOND + - ARB_ARRFOREACH_REVERSE + - ARB_FOREACH + - ARB_FOREACH_FROM + - ARB_FOREACH_SAFE + - ARB_FOREACH_REVERSE + - ARB_FOREACH_REVERSE_FROM + - ARB_FOREACH_REVERSE_SAFE + - BIT_FOREACH_ISCLR + - BIT_FOREACH_ISSET + - CPU_FOREACH + - CPU_FOREACH_ISCLR + - CPU_FOREACH_ISSET + - FOREACH_THREAD_IN_PROC + - FOREACH_PROC_IN_SYSTEM + - FOREACH_PRISON_CHILD + - FOREACH_PRISON_DESCENDANT + - FOREACH_PRISON_DESCENDANT_LOCKED + - FOREACH_PRISON_DESCENDANT_LOCKED_LEVEL + - MNT_VNODE_FOREACH_ALL + - MNT_VNODE_FOREACH_ACTIVE + - RB_FOREACH + - RB_FOREACH_FROM + - RB_FOREACH_SAFE + - RB_FOREACH_REVERSE + - RB_FOREACH_REVERSE_FROM + - RB_FOREACH_REVERSE_SAFE + - SLIST_FOREACH + - SLIST_FOREACH_FROM + - SLIST_FOREACH_FROM_SAFE + - SLIST_FOREACH_SAFE + - SLIST_FOREACH_PREVPTR + - SPLAY_FOREACH + - LIST_FOREACH + - LIST_FOREACH_FROM + - LIST_FOREACH_FROM_SAFE + - LIST_FOREACH_SAFE + - STAILQ_FOREACH + - STAILQ_FOREACH_FROM + - STAILQ_FOREACH_FROM_SAFE + - STAILQ_FOREACH_SAFE + - TAILQ_FOREACH + - TAILQ_FOREACH_FROM + - TAILQ_FOREACH_FROM_SAFE + - TAILQ_FOREACH_REVERSE + - TAILQ_FOREACH_REVERSE_FROM + - TAILQ_FOREACH_REVERSE_FROM_SAFE + - TAILQ_FOREACH_REVERSE_SAFE + - TAILQ_FOREACH_SAFE + - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH +SpaceBeforeParens: ControlStatementsExceptForEachMacros +IndentCaseLabels: false +IndentPPDirectives: None +Language: Cpp +NamespaceIndentation: None +PointerAlignment: Right +ContinuationIndentWidth: 4 +IndentWidth: 8 +TabWidth: 8 +ColumnLimit: 80 +UseTab: Always +SpaceAfterCStyleCast: false +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^\"opt_.*\.h\"' + Priority: 1 + SortPriority: 10 + - Regex: '^' + Priority: 2 + SortPriority: 20 + - Regex: '^' + Priority: 2 + SortPriority: 21 + - Regex: '^' + Priority: 2 + SortPriority: 22 + - Regex: '^' + Priority: 2 + SortPriority: 23 + - Regex: '^' + Priority: 3 + SortPriority: 30 + - Regex: '^ diff --git a/compress.h b/compress.h new file mode 100644 index 0000000..e423a03 --- /dev/null +++ b/compress.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __EROFS_FS_COMPRESS_H +#define __EROFS_FS_COMPRESS_H + +#include "internal.h" + +struct z_erofs_decompress_req { + struct erofs_sb_info *sbi; + const struct erofs_map_blocks *map; + const void *in; + size_t inputsize; + void *out; + size_t outputsize; + bool partial_decoding; +}; + +struct z_erofs_decompressor { + /* Callbacks return zero or a positive FreeBSD errno. */ + int (*config)(struct erofs_sb_info *, const struct erofs_super_block *, + const void *, size_t); + int (*decompress)(const struct z_erofs_decompress_req *); + bool supports_subextent; + const char *name; +}; + +bool z_erofs_decompress_supports_subextent( + const struct erofs_map_blocks *map); +int z_erofs_lz4_decompress(const struct z_erofs_decompress_req *rq); +extern const struct z_erofs_decompressor z_erofs_lzma_decomp; +extern const struct z_erofs_decompressor z_erofs_deflate_decomp; +extern const struct z_erofs_decompressor z_erofs_zstd_decomp; + +#endif /* __EROFS_FS_COMPRESS_H */ diff --git a/data.c b/data.c new file mode 100644 index 0000000..47945c1 --- /dev/null +++ b/data.c @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define EROFS_DIR_READAHEAD_BYTES (1024 * 1024) +#define EROFS_DIR_READAHEAD_SLOTS (EROFS_DIR_READAHEAD_BYTES / PAGE_SIZE) + +/* + * For flat inline files, compute the "inline tail" start offset. + * Linux EROFS semantics: the last logical block may be tailpacked + * into the inode metadata area. + */ +static erofs_off_t +erofs_inline_tail_start(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + if (vi->size == 0) + return (0); + return (roundup2(vi->size, (uint64_t)sbi->block_size) - sbi->block_size); +} + +static int +erofs_check_device_range(const struct erofs_sb_info *sbi, + const struct erofs_device_info *dif, erofs_off_t off, uint64_t len) +{ + erofs_off_t end, limit; + + if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(off, len, &end)) + return (EINTEGRITY); + limit = dif->blocks << sbi->blkszbits; + return (end > limit ? EINTEGRITY : 0); +} + +static void +erofs_fill_from_devinfo(struct erofs_map_dev *map, + struct erofs_device_info *dif, erofs_off_t pa) +{ + map->m_dif = dif; + map->m_pa = pa; +} + +static int +erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + struct erofs_device_info *dif; + uint64_t start; + unsigned int id; + int error; + + erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa); + if (map->m_deviceid != 0) { + if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL) + return (ENODEV); + dif = &sbi->devs[map->m_deviceid - 1]; + error = erofs_check_device_range(sbi, dif, map->m_pa, + map->m_plen); + if (error != 0) + return (error); + if (sbi->flatdev) { + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (__builtin_add_overflow(map->m_pa, start, &map->m_pa)) + return (EINTEGRITY); + return (0); + } + if (dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa); + return (0); + } + + if (sbi->extra_devices == 0) + return (0); + if (sbi->flatdev) { + error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa, + map->m_plen); + if (error == 0) + return (0); + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0 || + dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + continue; + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa < start) + continue; + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error == 0) + return (0); + if (map->m_pa - start < + (dif->blocks << sbi->blkszbits)) + return (error); + } + return (EINTEGRITY); + } + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0) + continue; + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa >= start && + map->m_pa - start < (dif->blocks << sbi->blkszbits)) { + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error != 0) + return (error); + if (dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa - start); + break; + } + } + return (0); +} + +/* Map chunk-based file to physical extent */ +static int +erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_inode_chunk_index *idx; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t chunk_idx, chunk_size, entry_size; + erofs_off_t loff, idx_off, chunk_off, idx_base; + uint64_t image_size, addrmask; + uint64_t blkaddr; + uint16_t raw_device_id; + int error; + + loff = map->m_la; + chunk_size = 1ULL << vi->chunkbits; + chunk_idx = loff >> vi->chunkbits; + chunk_off = loff & (chunk_size - 1); + + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) + entry_size = sizeof(struct erofs_inode_chunk_index); + else + entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE; + if (vi->inode_off > UINT64_MAX - vi->inode_isize || + vi->inode_off + vi->inode_isize > UINT64_MAX - vi->xattr_isize) + return (EOVERFLOW); + idx_base = vi->inode_off + vi->inode_isize + vi->xattr_isize; + if (idx_base > UINT64_MAX - (entry_size - 1)) + return (EOVERFLOW); + idx_base = roundup2(idx_base, entry_size); + if (chunk_idx > (UINT64_MAX - idx_base) / entry_size) + return (EOVERFLOW); + idx_off = idx_base + chunk_idx * entry_size; + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || idx_off > sbi->metabox_en->size || + entry_size > sbi->metabox_en->size - idx_off) + return (EINTEGRITY); + } else { + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + image_size = sbi->blocks << sbi->blkszbits; + if (idx_off > image_size || entry_size > image_size - idx_off) + return (EINTEGRITY); + } + + error = erofs_read_metadata(sbi, vi->nid, idx_off, entry_size, &buf); + if (error != 0) + return (error); + + idx = buf.data; + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) { + blkaddr = le32toh(idx->startblk_lo); + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0) + blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32; + raw_device_id = le16toh(idx->device_id); + addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ? + ((1ULL << 48) - 1) : UINT32_MAX; + } else { + blkaddr = le32dec(idx); + raw_device_id = 0; + addrmask = UINT32_MAX; + } + erofs_put_metabuf(&buf); + + if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) { + map->m_pa = 0; + map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff); + map->m_plen = map->m_llen; + return (0); + } + + map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff); + map->m_plen = map->m_llen; + map->m_deviceid = raw_device_id & sbi->device_id_mask; + if (blkaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + map->m_pa = blkaddr << sbi->blkszbits; + if (chunk_off > UINT64_MAX - map->m_pa) + return (EOVERFLOW); + map->m_pa += chunk_off; + map->m_flags |= EROFS_MAP_MAPPED; + return (0); +} + +static int +erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno, + int *rabsize, int racnt, void **bufp) +{ + struct buf *bp; + erofs_off_t end, limit; + off_t blkoff, current; + size_t blklen, done, iosize; + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (dif == NULL || dif->devvp == NULL || dif->cp == NULL) + return (ENODEV); + if (__builtin_add_overflow(off, (uint64_t)len, &end)) + return (EINTEGRITY); + if (blocks != 0) { + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + limit = blocks << sbi->blkszbits; + if (end > limit) + return (EINTEGRITY); + } + if (end > dif->mediasize) + return (ENXIO); + if (off > INT64_MAX || end > (uint64_t)INT64_MAX + 1) + return (EOVERFLOW); + + iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize; + if (iosize == 0 || (iosize & (iosize - 1)) != 0) + return (EINVAL); + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + current = (off_t)(off + done); + blkoff = rounddown2(current, (off_t)iosize); + blklen = MIN(iosize - (size_t)(current - blkoff), len - done); + if (done == 0 && racnt != 0) + error = breadn(dif->devvp, btodb(blkoff), iosize, + rablkno, rabsize, racnt, NOCRED, &bp); + else + error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, &bp); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (bp->b_data == NULL) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + memcpy(out + done, (char *)bp->b_data + (current - blkoff), + blklen); + brelse(bp); + done += blklen; + } + *bufp = out; + return (0); +} + +int +erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp) +{ + return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len, + NULL, NULL, 0, bufp)); +} + +int +erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp) +{ + struct erofs_map_dev map; + erofs_blk_t blocks; + int error; + + map = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, &map); + if (error != 0) + return (error); + blocks = map.m_dif->blocks; + if (map.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len, + NULL, NULL, 0, bufp)); +} + +static int +erofs_read_physical_readahead(struct erofs_sb_info *sbi, + unsigned int device_id, erofs_off_t off, size_t len, + unsigned int rablocks, void **bufp) +{ + struct erofs_map_dev current, future; + daddr_t rablkno[EROFS_DIR_READAHEAD_SLOTS]; + int rabsize[EROFS_DIR_READAHEAD_SLOTS]; + erofs_off_t step; + erofs_blk_t blocks; + unsigned int count; + int error; + + current = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, ¤t); + if (error != 0) + return (error); + blocks = current.m_dif->blocks; + if (current.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + rablocks = MIN(rablocks, (unsigned int)nitems(rablkno)); + for (count = 0; count < rablocks; count++) { + step = (erofs_off_t)(count + 1) * sbi->block_size; + if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step) + break; + future = (struct erofs_map_dev) { + .m_pa = off + step, + .m_deviceid = device_id, + .m_plen = sbi->block_size, + }; + if (erofs_map_dev(sbi, &future) != 0 || + future.m_dif != current.m_dif || + future.m_pa != current.m_pa + step) + break; + rablkno[count] = btodb(future.m_pa); + rabsize[count] = sbi->block_size; + } + return (erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len, + rablkno, rabsize, count, bufp)); +} + +/* Release a contiguous buffer returned by erofs_bread(). */ +void +erofs_brelse(void *buf) +{ + free(buf, M_EROFS); +} + +void +erofs_put_metabuf(struct erofs_buf *buf) +{ + void (*release)(void *); + void *data; + + if (buf == NULL) + return; + data = buf->data; + release = buf->release; + buf->data = NULL; + buf->release = NULL; + if (data != NULL && release != NULL) + release(data); +} + +/* Read inode metadata from either the primary image or the metabox file. */ +int +erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, erofs_off_t off, + size_t len, struct erofs_buf *buf) +{ + void *data; + int error; + + if (!erofs_nid_in_metabox(nid)) { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } else { + if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + erofs_off_t loff, tail_start; + uint64_t remain, block_rem; + + loff = map->m_la; + if (loff >= vi->size) + return (0); + + remain = vi->size - loff; + switch (vi->datalayout) { + case EROFS_INODE_CHUNK_BASED: + return (erofs_map_blocks_chunk(sbi, vi, map)); + case EROFS_INODE_FLAT_PLAIN: + map->m_llen = remain; + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff, + &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + case EROFS_INODE_FLAT_INLINE: + tail_start = erofs_inline_tail_start(sbi, vi); + if (loff < tail_start) { + map->m_llen = MIN(remain, tail_start - loff); + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, + loff, &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + } + block_rem = sbi->block_size - + ((loff - tail_start) & (sbi->block_size - 1)); + map->m_llen = MIN(remain, block_rem); + map->m_plen = map->m_llen; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &map->m_pa) || __builtin_add_overflow(map->m_pa, vi->xattr_isize, + &map->m_pa) || __builtin_add_overflow(map->m_pa, loff - tail_start, + &map->m_pa)) + return (EINTEGRITY); + map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META; + return (0); + case EROFS_INODE_COMPRESSED_FULL: + case EROFS_INODE_COMPRESSED_COMPACT: + return (EOPNOTSUPP); + default: + return (EOPNOTSUPP); + } +} + +int +erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_map_blocks next = { .m_la = map->m_la }; + int error; + + if (erofs_inode_is_data_compressed(vi->datalayout)) { + error = z_erofs_map_blocks(sbi, vi, &next); + } else { + error = erofs_map_blocks_flatmode(sbi, vi, &next); + } + *map = next; + return (error); +} +/* + * Read a small range at a logical file offset into a contiguous buffer. + * Primarily used for directory block reads, lookup, and symlink fragment + * parsing. + */ +static int +erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char *out; + void *blk; + size_t done, want; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (loff > UINT64_MAX - (uint64_t)len) + return (EOVERFLOW); + if (loff > vi->size || (uint64_t)len > vi->size - loff) + return (EINTEGRITY); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) + return (z_erofs_read_data(sbi, vi, loff, len, bufp)); + + /* Uncompressed file path */ + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (map.m_llen == 0) { + free(out, M_EROFS); + return (EINTEGRITY); + } + want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS), + len - done); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map.m_pa, + want, &buf); + } else if (done == 0 && rablocks != 0 && + map.m_flags == EROFS_MAP_MAPPED) { + error = erofs_read_physical_readahead(sbi, + map.m_deviceid, map.m_pa, want, rablocks, &blk); + } else { + error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa, + want, &blk); + } + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + memcpy(out + done, buf.data, want); + erofs_put_metabuf(&buf); + } else { + memcpy(out + done, blk, want); + erofs_brelse(blk); + } + } + done += want; + } + *bufp = out; + return (0); +} + +int +erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp)); +} + +int +erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, bool sequential, void **bufp) +{ + uint64_t remaining; + unsigned int rablocks; + + rablocks = 0; + if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN && + sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 && + len <= sbi->block_size && loff <= vi->size && len <= vi->size - loff) { + remaining = vi->size - loff - len; + rablocks = MIN(howmany(remaining, sbi->block_size), + (uint64_t)EROFS_DIR_READAHEAD_SLOTS); + } + return (erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp)); +} + +/* + * Transfer the logical content of an inode directly into a uio. + * Regular files and symlinks both use this read path. + */ +static int +erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, struct uio *uio) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char zerobuf[PAGE_SIZE]; + void *blk; + size_t want, chunk; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + if ((uint64_t)uio->uio_offset >= vi->size) + return (0); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) + return (z_erofs_read_uio(sbi, vi, uio)); + + /* Uncompressed file path */ + bzero(zerobuf, sizeof(zerobuf)); + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + map = (struct erofs_map_blocks) { .m_la = uio->uio_offset }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0) + break; + want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS), + (size_t)uio->uio_resid); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + chunk = want; + while (chunk > 0) { + size_t zlen = MIN(chunk, sizeof(zerobuf)); + + error = uiomove(zerobuf, zlen, uio); + if (error != 0) + return (error); + chunk -= zlen; + } + continue; + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map.m_pa, want, + &buf); + } else { + error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa, + want, &blk); + } + if (error != 0) + return (error); + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = uiomove(buf.data, want, uio); + erofs_put_metabuf(&buf); + } else { + error = uiomove(blk, want, uio); + erofs_brelse(blk); + } + if (error != 0) + return (error); + } + return (0); +} + +/* Read regular file data. */ +int +erofs_read_file(struct vnode *vp, struct uio *uio, int ioflag) +{ + (void)ioflag; + return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio)); +} + +int +erofs_validate_symlink_target(struct erofs_sb_info *sbi, + struct erofs_inode *vi) +{ + void *target; + int error; + + if (vi->vtype != VLNK) + return (EINVAL); + if (vi->size == 0) + return (EINTEGRITY); + if (vi->size > MAXPATHLEN) + return (ENAMETOOLONG); + error = erofs_read_data(sbi, vi, 0, (size_t)vi->size, &target); + if (error != 0) + return (error); + if (memchr(target, '\0', (size_t)vi->size) != NULL) + error = EINTEGRITY; + erofs_brelse(target); + return (error); +} + +/* Read symlink target string. */ +int +erofs_readlink_target(struct vnode *vp, struct uio *uio) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + if (vi->size == 0) + return (EINTEGRITY); + if (vi->size > MAXPATHLEN) + return (ENAMETOOLONG); + return (erofs_read_uio(MTOE(vp->v_mount), vi, uio)); +} diff --git a/decompressor.c b/decompressor.c new file mode 100644 index 0000000..3988969 --- /dev/null +++ b/decompressor.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ + +#include +#include +#include +#include +#include + +#include "compress.h" + +static int +z_erofs_load_lz4_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lz4_cfgs *lz4; + uint32_t max_pclusterblks; + uint16_t distance; + + if (data != NULL) { + if (size < sizeof(*lz4)) + return (EINTEGRITY); + lz4 = data; + max_pclusterblks = le16toh(lz4->max_pclusterblks); + if (max_pclusterblks == 0) + max_pclusterblks = 1; + else if (max_pclusterblks > + (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits)) + return (EOPNOTSUPP); + } else { + distance = le16toh(dsb->u1.lz4_max_distance); + if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi)) + return (0); + sbi->available_compr_algs = 1U << Z_EROFS_COMPRESSION_LZ4; + } + return (0); +} + +static int +z_erofs_transform_plain(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *src; + uint8_t *dst; + size_t first, offset; + + if (rq->outputsize > rq->inputsize) + return (EINTEGRITY); + src = rq->in; + dst = rq->out; + if (rq->map->m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) { + memmove(dst, src, rq->outputsize); + return (0); + } + first = MIN((size_t)(rq->sbi->block_size - + (rq->map->m_la & (rq->sbi->block_size - 1))), rq->outputsize); + offset = (rq->inputsize - first) & (rq->sbi->block_size - 1); + if (offset > rq->inputsize || first > rq->inputsize - offset) + return (EINTEGRITY); + memmove(dst, src + offset, first); + if (first < rq->outputsize) + memmove(dst + first, src, rq->outputsize - first); + return (0); +} + +static const struct z_erofs_decompressor z_erofs_shifted_decomp = { + .decompress = z_erofs_transform_plain, + .name = "shifted", +}; + +static const struct z_erofs_decompressor z_erofs_interlaced_decomp = { + .decompress = z_erofs_transform_plain, + .name = "interlaced", +}; + +static const struct z_erofs_decompressor z_erofs_lz4_decomp = { + .config = z_erofs_load_lz4_config, + .decompress = z_erofs_lz4_decompress, + .supports_subextent = 1, + .name = "lz4", +}; + +static const struct z_erofs_decompressor * const z_erofs_decomp[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp, + [Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp, + [Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp, + [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, + [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, + [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, +}; + +bool +z_erofs_decompress_supports_subextent(const struct erofs_map_blocks *map) +{ + uint8_t algorithm; + + algorithm = map->m_algorithmformat; + return (algorithm < nitems(z_erofs_decomp) && + z_erofs_decomp[algorithm] != NULL && + z_erofs_decomp[algorithm]->supports_subextent); +} + +static int +z_erofs_read_cfg(struct erofs_sb_info *sbi, uint64_t *offset, + struct erofs_buf *buf, + size_t *sizep) +{ + struct erofs_buf metabuf = EROFS_BUF_INITIALIZER; + uint8_t length_buf[2]; + uint64_t aligned; + uint16_t length; + int error; + + aligned = roundup2(*offset, 4); + if (aligned > UINT64_MAX - sizeof(length_buf)) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, 0, aligned, sizeof(length_buf), &metabuf); + if (error != 0) + return (error); + memcpy(length_buf, metabuf.data, sizeof(length_buf)); + erofs_put_metabuf(&metabuf); + length = le16dec(length_buf); + *sizep = length != 0 ? length : UINT16_MAX + 1U; + if (*sizep > 65536 || aligned + sizeof(length_buf) > + UINT64_MAX - *sizep) + return (EINTEGRITY); + *offset = aligned + sizeof(length_buf); + error = erofs_read_metadata(sbi, 0, *offset, *sizep, buf); + if (error == 0) + *offset += *sizep; + return (error); +} + +int +z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf data = EROFS_BUF_INITIALIZER; + const struct z_erofs_decompressor *decompressor; + uint64_t offset; + uint16_t algorithms; + size_t size; + int algorithm, error; + + if (!erofs_sb_has_compr_cfgs(sbi)) + return (z_erofs_load_lz4_config(sbi, dsb, NULL, 0)); + algorithms = le16toh(dsb->u1.available_compr_algs); + sbi->available_compr_algs = algorithms; + if ((algorithms & ~Z_EROFS_ALL_COMPR_ALGS) != 0) + return (EOPNOTSUPP); + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; + ++algorithm) { + if ((algorithms & (1U << algorithm)) == 0) + continue; + error = z_erofs_read_cfg(sbi, &offset, &data, &size); + if (error != 0) + return (error); + decompressor = z_erofs_decomp[algorithm]; + if (decompressor == NULL || decompressor->config == NULL) + error = EOPNOTSUPP; + else + error = decompressor->config(sbi, dsb, data.data, size); + erofs_put_metabuf(&data); + if (error != 0) + return (error); + } + return (0); +} + +int +z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src0, size_t srclen, + void *dst, size_t dstlen, bool partial) +{ + const struct z_erofs_decompressor *decompressor; + struct z_erofs_decompress_req rq; + const uint8_t *src; + uint8_t algorithm; + size_t padding, padding_limit; + + algorithm = map->m_algorithmformat; + if (algorithm >= nitems(z_erofs_decomp) || + z_erofs_decomp[algorithm] == NULL || + z_erofs_decomp[algorithm]->decompress == NULL) + return (EOPNOTSUPP); + decompressor = z_erofs_decomp[algorithm]; + rq = (struct z_erofs_decompress_req) { + .sbi = sbi, + .map = map, + .in = src0, + .inputsize = srclen, + .out = dst, + .outputsize = dstlen, + .partial_decoding = partial, + }; + if (algorithm == Z_EROFS_COMPRESSION_SHIFTED || + algorithm == Z_EROFS_COMPRESSION_INTERLACED) + return (decompressor->decompress(&rq)); + + src = src0; + if (map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4 || + erofs_sb_has_lz4_0padding(sbi)) { + padding_limit = MIN(srclen, sbi->block_size - + (map->m_pa & (sbi->block_size - 1))); + for (padding = 0; padding < padding_limit && src[padding] == 0; + ++padding) + ; + if (padding == padding_limit) + return (EINTEGRITY); + src += padding; + srclen -= padding; + } + if (algorithm == Z_EROFS_COMPRESSION_LZMA) { + if (sbi->lzma_dict_size == 0) + return (EINTEGRITY); + } + rq.in = src; + rq.inputsize = srclen; + return (decompressor->decompress(&rq)); +} diff --git a/decompressor_deflate.c b/decompressor_deflate.c new file mode 100644 index 0000000..8a4bd05 --- /dev/null +++ b/decompressor_deflate.c @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal DEFLATE decompressor for EROFS FreeBSD */ +#include +#include +#include +#include + +#include "compress.h" + +struct z_erofs_deflate_ctx { + struct erofs_stream_ctx pool; + z_stream stream; + bool initialized; +}; + +_Static_assert(sizeof(struct z_erofs_deflate_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Deflate stream wrapper exceeds UMA item"); + +static voidpf +z_erofs_deflate_alloc(voidpf opaque, uInt items, uInt size) +{ + struct erofs_stream_ctx *pool; + size_t bytes; + + pool = opaque; + if (__builtin_mul_overflow((size_t)items, (size_t)size, &bytes)) { + pool->allocation_failed = true; + return (NULL); + } + return (z_erofs_stream_ctx_alloc(pool, bytes)); +} + +static void +z_erofs_deflate_free(voidpf opaque, voidpf address) +{ + + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_load_deflate_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_deflate_cfgs *deflate; + + (void)dsb; + if (size < sizeof(*deflate)) + return (EINTEGRITY); + deflate = data; + if (deflate->windowbits < 8 || deflate->windowbits > 15) + return (EOPNOTSUPP); + sbi->deflate_windowbits = deflate->windowbits; + return (0); +} + +static int +z_erofs_deflate_error(int ret) +{ + + switch (ret) { + case Z_MEM_ERROR: + return (ENOMEM); + case Z_VERSION_ERROR: + return (EOPNOTSUPP); + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_BUF_ERROR: + return (EINTEGRITY); + default: + return (EIO); + } +} + +static int +z_erofs_deflate_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + int ret; + + ctx = (struct z_erofs_deflate_ctx *)pool; + bzero(&ctx->stream, sizeof(ctx->stream)); + ctx->stream.zalloc = z_erofs_deflate_alloc; + ctx->stream.zfree = z_erofs_deflate_free; + ctx->stream.opaque = pool; + ret = inflateInit2(&ctx->stream, -pool->sbi->deflate_windowbits); + if (ret != Z_OK) + return (z_erofs_deflate_error(ret)); + ctx->initialized = true; + return (0); +} + +static void +z_erofs_deflate_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + + ctx = (struct z_erofs_deflate_ctx *)pool; + if (ctx->initialized) { + (void)inflateEnd(&ctx->stream); + ctx->initialized = false; + } +} + +static int +z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret, + uInt avail_in) +{ + + if (rq->partial_decoding) + return (0); + if (ret != Z_STREAM_END || avail_in != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_deflate_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_deflate_ctx *ctx; + z_stream *strm; + uInt in_before, out_before; + int error, ret; + + if (rq->sbi->deflate_windowbits < 8 || + rq->sbi->deflate_windowbits > MAX_WBITS) + return (EOPNOTSUPP); + if (rq->inputsize > (size_t)(uInt)-1 || + rq->outputsize > (size_t)(uInt)-1) + return (EOVERFLOW); + if (rq->outputsize == 0) + return (EINTEGRITY); + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_DEFLATE, + sizeof(*ctx), z_erofs_deflate_ctx_init, z_erofs_deflate_ctx_fini, + &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_deflate_ctx *)pool; + strm = &ctx->stream; + pool->allocation_failed = false; + ret = inflateReset2(strm, -rq->sbi->deflate_windowbits); + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + z_erofs_stream_ctx_put(pool, false); + return (error); + } + strm->next_in = __DECONST(void *, rq->in); + strm->avail_in = rq->inputsize; + strm->next_out = rq->out; + strm->avail_out = rq->outputsize; + + error = 0; + ret = Z_OK; + while (strm->avail_out != 0) { + in_before = strm->avail_in; + out_before = strm->avail_out; + ret = inflate(strm, Z_SYNC_FLUSH); + if (ret == Z_STREAM_END) + break; + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + break; + } + if (strm->avail_in == in_before && strm->avail_out == out_before) { + error = EINTEGRITY; + break; + } + } + if (error == 0 && strm->avail_out != 0) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_deflate_finish(rq, ret, strm->avail_in); + if (pool->allocation_failed) + error = ENOMEM; + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, + .supports_subextent = 1, + .name = "deflate", +}; diff --git a/decompressor_lz4.c b/decompressor_lz4.c new file mode 100644 index 0000000..39bd897 --- /dev/null +++ b/decompressor_lz4.c @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal LZ4 decompressor for EROFS FreeBSD */ +#include +#include +#include + +#include "compress.h" + +#define EROFS_LZ4_TOKEN_LITERAL_SHIFT 4 +#define EROFS_LZ4_TOKEN_MATCH_MASK 0x0f +#define EROFS_LZ4_MAX_RUN 15 +#define EROFS_LZ4_EXT_SENTINEL 255 +#define EROFS_LZ4_MIN_MATCH 4 +#define EROFS_LZ4_OFFSET_BYTES 2 + +static int +lz4_finish(const uint8_t *ip, const uint8_t *iend, int partial) +{ + if (partial) + return (0); + while (ip < iend) { + if (*ip++ != 0) + return (EINTEGRITY); + } + return (0); +} + +int +z_erofs_lz4_decompress(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *ip, *iend; + uint8_t *op, *oend; + unsigned int token; + size_t length, copylen; + size_t offset; + + ip = rq->in; + iend = ip + rq->inputsize; + op = rq->out; + oend = op + rq->outputsize; + if (iend < ip || oend < op) + return (EINTEGRITY); + + while (ip < iend) { + token = *ip++; + length = token >> EROFS_LZ4_TOKEN_LITERAL_SHIFT; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (ip >= iend) + return (EINTEGRITY); + value = *ip++; + if (length > SIZE_MAX - value) + return (EINTEGRITY); + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > (size_t)(iend - ip)) + return (EINTEGRITY); + if (!rq->partial_decoding && length > (size_t)(oend - op)) + return (EINTEGRITY); + copylen = MIN(length, (size_t)(oend - op)); + memcpy(op, ip, copylen); + ip += length; + op += copylen; + if (op == oend) + return (lz4_finish(ip, iend, rq->partial_decoding)); + if (ip >= iend) + break; + if (ip + EROFS_LZ4_OFFSET_BYTES > iend) + return (EINTEGRITY); + offset = le16dec(ip); + ip += EROFS_LZ4_OFFSET_BYTES; + if (offset == 0 || offset > + (size_t)(op - (uint8_t *)rq->out)) + return (EINTEGRITY); + length = token & EROFS_LZ4_TOKEN_MATCH_MASK; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (ip >= iend) + return (EINTEGRITY); + value = *ip++; + if (length > SIZE_MAX - value) + return (EINTEGRITY); + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > SIZE_MAX - EROFS_LZ4_MIN_MATCH) + return (EINTEGRITY); + length += EROFS_LZ4_MIN_MATCH; + if (!rq->partial_decoding && length > (size_t)(oend - op)) + return (EINTEGRITY); + copylen = MIN(length, (size_t)(oend - op)); + while (copylen-- != 0) { + *op = *(op - offset); + ++op; + } + if (op == oend) + return (lz4_finish(ip, iend, rq->partial_decoding)); + } + return (op == oend ? + lz4_finish(ip, iend, rq->partial_decoding) : EINTEGRITY); +} diff --git a/decompressor_lzma.c b/decompressor_lzma.c new file mode 100644 index 0000000..2ce258a --- /dev/null +++ b/decompressor_lzma.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * EROFS MicroLZMA wrapper around FreeBSD's bundled XZ Embedded decoder. + * The decoder source is compiled with private symbol names because the + * stock xz.ko does not enable its optional MicroLZMA entry points. + */ + +#include +#include +#include + +#include "compress.h" + +#define XZ_DEC_MICROLZMA +#define xz_dec_lzma2_create erofs_xz_dec_lzma2_create +#define xz_dec_lzma2_reset erofs_xz_dec_lzma2_reset +#define xz_dec_lzma2_run erofs_xz_dec_lzma2_run +#define xz_dec_lzma2_end erofs_xz_dec_lzma2_end +#define xz_dec_microlzma_alloc erofs_xz_dec_microlzma_alloc +#define xz_dec_microlzma_reset erofs_xz_dec_microlzma_reset +#define xz_dec_microlzma_run erofs_xz_dec_microlzma_run +#define xz_dec_microlzma_end erofs_xz_dec_microlzma_end +#define xz_malloc erofs_xz_malloc +#define xz_free erofs_xz_free + +static void * +erofs_xz_malloc(unsigned long size) +{ + return (malloc(size, M_EROFS, M_NOWAIT)); +} + +static void +erofs_xz_free(void *ptr) +{ + free(ptr, M_EROFS); +} + +#include + +#undef bool +#undef false +#undef true +#undef min + +struct z_erofs_lzma_ctx { + struct erofs_stream_ctx pool; + struct xz_dec_microlzma *state; + size_t state_bytes; +}; + +_Static_assert(sizeof(struct z_erofs_lzma_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "LZMA stream wrapper exceeds UMA item"); + +static int +z_erofs_lzma_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + int error; + + ctx = (struct z_erofs_lzma_ctx *)pool; + ctx->state_bytes = sizeof(*ctx->state); + error = z_erofs_stream_ctx_charge(pool, ctx->state_bytes); + if (error != 0) + return (error); + ctx->state = xz_dec_microlzma_alloc(XZ_SINGLE, + pool->sbi->lzma_dict_size); + if (ctx->state == NULL) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + pool->allocation_failed = 1; + return (ENOMEM); + } + return (0); +} + +static void +z_erofs_lzma_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + + ctx = (struct z_erofs_lzma_ctx *)pool; + if (ctx->state != NULL) { + xz_dec_microlzma_end(ctx->state); + ctx->state = NULL; + } + if (ctx->state_bytes != 0) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + } +} + +static int +z_erofs_load_lzma_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lzma_cfgs *lzma; + uint32_t dict_size; + + (void)dsb; + if (size < sizeof(*lzma)) + return (EINTEGRITY); + lzma = data; + if (le16toh(lzma->format) != 0) + return (EOPNOTSUPP); + dict_size = le32toh(lzma->dict_size); + if (dict_size < 4096) + return (EINTEGRITY); + if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) + return (EOPNOTSUPP); + sbi->lzma_dict_size = dict_size; + return (0); +} + +static int +z_erofs_lzma_error(enum xz_ret ret) +{ + + switch (ret) { + case XZ_MEM_ERROR: + return (ENOMEM); + case XZ_MEMLIMIT_ERROR: + case XZ_OPTIONS_ERROR: + case XZ_UNSUPPORTED_CHECK: + return (EOPNOTSUPP); + default: + return (EINTEGRITY); + } +} + +static int +z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret, + size_t input_pos) +{ + + if (rq->partial_decoding && + (ret == XZ_OK || ret == XZ_STREAM_END)) + return (0); + if (!rq->partial_decoding && ret == XZ_STREAM_END && + input_pos == rq->inputsize) + return (0); + return (z_erofs_lzma_error(ret)); +} + +static int +z_erofs_lzma_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_lzma_ctx *ctx; + struct xz_buf buffer; + enum xz_ret ret; + int error; + + if (rq->inputsize > UINT32_MAX || rq->outputsize > UINT32_MAX) + return (EOVERFLOW); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_LZMA, + sizeof(*ctx), z_erofs_lzma_ctx_init, z_erofs_lzma_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_lzma_ctx *)pool; + bzero(&buffer, sizeof(buffer)); + buffer.in = rq->in; + buffer.in_size = rq->inputsize; + buffer.out = rq->out; + buffer.out_size = rq->outputsize; + xz_dec_microlzma_reset(ctx->state, (uint32_t)rq->inputsize, + (uint32_t)rq->outputsize, !rq->partial_decoding); + ret = xz_dec_microlzma_run(ctx->state, &buffer); + if (buffer.out_pos != rq->outputsize) + error = EINTEGRITY; + else + error = z_erofs_lzma_finish(rq, ret, buffer.in_pos); + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_lzma_decomp = { + .config = z_erofs_load_lzma_config, + .decompress = z_erofs_lzma_decompress, + .supports_subextent = 1, + .name = "lzma", +}; diff --git a/decompressor_zstd.c b/decompressor_zstd.c new file mode 100644 index 0000000..ee6f7ac --- /dev/null +++ b/decompressor_zstd.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal zstd decompressor for EROFS FreeBSD */ +#include +#include +#include +#include + +#include "compress.h" + +#ifdef ZSTDIO +#define ZSTD_STATIC_LINKING_ONLY +#include + +struct z_erofs_zstd_ctx { + struct erofs_stream_ctx pool; + ZSTD_DCtx *dctx; +}; + +_Static_assert(sizeof(struct z_erofs_zstd_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Zstd stream wrapper exceeds UMA item"); +#endif + +static bool +erofs_zstd_available(void) +{ +#ifdef ZSTDIO + return (true); +#else + return (false); +#endif +} + +static int +z_erofs_load_zstd_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_zstd_cfgs *zstd; + + (void)dsb; + if (!erofs_zstd_available()) { + vfs_mount_error(sbi->mnt, + "erofs: ZSTD compression requires ZSTDIO support"); + return (EOPNOTSUPP); + } + if (size < sizeof(*zstd)) + return (EINTEGRITY); + zstd = data; + if (zstd->format != 0 || zstd->windowlog > 10) + return (EOPNOTSUPP); + sbi->zstd_windowlog = zstd->windowlog; + return (0); +} + +#ifdef ZSTDIO +static void * +zstd_alloc(void *opaque, size_t size) +{ + return (z_erofs_stream_ctx_alloc(opaque, size)); +} + +static void +zstd_free(void *opaque, void *address) +{ + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_zstd_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + ZSTD_customMem alloc; + size_t ret; + + ctx = (struct z_erofs_zstd_ctx *)pool; + alloc = (ZSTD_customMem) { + .customAlloc = zstd_alloc, + .customFree = zstd_free, + .opaque = pool, + }; + ctx->dctx = ZSTD_createDCtx_advanced(alloc); + if (ctx->dctx == NULL) + return (ENOMEM); + ret = ZSTD_DCtx_setParameter(ctx->dctx, ZSTD_d_windowLogMax, + pool->sbi->zstd_windowlog + 10); + if (ZSTD_isError(ret)) + return (pool->allocation_failed ? ENOMEM : EOPNOTSUPP); + return (0); +} + +static void +z_erofs_zstd_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + + ctx = (struct z_erofs_zstd_ctx *)pool; + if (ctx->dctx != NULL) { + (void)ZSTD_freeDCtx(ctx->dctx); + ctx->dctx = NULL; + } +} + +static int +z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret, + size_t input_pos, size_t input_size) +{ + + if (rq->partial_decoding) + return (0); + if (ret != 0 || input_pos != input_size) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_zstd_ctx *ctx; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t in_before, out_before, ret; + int error; + + if (rq->sbi->zstd_windowlog + 10 > 20) + return (EOPNOTSUPP); + if (rq->outputsize == 0) + return (EINTEGRITY); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_ZSTD, + sizeof(*ctx), z_erofs_zstd_ctx_init, z_erofs_zstd_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_zstd_ctx *)pool; + pool->allocation_failed = false; + + input = (ZSTD_inBuffer) { + .src = rq->in, + .size = rq->inputsize, + }; + output = (ZSTD_outBuffer) { + .dst = rq->out, + .size = rq->outputsize, + }; + error = 0; + ret = 1; + while (output.pos != output.size) { + in_before = input.pos; + out_before = output.pos; + ret = ZSTD_decompressStream(ctx->dctx, &output, &input); + if (ZSTD_isError(ret)) { + error = pool->allocation_failed ? ENOMEM : EINTEGRITY; + break; + } + if (input.pos == in_before && output.pos == out_before) { + error = EINTEGRITY; + break; + } + if (ret == 0) + break; + } + if (error == 0 && output.pos != output.size) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_zstd_finish(rq, ret, input.pos, input.size); + z_erofs_stream_ctx_put(pool, error == 0 && !rq->partial_decoding); + return (error); +} +#else +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + (void)rq; + return (EOPNOTSUPP); +} +#endif + +const struct z_erofs_decompressor z_erofs_zstd_decomp = { + .config = z_erofs_load_zstd_config, + .decompress = z_erofs_zstd_decompress, + .supports_subextent = 0, + .name = "zstd", +}; diff --git a/dir.c b/dir.c new file mode 100644 index 0000000..21d9352 --- /dev/null +++ b/dir.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* Map EROFS directory entry file type to FreeBSD dirent.d_type. */ +static unsigned char +erofs_ftype_to_dtype(uint8_t ftype) +{ + switch (ftype) { + case EROFS_FT_REG_FILE: + return (DT_REG); + case EROFS_FT_DIR: + return (DT_DIR); + case EROFS_FT_CHRDEV: + return (DT_CHR); + case EROFS_FT_BLKDEV: + return (DT_BLK); + case EROFS_FT_FIFO: + return (DT_FIFO); + case EROFS_FT_SOCK: + return (DT_SOCK); + case EROFS_FT_SYMLINK: + return (DT_LNK); + default: + return (DT_UNKNOWN); + } +} + +static int +erofs_dirname_order(const char *left, size_t leftlen, const char *right, + size_t rightlen) +{ + size_t common; + int order; + + common = MIN(leftlen, rightlen); + order = memcmp(left, right, common); + if (order != 0) + return (order); + if (leftlen == rightlen) + return (0); + return (leftlen < rightlen ? -1 : 1); +} + +/* Validate one name slot and return its Linux-visible length. */ +int +erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp) +{ + size_t namelen, span; + + if (endoff <= nameoff) + return (EINTEGRITY); + span = endoff - nameoff; + if (trailing) { + namelen = strnlen(blk + nameoff, span); + } else { + namelen = span; + if (memchr(blk + nameoff, '\0', span) != NULL) + return (EINTEGRITY); + } + if (namelen == 0 || namelen > EROFS_NAME_LEN) + return (EINTEGRITY); + for (size_t i = 0; i < namelen; i++) { + if (blk[nameoff + i] == '/') + return (EINTEGRITY); + } + *namelenp = namelen; + return (0); +} + +/* Validate the dirent array, name offsets, and names in a block. */ +int +erofs_validate_dirblock(const char *blk, uint32_t blksz, uint32_t maxsize, + uint32_t *ndirentsp) +{ + const struct erofs_dirent *de; + uint32_t endoff, first_nameoff, idx, nameoff, ndirents, prev_nameoff; + size_t namelen, prev_namelen; + int error; + + if (blksz < sizeof(struct erofs_dirent) || + maxsize < sizeof(struct erofs_dirent) || + maxsize > blksz) + return (EINTEGRITY); + de = (const struct erofs_dirent *)blk; + first_nameoff = le16toh(de[0].nameoff); + if (first_nameoff < sizeof(struct erofs_dirent) || + first_nameoff >= maxsize || + (first_nameoff % sizeof(struct erofs_dirent)) != 0) + return (EINTEGRITY); + ndirents = first_nameoff / sizeof(struct erofs_dirent); + prev_nameoff = 0; + prev_namelen = 0; + for (idx = 0; idx < ndirents; idx++) { + nameoff = le16toh(de[idx].nameoff); + if ((idx == 0 && nameoff != first_nameoff) || + (idx != 0 && nameoff <= prev_nameoff) || + nameoff < first_nameoff || nameoff >= maxsize) + return (EINTEGRITY); + endoff = idx + 1 < ndirents ? + le16toh(de[idx + 1].nameoff) : maxsize; + if (endoff <= nameoff || endoff > maxsize) + return (EINTEGRITY); + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + if (idx != 0 && erofs_dirname_order(blk + prev_nameoff, + prev_namelen, blk + nameoff, namelen) >= 0) + return (EINTEGRITY); + prev_nameoff = nameoff; + prev_namelen = namelen; + } + *ndirentsp = ndirents; + return (0); +} + +/* Extract name, nid, and type of the idx'th directory entry from a block. */ +static int +erofs_dirent_name(const char *blk, uint32_t maxsize, + uint32_t idx, uint32_t ndirents, char *name, size_t namesz, erofs_nid_t *nid, + uint8_t *ftype, size_t *namelenp) +{ + const struct erofs_dirent *de; + uint32_t nameoff, endoff; + size_t namelen; + int error; + + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[idx].nameoff); + if (idx + 1 < ndirents) + endoff = le16toh(de[idx + 1].nameoff); + else + endoff = maxsize; + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + if (namelen >= namesz) + return (EINTEGRITY); + memcpy(name, blk + nameoff, namelen); + name[namelen] = '\0'; + *nid = le64toh(de[idx].nid); + *ftype = de[idx].file_type; + *namelenp = namelen; + return (0); +} + +static int +erofs_previous_dirname(struct erofs_sb_info *sbi, struct erofs_inode *dir, + erofs_off_t block_off, char *name, size_t namesz, size_t *namelenp) +{ + erofs_nid_t nid; + uint32_t maxsize, ndirents; + uint8_t ftype; + char *blk; + int error; + + block_off -= sbi->block_size; + maxsize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, maxsize, (void **)&blk); + if (error != 0) + return (error); + error = erofs_validate_dirblock(blk, sbi->block_size, maxsize, + &ndirents); + if (error == 0) + error = erofs_dirent_name(blk, maxsize, ndirents - 1, + ndirents, name, namesz, &nid, &ftype, namelenp); + erofs_brelse(blk); + return (error); +} + +/* Per-call state for readdir dirent/cookie output. */ +struct erofs_uiodir { + struct dirent *dirent; + uint64_t *cookies; + uint64_t last_cookie; + int ncookies; + int acookies; + int eofflag; +}; + +enum erofs_uiodir_result { + EROFS_UIODIR_BUFFER_FULL = -1, + EROFS_UIODIR_OK = 0, +}; + +/* Push a dirent and its cookie to the caller, modelled after UDF. */ +static int +erofs_uiodir(struct erofs_uiodir *uiodir, int de_size, struct uio *uio, + uint64_t cookie) +{ + int error; + + if (cookie <= uiodir->last_cookie) + return (EINTEGRITY); + if (uio->uio_resid < de_size || + (uiodir->cookies != NULL && + uiodir->acookies >= uiodir->ncookies)) { + return (EROFS_UIODIR_BUFFER_FULL); + } + error = uiomove(uiodir->dirent, de_size, uio); + if (error != 0) + return (error); + uiodir->last_cookie = cookie; + if (uiodir->cookies != NULL) + uiodir->cookies[uiodir->acookies++] = cookie; + return (EROFS_UIODIR_OK); +} + +/* + * Process directory entries within a single block and output them to uio. + * (Linux equivalent: erofs_fill_dentries in Linux's dir.c) + * + * Returns 0 on success (all entries consumed), -1 if uio is full, or a + * positive error code on corruption. + */ +static int +erofs_fill_dentries(struct erofs_sb_info *sbi, struct erofs_uiodir *uiodir, + struct uio *uio, struct dirent *d, const char *blk, uint32_t maxsize, + uint32_t start_idx, uint32_t ndirents, erofs_off_t block_off, + uint64_t *logical_offp) +{ + char name[EROFS_NAME_LEN + 1]; + uint32_t idx; + uint64_t curpos, nextoff; + erofs_nid_t nid; + size_t namelen; + uint8_t ftype; + int error; + + for (idx = start_idx; idx < ndirents; idx++) { + curpos = block_off + idx * sizeof(struct erofs_dirent); + nextoff = (idx + 1 < ndirents) ? + (curpos + sizeof(struct erofs_dirent)) : + (block_off + maxsize); + error = erofs_dirent_name(blk, maxsize, idx, ndirents, name, + sizeof(name), &nid, &ftype, &namelen); + if (error != 0) + return (error); + if (!erofs_nid_is_valid(sbi, nid)) + return (EINTEGRITY); + bzero(d, sizeof(*d)); + d->d_fileno = nid; + d->d_type = erofs_ftype_to_dtype(ftype); + d->d_namlen = namelen; + d->d_reclen = GENERIC_DIRSIZ(d); + d->d_off = nextoff; + strlcpy(d->d_name, name, sizeof(d->d_name)); + error = erofs_uiodir(uiodir, d->d_reclen, uio, d->d_off); + if (error != 0) + return (error); + *logical_offp = nextoff; + uio->uio_offset = *logical_offp; + } + return (0); +} + +/* + * Read directory contents and output a FreeBSD dirent stream to uio. + * + * Key points: + * - On-disk entries use their logical file offsets as cookies; + * - A dot_omitted directory appends a synthetic "." at i_size, matching + * Linux, so existing on-disk cookies are not shifted; + * - The dirent array occupies only the front portion of a block, so after + * scanning all entries offset must jump to maxsize (the block end), + * otherwise the loop would get stuck on the same block; + * - Supports a_ncookies / a_cookies for NFS and other callers that need + * resumable iteration. + */ +int +erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies) +{ + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct erofs_uiodir uiodir; + struct dirent d; + uint64_t *cookiebuf; + size_t cookie_count; + char first_name[EROFS_NAME_LEN + 1]; + char previous_name[EROFS_NAME_LEN + 1]; + char *blk; + erofs_off_t block_off; + uint64_t logical_off; + uint32_t block_pos, blksz, ndirents, start_idx, maxsize; + erofs_nid_t edge_nid; + size_t first_namelen, previous_namelen; + uint8_t edge_ftype; + bool have_previous, sequential; + int error; + + dir = VTOE(vp); + sbi = MTOE(vp->v_mount); + blksz = sbi->block_size; + error = 0; + cookiebuf = NULL; + cookie_count = 0; + have_previous = false; + uiodir.eofflag = 0; + uiodir.acookies = 0; + uiodir.dirent = &d; + uiodir.cookies = NULL; + uiodir.ncookies = 0; + if (cookies != NULL && ncookies != NULL) { + *cookies = NULL; + *ncookies = 0; + if (uio->uio_resid > 0) { + cookie_count = (size_t)uio->uio_resid / + GENERIC_MINDIRSIZ; + cookie_count = MIN(cookie_count, (size_t)INT_MAX); + cookie_count = MIN(cookie_count, + SIZE_MAX / sizeof(*cookiebuf)); + } + uiodir.ncookies = (int)cookie_count; + if (cookie_count != 0) + cookiebuf = malloc(sizeof(*cookiebuf) * cookie_count, + M_TEMP, M_WAITOK); + uiodir.cookies = cookiebuf; + } + + if (uio->uio_offset < 0) { + error = EINVAL; + goto out; + } + if (dir->dot_omitted && dir->size == (uint64_t)OFF_MAX) { + error = EINTEGRITY; + goto out; + } + + logical_off = uio->uio_offset; + sequential = logical_off == 0; + uiodir.last_cookie = logical_off; + uio->uio_offset = logical_off; + + while (logical_off < dir->size) { + block_off = rounddown2(logical_off, (uint64_t)blksz); + maxsize = MIN((uint64_t)blksz, dir->size - block_off); + block_pos = logical_off - block_off; + if ((block_pos % sizeof(struct erofs_dirent)) != 0) { + block_pos = roundup(block_pos, sizeof(struct erofs_dirent)); + logical_off = block_off + block_pos; + uio->uio_offset = logical_off; + } + error = erofs_read_data_readahead(sbi, dir, block_off, + maxsize, sequential, (void **)&blk); + if (error != 0) + goto out; + error = erofs_validate_dirblock(blk, blksz, maxsize, &ndirents); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + if (!have_previous && block_off != 0) { + error = erofs_previous_dirname(sbi, dir, block_off, + previous_name, sizeof(previous_name), + &previous_namelen); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + have_previous = true; + } + error = erofs_dirent_name(blk, maxsize, 0, ndirents, + first_name, sizeof(first_name), &edge_nid, &edge_ftype, + &first_namelen); + if (error == 0 && have_previous && + erofs_dirname_order(previous_name, previous_namelen, + first_name, first_namelen) >= 0) + error = EINTEGRITY; + if (error == 0) + error = erofs_dirent_name(blk, maxsize, ndirents - 1, + ndirents, previous_name, sizeof(previous_name), + &edge_nid, &edge_ftype, &previous_namelen); + if (error != 0) { + erofs_brelse(blk); + goto out; + } + have_previous = true; + start_idx = block_pos / sizeof(struct erofs_dirent); + if (start_idx >= ndirents) { + logical_off = block_off + maxsize; + uio->uio_offset = logical_off; + erofs_brelse(blk); + continue; + } + error = erofs_fill_dentries(sbi, &uiodir, uio, &d, blk, maxsize, + start_idx, ndirents, block_off, &logical_off); + erofs_brelse(blk); + if (error != 0) + goto out; + } + if (dir->dot_omitted && logical_off == dir->size) { + bzero(&d, sizeof(d)); + d.d_fileno = dir->nid; + d.d_type = DT_DIR; + d.d_namlen = 1; + d.d_reclen = GENERIC_DIRSIZ(&d); + d.d_off = dir->size + 1; + d.d_name[0] = '.'; + d.d_name[1] = '\0'; + error = erofs_uiodir(&uiodir, d.d_reclen, uio, d.d_off); + if (error != 0) + goto out; + logical_off++; + uio->uio_offset = logical_off; + } + uiodir.eofflag = 1; +out: + if (error == EROFS_UIODIR_BUFFER_FULL) + error = 0; + if (eofflag != NULL && error == 0) + *eofflag = uiodir.eofflag; + if (cookies != NULL && ncookies != NULL) { + if (error != 0) { + if (cookiebuf != NULL) + free(cookiebuf, M_TEMP); + } else { + *ncookies = uiodir.acookies; + *cookies = cookiebuf; + } + } + return (error); +} diff --git a/erofs_fs.h b/erofs_fs.h new file mode 100644 index 0000000..378af2c --- /dev/null +++ b/erofs_fs.h @@ -0,0 +1,496 @@ +/* SPDX-License-Identifier: MIT */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +#include +#include + +/* FreeBSD compatibility - Linux-style little-endian types */ +#ifndef __le16 +typedef uint16_t __le16; +typedef uint32_t __le32; +typedef uint64_t __le64; +typedef uint8_t __u8; +#endif + +/* to allow for x86 boot sectors and other oddities. */ +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 +#define EROFS_FEATURE_COMPAT_XATTR_FILTER 0x00000004 +#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX 0x00000008 +#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX 0x00000010 +#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS 0x00000020 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 +#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 +#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES 0x00000040 +#define EROFS_FEATURE_INCOMPAT_48BIT 0x00000080 +#define EROFS_FEATURE_INCOMPAT_METABOX 0x00000100 + +#define EROFS_ALL_FEATURE_INCOMPAT \ + (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | EROFS_FEATURE_INCOMPAT_48BIT | \ + EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ + EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES | \ + EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \ + EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ + EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \ + EROFS_FEATURE_INCOMPAT_FRAGMENTS | \ + EROFS_FEATURE_INCOMPAT_METABOX) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +/* Device table slot (128 bytes) */ +struct erofs_deviceslot { + uint8_t tag[64]; + __le32 blocks_lo; + __le32 uniaddr_lo; + __le16 blocks_hi; + __le16 uniaddr_hi; + uint8_t reserved[52]; +} __packed; +#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) + +/* erofs on-disk super block (currently 144 bytes at maximum) */ +struct erofs_super_block { + __le32 magic; + __le32 checksum; + __le32 feature_compat; + uint8_t blkszbits; + uint8_t sb_extslots; + union { + __le16 rootnid_2b; + __le16 blocks_hi; + } __packed rb; + __le64 inos; + __le64 epoch; + __le32 fixed_nsec; + __le32 blocks_lo; + __le32 meta_blkaddr; + __le32 xattr_blkaddr; + uint8_t uuid[16]; + uint8_t volume_name[16]; + __le32 feature_incompat; + union { + __le16 available_compr_algs; + __le16 lz4_max_distance; + } __packed u1; + __le16 extra_devices; + __le16 devt_slotoff; + uint8_t dirblkbits; + uint8_t xattr_prefix_count; + __le32 xattr_prefix_start; + __le64 packed_nid; + uint8_t xattr_filter_reserved; + uint8_t ishare_xattr_prefix_id; + uint8_t reserved[2]; + __le32 build_time; + __le64 rootnid_8b; + __le64 reserved2; + __le64 metabox_nid; + __le64 reserved3; +} __packed; + +/* EROFS inode datalayout (i_format in on-disk inode) */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_COMPRESSED_FULL = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_COMPRESSED_COMPACT = 3, + EROFS_INODE_CHUNK_BASED = 4, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool +erofs_inode_is_data_compressed(unsigned int datamode) +{ + return (datamode == EROFS_INODE_COMPRESSED_FULL || + datamode == EROFS_INODE_COMPRESSED_COMPACT); +} + +/* bit definitions of inode i_format */ +#define EROFS_I_VERSION_MASK 0x01 +#define EROFS_I_DATALAYOUT_MASK 0x07 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_NLINK_1_BIT 4 /* non-directory compact inodes only */ +#define EROFS_I_DOT_OMITTED_BIT 4 /* (directories) omit the `.` dirent */ +#define EROFS_I_ALL ((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1) + +/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS_MASK 0x001F +/* with chunk indexes or just a 4-byte block array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 +#define EROFS_CHUNK_FORMAT_48BIT 0x0040 +#define EROFS_CHUNK_FORMAT_ALL ((EROFS_CHUNK_FORMAT_48BIT << 1) - 1) + +/* 32-byte and 64-byte on-disk inode record layouts. */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +#define EROFS_INODE_LAYOUT_EXTENDED 1 +#define EROFS_INODE_LAYOUT_PLAIN EROFS_INODE_FLAT_PLAIN + +struct erofs_inode_chunk_info { + __le16 format; + __le16 reserved; +} __packed; + +union erofs_inode_i_u { + __le32 blocks_lo; + __le32 startblk_lo; + __le32 rdev; + struct erofs_inode_chunk_info c; +}; + +union erofs_inode_i_nb { + __le16 nlink; /* if EROFS_I_NLINK_1_BIT is unset */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 startblk_hi; /* starting block number MSB */ +} __packed; + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le32 i_size; + __le32 i_mtime; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved; +} __packed; + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le64 i_size; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le32 i_uid; + __le32 i_gid; + __le64 i_mtime; + __le32 i_mtime_nsec; + __le32 i_nlink; + uint8_t i_reserved2[16]; +} __packed; + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_name_filter; /* bit value 1 indicates not-present */ + uint8_t h_shared_count; + uint8_t h_reserved2[7]; + __le32 h_shared_xattrs[]; /* shared xattr id array */ +} __packed; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* + * bit 7 of e_name_index is set when it refers to a long xattr name prefix, + * while the remained lower bits represent the index of the prefix. + */ +#define EROFS_XATTR_LONG_PREFIX 0x80 +#define EROFS_XATTR_LONG_PREFIX_MASK 0x7f + +#define EROFS_XATTR_FILTER_BITS 32 +#define EROFS_XATTR_FILTER_DEFAULT UINT32_MAX +#define EROFS_XATTR_FILTER_SEED 0x25BBE08F + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + uint8_t e_name_len; + uint8_t e_name_index; + __le16 e_value_size; + char e_name[]; /* attribute name */ +} __packed; + +/* long xattr name prefix */ +struct erofs_xattr_long_prefix { + uint8_t base_index; /* short xattr name prefix index */ + char infix[]; /* infix apart from short prefix */ +} __packed; + +static inline unsigned int +erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + return (sizeof(struct erofs_xattr_ibody_header) + + sizeof(uint32_t) * (le16toh(i_xattr_icount) - 1)); +} + +#define EROFS_XATTR_ALIGN(size) \ + (((size) + sizeof(struct erofs_xattr_entry) - 1) & \ + ~(sizeof(struct erofs_xattr_entry) - 1)) + +static inline unsigned int +erofs_xattr_entry_size(const struct erofs_xattr_entry *entry) +{ + return (EROFS_XATTR_ALIGN( + sizeof(*entry) + entry->e_name_len + le16toh(entry->e_value_size))); +} + +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR ((uint64_t)-1) + +/* 4-byte block address array */ +#define EROFS_BLOCK_MAP_ENTRY_SIZE sizeof(__le32) + +/* 8-byte inode chunk index */ +struct erofs_inode_chunk_index { + __le16 startblk_hi; + __le16 device_id; + __le32 startblk_lo; +} __packed; + +#define EROFS_DIRENT_NID_METABOX_BIT 63 +#define EROFS_DIRENT_NID_METABOX \ + (1ULL << EROFS_DIRENT_NID_METABOX_BIT) +#define EROFS_DIRENT_NID_MASK \ + ((1ULL << EROFS_DIRENT_NID_METABOX_BIT) - 1) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; + __le16 nameoff; + uint8_t file_type; + uint8_t reserved; +} __packed; + +/* file type definitions in directory entries */ +#define EROFS_FT_UNKNOWN 0 +#define EROFS_FT_REG_FILE 1 +#define EROFS_FT_DIR 2 +#define EROFS_FT_CHRDEV 3 +#define EROFS_FT_BLKDEV 4 +#define EROFS_FT_FIFO 5 +#define EROFS_FT_SOCK 6 +#define EROFS_FT_SYMLINK 7 + +#define EROFS_NAME_LEN 255 + +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) +#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) + +/* compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_LZMA = 1, + Z_EROFS_COMPRESSION_DEFLATE = 2, + Z_EROFS_COMPRESSION_ZSTD = 3, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS ((1 << Z_EROFS_COMPRESSION_MAX) - 1) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + uint8_t reserved[10]; +} __packed; + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lzma_cfgs { + __le32 dict_size; + __le16 format; + uint8_t reserved[8]; +} __packed; + +#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE) + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_deflate_cfgs { + uint8_t windowbits; + uint8_t reserved[5]; +} __packed; + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_zstd_cfgs { + uint8_t format; + uint8_t windowlog; + uint8_t reserved[4]; +} __packed; + +#define Z_EROFS_ZSTD_MAX_DICT_SIZE Z_EROFS_PCLUSTER_MAX_SIZE + +/* z_advise flags */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +#define Z_EROFS_ADVISE_EXTENTS 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 +#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 + +#define Z_EROFS_FRAGMENT_INODE_BIT 7 + +struct z_erofs_map_header { + union { + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + __le16 h_idata_size; + }; + __le32 h_extents_lo; + }; + __le16 h_advise; + union { + struct { + uint8_t h_algorithmtype; + uint8_t h_clusterbits; + } __packed; + __le16 h_extents_hi; + } __packed; +} __packed; + +/* Logical cluster types */ +enum { + Z_EROFS_LCLUSTER_TYPE_PLAIN = 0, + Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1, + Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3, + Z_EROFS_LCLUSTER_TYPE_MAX +}; + +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) +#define Z_EROFS_LI_PARTIAL_REF (1 << 15) +#define Z_EROFS_LI_D0_CBLKCNT (1 << 11) + +/* Compression extent index structures */ +struct z_erofs_lcluster_index { + __le16 di_advise; + __le16 di_clusterofs; + union { + __le32 blkaddr; + __le16 delta[2]; + } di_u; +} __packed; + +#define Z_EROFS_MAP_HEADER_END(end) \ + (roundup2((end), 8) + sizeof(struct z_erofs_map_header)) +#define Z_EROFS_FULL_INDEX_START(end) (Z_EROFS_MAP_HEADER_END(end) + 8) + +#define Z_EROFS_EXTENT_PLEN_PARTIAL (1U << 27) +#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 +#define Z_EROFS_EXTENT_PLEN_MASK \ + ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) +struct z_erofs_extent { + __le32 plen; + __le32 pstart_lo; + __le32 pstart_hi; + __le32 lstart_lo; + __le32 lstart_hi; + uint8_t reserved[12]; +} __packed; + +static inline unsigned int +z_erofs_extent_recsize(unsigned int advise) +{ + return (4U << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & + Z_EROFS_ADVISE_EXTRECSZ_MASK)); +} + +_Static_assert(sizeof(struct erofs_deviceslot) == 128, + "EROFS device slot ABI size"); +_Static_assert(sizeof(struct erofs_super_block) == 144, + "EROFS super block ABI size"); +_Static_assert(sizeof(struct erofs_inode_compact) == 32, + "EROFS compact inode ABI size"); +_Static_assert(sizeof(struct erofs_inode_extended) == 64, + "EROFS extended inode ABI size"); +_Static_assert(sizeof(struct erofs_xattr_ibody_header) == 12, + "EROFS xattr ibody header ABI size"); +_Static_assert(sizeof(struct erofs_xattr_entry) == 4, + "EROFS xattr entry ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_info) == 4, + "EROFS chunk info ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_index) == 8, + "EROFS chunk index ABI size"); +_Static_assert(sizeof(struct z_erofs_map_header) == 8, + "EROFS zmap header ABI size"); +_Static_assert(sizeof(struct z_erofs_lcluster_index) == 8, + "EROFS lcluster index ABI size"); +_Static_assert(sizeof(struct z_erofs_extent) == 32, + "EROFS compression extent ABI size"); +_Static_assert(sizeof(struct erofs_dirent) == 12, + "EROFS dirent ABI size"); +_Static_assert(sizeof(struct erofs_deviceslot) == EROFS_DEVT_SLOT_SIZE, + "EROFS device slot ABI macro"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, extra_devices) == 86, + "EROFS extra device count ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, devt_slotoff) == 88, + "EROFS device table slot offset ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, rootnid_8b) == 112, + "EROFS 48-bit root nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, metabox_nid) == 128, + "EROFS metabox nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_compact, i_u) == 16, + "EROFS compact inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_u) == 16, + "EROFS extended inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_reserved2) == 48, + "EROFS extended inode reserved ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_chunk_index, device_id) == 2, + "EROFS chunk device id ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_chunk_index, startblk_lo) == 4, + "EROFS chunk start block ABI offset"); +_Static_assert(__builtin_offsetof(struct z_erofs_map_header, h_advise) == 4, + "EROFS zmap advise ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_lo) == 64, + "EROFS device blocks ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_lo) == 68, + "EROFS device unified address ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_hi) == 72, + "EROFS device blocks high ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_hi) == 74, + "EROFS device unified address high ABI offset"); + +#endif diff --git a/erofs_vnops.c b/erofs_vnops.c new file mode 100644 index 0000000..83b176d --- /dev/null +++ b/erofs_vnops.c @@ -0,0 +1,570 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "xattr.h" + +static vop_inactive_t erofs_inactive; +static vop_reclaim_t erofs_reclaim; + +static vop_readdir_t erofs_readdir; +static vop_readlink_t erofs_readlink; + +static vop_open_t erofs_open; +static vop_read_t erofs_read; +static vop_bmap_t erofs_bmap; + +static vop_getattr_t erofs_getattr; +static vop_setattr_t erofs_setattr; +static vop_access_t erofs_access; +static vop_pathconf_t erofs_pathconf; + +static vop_getextattr_t erofs_getextattr; +static vop_listextattr_t erofs_listextattr; +static vop_deleteextattr_t erofs_deleteextattr; +static vop_setextattr_t erofs_setextattr; +static vop_getacl_t erofs_vop_getacl; +static vop_aclcheck_t erofs_aclcheck; +static vop_setacl_t erofs_setacl; + +/* vop_fhtovp removed in FreeBSD 15.0 */ +static vop_vptofh_t erofs_vptofh; + +/* Access check: data nodes are read-only, but device/FIFO nodes are not denied + * writes. */ +static int +erofs_access(struct vop_access_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct acl *acl; + accmode_t accmode; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + accmode = ap->a_accmode; + if ((accmode & VMODIFY_PERMS) != 0) { + switch (vp->v_type) { + case VDIR: + case VLNK: + case VREG: + return (EROFS); + default: + break; + } + } + error = vfs_unixify_accmode(&accmode); + if (error != 0) + return (error); + if ((vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); + + acl = acl_alloc(M_WAITOK); + error = erofs_get_acl(vp, ACL_TYPE_ACCESS, acl); + if (error == 0) + error = vaccess_acl_posix1e(vp->v_type, vi->uid, vi->gid, acl, + accmode, ap->a_cred); + acl_free(acl); + return (error); +} + +/* + * Tell the generic pager that EROFS does not provide block-level bmap. + * + * Returning EOPNOTSUPP prevents the pager from assuming a bufobj/strategy is + * available, avoiding “No strategy for buffer” errors. VM will correctly + * fall back to the VOP_READ-based page-in path. + */ +static int +erofs_bmap(struct vop_bmap_args *ap) +{ + (void)ap; + return (EOPNOTSUPP); +} + +/* No dirty writeback on last ref release, so inactive is a no-op. */ +static int +erofs_inactive(struct vop_inactive_args *ap) +{ + (void)ap; + return (0); +} + +/* + * Create VM object when opening a regular vnode. + * + * The FreeBSD local vnode pager services synchronous and asynchronous faults + * through VOP_READ without requiring a block strategy method. + */ +static int +erofs_open(struct vop_open_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (VN_ISDEV(vp)) + return (EOPNOTSUPP); + if (vp->v_type == VREG) + vnode_create_vobject(vp, vi->size, ap->a_td); + return (0); +} + +static int +erofs_getattr(struct vop_getattr_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct erofs_sb_info *sbi; + struct vattr *vap; + + vp = ap->a_vp; + vi = VTOE(vp); + sbi = MTOE(vp->v_mount); + vap = ap->a_vap; + VATTR_NULL(vap); + vap->va_type = vp->v_type; + vap->va_mode = vi->mode & ALLPERMS; + vap->va_nlink = vi->nlink; + vap->va_uid = vi->uid; + vap->va_gid = vi->gid; + vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; + vap->va_fileid = vi->nid; + vap->va_size = vi->size; + vap->va_blocksize = sbi->block_size; + vap->va_atime.tv_sec = vi->mtime; + vap->va_mtime.tv_sec = vi->mtime; + vap->va_ctime.tv_sec = vi->mtime; + vap->va_atime.tv_nsec = vi->mtime_nsec; + vap->va_mtime.tv_nsec = vi->mtime_nsec; + vap->va_ctime.tv_nsec = vi->mtime_nsec; + vap->va_gen = vi->generation; + vap->va_flags = 0; + vap->va_rdev = VN_ISDEV(vp) ? vi->rdev : NODEV; + if (vi->data_blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + vap->va_bytes = vi->data_blocks << sbi->blkszbits; + vap->va_filerev = 0; + return (0); +} + +/* + * Read-only xattr get entry point. + * + * Delegates to erofs_getxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.* / security.*) + */ +static int +erofs_getextattr(struct vop_getextattr_args *ap) +{ + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + if (ap->a_name == NULL || ap->a_name[0] == '\0') + return (EINVAL); + if (strlen(ap->a_name) > EXTATTR_MAXNAMELEN) + return (EINVAL); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_getxattr(ap->a_vp, ap->a_attrnamespace, ap->a_name, + ap->a_uio, ap->a_size)); +} + +/* + * Read-only xattr list entry point. + * + * Delegates to erofs_listxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.* / security.*) + */ +static int +erofs_listextattr(struct vop_listextattr_args *ap) +{ + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_listxattr(ap->a_vp, ap->a_attrnamespace, ap->a_uio, + ap->a_size)); +} + +static int +erofs_deleteextattr(struct vop_deleteextattr_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +static int +erofs_setextattr(struct vop_setextattr_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +/* EROFS is read-only; mutations on regular files/dirs/symlinks are denied, size + * changes on special vnodes are treated as no-ops per read-only convention. */ +static int +erofs_setattr(struct vop_setattr_args *ap) +{ + struct vnode *vp; + struct vattr *vap; + + vp = ap->a_vp; + vap = ap->a_vap; + if (vap->va_mode != (mode_t)VNOVAL || vap->va_uid != (uid_t)VNOVAL || + vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL || + vap->va_atime.tv_nsec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || + vap->va_mtime.tv_nsec != VNOVAL || vap->va_flags != VNOVAL) + return (EROFS); + if (vap->va_size != VNOVAL) { + switch (vp->v_type) { + case VDIR: + return (EISDIR); + case VLNK: + case VREG: + return (EROFS); + case VCHR: + case VBLK: + case VSOCK: + case VFIFO: + case VNON: + case VBAD: + case VMARKER: + return (0); + } + } + return (0); +} + +static int +erofs_read(struct vop_read_args *ap) +{ + switch (ap->a_vp->v_type) { + case VREG: + return (erofs_read_file(ap->a_vp, ap->a_uio, ap->a_ioflag)); + case VDIR: + return (EISDIR); + default: + return (EINVAL); + } +} + +static int +erofs_readdir(struct vop_readdir_args *ap) +{ + if (ap->a_vp->v_type != VDIR) + return (ENOTDIR); + return (erofs_readdir_block(ap->a_vp, ap->a_uio, ap->a_eofflag, + ap->a_ncookies, ap->a_cookies)); +} + +static int +erofs_readlink(struct vop_readlink_args *ap) +{ + if (ap->a_vp->v_type != VLNK) + return (EINVAL); + return (erofs_readlink_target(ap->a_vp, ap->a_uio)); +} + +static int +erofs_pathconf(struct vop_pathconf_args *ap) +{ + switch (ap->a_name) { + case _PC_NAME_MAX: + *ap->a_retval = EROFS_NAME_LEN; + return (0); + case _PC_PATH_MAX: + *ap->a_retval = PATH_MAX; + return (0); + case _PC_FILESIZEBITS: + *ap->a_retval = 64; + return (0); + case _PC_LINK_MAX: + *ap->a_retval = INT_MAX; + return (0); + case _PC_CHOWN_RESTRICTED: + case _PC_NO_TRUNC: + *ap->a_retval = 1; + return (0); + case _PC_ACL_EXTENDED: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? 1 : 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? + ACL_MAX_ENTRIES : 3; + return (0); + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); + default: + return (vop_stdpathconf(ap)); + } +} + +static int +erofs_vop_getacl(struct vop_getacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + return (erofs_get_acl(ap->a_vp, ap->a_type, ap->a_aclp)); +} + +static int +erofs_aclcheck(struct vop_aclcheck_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + if (ap->a_aclp == NULL) + return (EINVAL); + switch (ap->a_type) { + case ACL_TYPE_ACCESS: + break; + case ACL_TYPE_DEFAULT: + if (ap->a_vp->v_type != VDIR) + return (EINVAL); + break; + default: + return (EINVAL); + } + return (acl_posix1e_check(ap->a_aclp)); +} + +static int +erofs_setacl(struct vop_setacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} + +static u_int +erofs_vfs_hash(erofs_nid_t nid) +{ + + return (fnv_32_buf(&nid, sizeof(nid), FNV1_32_INIT)); +} + +static int +erofs_vfs_hash_cmp(struct vnode *vp, void *pnid) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + return (vi == NULL || vi->nid != *(erofs_nid_t *)pnid); +} + +static void +erofs_fill_vnode(struct erofs_sb_info *sbi, struct vnode *vp, + const struct erofs_inode *vi) +{ + vp->v_type = vi->vtype; + if (vp->v_type == VFIFO) + vp->v_op = &erofs_fifoops; + if (vi->nid == sbi->root_nid) + vp->v_vflag |= VV_ROOT; +} + +/* + * Get vnode by raw on-disk nid. The raw nid is also the FreeBSD fileid and + * hash identity, so the metabox selector bit remains collision-free. + * Uses the standard FreeBSD vfs_hash API. + * (Linux equivalent: erofs_iget in Linux's inode.c) + */ +int +erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct thread *td; + struct vnode *vp; + erofs_nid_t nid; + u_int hash; + bool shared; + int error; + + td = curthread; + nid = (uint64_t)ino; + shared = (flags & LK_TYPE_MASK) == LK_SHARED; + hash = erofs_vfs_hash(nid); + error = vfs_hash_get(mp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + sbi = MTOE(mp); + vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO); + error = getnewvnode("erofs", mp, &erofs_vnodeops, &vp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + vp->v_data = vi; + vi->nid = nid; + lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); + error = insmntque(vp, mp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + error = vfs_hash_insert(vp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + error = erofs_read_inode(sbi, nid, vi); + if (error != 0) { + *vpp = NULL; + vgone(vp); + vput(vp); + return (error); + } + erofs_xattr_cache_init(vi); + erofs_fill_vnode(sbi, vp, vi); + vn_set_state(vp, VSTATE_CONSTRUCTED); + if (shared) + VOP_LOCK(vp, LK_DOWNGRADE); + *vpp = vp; + return (0); +} + +static int +erofs_reclaim(struct vop_reclaim_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (vi != NULL) { + vfs_hash_remove(vp); + erofs_xattr_cache_fini(MTOE(vp->v_mount), vi); + free(vi, M_EROFS); + vp->v_data = NULL; + } + return (0); +} + +/* Vnode pointer to persistent EROFS file handle. */ +static int +erofs_vptofh(struct vop_vptofh_args *ap) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + + vi = VTOE(ap->a_vp); + bzero(&efid, sizeof(efid)); + efid.len = sizeof(efid); + efid.nid_hi = vi->nid >> 32; + efid.nid_lo = vi->nid; + efid.gen = vi->generation; + memcpy(ap->a_fhp, &efid, sizeof(efid)); + return (0); +} + +struct vop_vector erofs_vnodeops = { + .vop_default = &default_vnodeops, + .vop_inactive = erofs_inactive, + .vop_reclaim = erofs_reclaim, + + .vop_lookup = vfs_cache_lookup, + .vop_cachedlookup = erofs_lookup, + .vop_readdir = erofs_readdir, + .vop_readlink = erofs_readlink, + + .vop_open = erofs_open, + .vop_read = erofs_read, + .vop_bmap = erofs_bmap, + .vop_getpages = vnode_pager_local_getpages, + .vop_getpages_async = vnode_pager_local_getpages_async, + + .vop_getattr = erofs_getattr, + .vop_setattr = erofs_setattr, + .vop_access = erofs_access, + .vop_pathconf = erofs_pathconf, + + .vop_getextattr = erofs_getextattr, + .vop_listextattr = erofs_listextattr, + .vop_deleteextattr = erofs_deleteextattr, + .vop_setextattr = erofs_setextattr, + .vop_getacl = erofs_vop_getacl, + .vop_aclcheck = erofs_aclcheck, + .vop_setacl = erofs_setacl, + + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_vnodeops); + +struct vop_vector erofs_fifoops = { + .vop_default = &fifo_specops, + .vop_access = erofs_access, + .vop_aclcheck = erofs_aclcheck, + .vop_deleteextattr = erofs_deleteextattr, + .vop_getacl = erofs_vop_getacl, + .vop_getextattr = erofs_getextattr, + .vop_getattr = erofs_getattr, + .vop_listextattr = erofs_listextattr, + .vop_pathconf = erofs_pathconf, + .vop_reclaim = erofs_reclaim, + .vop_setacl = erofs_setacl, + .vop_setattr = erofs_setattr, + .vop_setextattr = erofs_setextattr, + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_fifoops); diff --git a/inode.c b/inode.c new file mode 100644 index 0000000..36ba7da --- /dev/null +++ b/inode.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static uint64_t +erofs_addrmask(const struct erofs_sb_info *sbi) +{ + if (erofs_sb_has_48bit(sbi)) + return ((1ULL << 48) - 1); + return (UINT32_MAX); +} + +static dev_t +erofs_decode_dev(uint32_t dev) +{ + unsigned int major, minor; + + major = (dev & 0xfff00) >> 8; + minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); + return (makedev(major, minor)); +} + +static uint32_t +erofs_inode_generation(const struct erofs_sb_info *sbi, erofs_nid_t nid, + const void *inode, size_t inode_size) +{ + uint8_t encoded_nid[sizeof(nid)]; + uint32_t generation; + + le64enc(encoded_nid, nid); + generation = fnv_32_buf(encoded_nid, sizeof(encoded_nid), + sbi->generation_seed); + generation = fnv_32_buf(inode, inode_size, generation); + return (generation != 0 ? generation : 1); +} + +static int +erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds, + uint32_t nanoseconds) +{ + time_t mtime; + + if (nanoseconds >= 1000000000 || + __builtin_add_overflow(seconds, 0, &mtime)) + return (EINTEGRITY); + vi->mtime = mtime; + vi->mtime_nsec = nanoseconds; + return (0); +} + +static int +erofs_set_data_blocks(const struct erofs_sb_info *sbi, struct erofs_inode *vi, + uint64_t compressed_blocks) +{ + if (erofs_inode_is_data_compressed(vi->datalayout)) { + vi->data_blocks = compressed_blocks; + return (0); + } + if (vi->size == 0) { + vi->data_blocks = 0; + return (0); + } + if (vi->size > UINT64_MAX - (sbi->block_size - 1)) + return (EINTEGRITY); + vi->data_blocks = roundup2(vi->size, (uint64_t)sbi->block_size) >> + sbi->blkszbits; + return (0); +} + +static int +erofs_validate_inline_data(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + uint64_t image_size, inline_end, inline_off, inline_size, tail_start; + + if (vi->datalayout != EROFS_INODE_FLAT_INLINE || vi->size == 0) + return (0); + tail_start = roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size; + inline_size = vi->size - tail_start; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, &inline_off) || + __builtin_add_overflow(inline_off, vi->xattr_isize, &inline_off) || + __builtin_add_overflow(inline_off, inline_size, &inline_end)) + return (EINTEGRITY); + if ((inline_off & (sbi->block_size - 1)) + inline_size > sbi->block_size) + return (EINTEGRITY); + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || inline_end > sbi->metabox_en->size) + return (EINTEGRITY); + return (0); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + image_size = sbi->blocks << sbi->blkszbits; + if (inline_end > image_size || inline_end > sbi->dif0.mediasize) + return (EINTEGRITY); + return (0); +} + +/* + * Convert a logical nid to its inode-table byte offset. Normal NIDs are + * relative to the primary metadata area. For metabox NIDs, bit 63 selects + * the metabox backing inode and the remaining bits are relative to its data. + * EROFS_NULL_ADDR is returned when the address cannot be represented. + */ +static erofs_off_t +erofs_iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t meta_offset; + erofs_nid_t nid_lo; + erofs_off_t result; + bool in_metabox; + + in_metabox = erofs_nid_in_metabox(nid); + if (in_metabox && !erofs_sb_has_metabox(sbi)) + return (EROFS_NULL_ADDR); + nid_lo = nid & EROFS_DIRENT_NID_MASK; + if (nid_lo > (UINT64_MAX >> 5)) + return (EROFS_NULL_ADDR); + result = nid_lo << 5; + if (in_metabox) + return (result); + + if (sbi->blkszbits > 58) + return (EROFS_NULL_ADDR); + meta_offset = (uint64_t)sbi->meta_blkaddr << sbi->blkszbits; + if (result > UINT64_MAX - meta_offset) + return (EROFS_NULL_ADDR); + + return (meta_offset + result); +} + +/* + * Check that a NID can address at least one compact inode slot without + * crossing the declared primary image or metabox backing-file boundary. + */ +bool +erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t image_size, off; + + off = erofs_iloc(sbi, nid); + if (off == EROFS_NULL_ADDR) + return (false); + if (erofs_nid_in_metabox(nid)) { + if (sbi->metabox_en == NULL || off > sbi->metabox_en->size) + return (false); + return (sizeof(struct erofs_inode_compact) <= + sbi->metabox_en->size - off); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (false); + image_size = sbi->blocks << sbi->blkszbits; + if (off > image_size || sizeof(struct erofs_inode_compact) > + image_size - off) + return (false); + if (off > sbi->dif0.mediasize || sizeof(struct erofs_inode_compact) > + sbi->dif0.mediasize - off) + return (false); + return (true); +} + +bool +erofs_dirent_type_matches(uint8_t file_type, __enum_uint8(vtype) vtype) +{ + switch (file_type) { + case EROFS_FT_REG_FILE: + return (vtype == VREG); + case EROFS_FT_DIR: + return (vtype == VDIR); + case EROFS_FT_CHRDEV: + return (vtype == VCHR); + case EROFS_FT_BLKDEV: + return (vtype == VBLK); + case EROFS_FT_FIFO: + return (vtype == VFIFO); + case EROFS_FT_SOCK: + return (vtype == VSOCK); + case EROFS_FT_SYMLINK: + return (vtype == VLNK); + default: + return (true); + } +} + +/* + * Read and decode a disk inode. + * + * Currently supports: + * - compact / extended inode; + * - plain / inline uncompressed layouts; + * - basic 48-bit address parsing; + * - compact inode epoch/fixed_nsec timestamp semantics; + * - dot_omitted / nlink==1 i_format details. + */ +int +erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, struct erofs_inode *vi) +{ + struct erofs_inode_compact *dic; + struct erofs_inode_extended *die; + struct erofs_inode_chunk_info chunk_info; + union erofs_inode_i_nb inode_nb; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t addrmask; + int64_t mtime; + erofs_off_t off; + erofs_blk_t startblk; + uint64_t compressed_blocks; + uint32_t raw_rdev, startblk_lo; + uint16_t ifmt, startblk_hi; + int error; + + if (!erofs_nid_is_valid(sbi, nid)) + return (EINTEGRITY); + off = erofs_iloc(sbi, nid); + error = erofs_read_metadata(sbi, nid, off, + sizeof(struct erofs_inode_compact), &buf); + if (error != 0) + return (error); + + bzero(&vi->size, sizeof(*vi) - offsetof(struct erofs_inode, size)); + vi->nid = nid; + vi->inode_off = off; + ifmt = le16dec(buf.data); + if ((ifmt & ~EROFS_I_ALL) != 0) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + vi->compact_inode = (erofs_inode_version(ifmt) == 0); + if (!vi->compact_inode) { + erofs_put_metabuf(&buf); + error = erofs_read_metadata(sbi, nid, off, + sizeof(struct erofs_inode_extended), &buf); + if (error != 0) + return (error); + } + addrmask = erofs_addrmask(sbi); + startblk = EROFS_NULL_ADDR; + startblk_lo = 0; + startblk_hi = 0; + compressed_blocks = 0; + raw_rdev = 0; + bzero(&inode_nb, sizeof(inode_nb)); + dic = buf.data; + if (vi->compact_inode) { + vi->inode_isize = sizeof(struct erofs_inode_compact); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(dic->i_mode); + vi->size = le32toh(dic->i_size); + vi->uid = le16toh(dic->i_uid); + vi->gid = le16toh(dic->i_gid); + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + if (__builtin_add_overflow(sbi->epoch, + (int64_t)le32toh(dic->i_mtime), &mtime)) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + error = erofs_set_timestamp(vi, mtime, sbi->fixed_nsec); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + startblk_lo = le32toh(dic->i_u.startblk_lo); + compressed_blocks = le32toh(dic->i_u.blocks_lo); + raw_rdev = le32toh(dic->i_u.rdev); + if (!S_ISDIR(vi->mode) && + ((ifmt >> EROFS_I_NLINK_1_BIT) & 0x1) != 0) { + vi->nlink = 1; + inode_nb = dic->i_nb; + } else { + vi->nlink = le16toh(dic->i_nb.nlink); + addrmask = UINT32_MAX; + } + } else { + die = buf.data; + vi->inode_isize = sizeof(struct erofs_inode_extended); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(die->i_mode); + vi->size = le64toh(die->i_size); + vi->uid = le32toh(die->i_uid); + vi->gid = le32toh(die->i_gid); + vi->nlink = le32toh(die->i_nlink); + inode_nb = die->i_nb; + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + error = erofs_set_timestamp(vi, + (int64_t)le64toh(die->i_mtime), + le32toh(die->i_mtime_nsec)); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + startblk_lo = le32toh(die->i_u.startblk_lo); + compressed_blocks = le32toh(die->i_u.blocks_lo); + raw_rdev = le32toh(die->i_u.rdev); + } + startblk_hi = le16toh(inode_nb.startblk_hi); + compressed_blocks |= (uint64_t)le16toh(inode_nb.blocks_hi) << 32; + if (vi->size > (uint64_t)OFF_MAX) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + + vi->vtype = IFTOVT(vi->mode); + if (vi->mode != 0 && vi->vtype == VNON) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->vtype == VLNK) { + if (vi->size == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->size > MAXPATHLEN) { + erofs_put_metabuf(&buf); + return (ENAMETOOLONG); + } + } + vi->dot_omitted = (vi->vtype == VDIR) && + (((ifmt >> EROFS_I_DOT_OMITTED_BIT) & 0x1) != 0); + + if (erofs_inode_is_data_compressed(vi->datalayout)) { + error = z_erofs_fill_inode(sbi, vi); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + } else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + if (!erofs_sb_has_chunked_file(sbi)) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->compact_inode) + chunk_info = dic->i_u.c; + else + chunk_info = die->i_u.c; + if (le16toh(chunk_info.reserved) != 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->chunkformat = le16toh(chunk_info.format); + if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 && + (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->chunkbits = sbi->blkszbits + + (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); + if (vi->chunkbits >= 64) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + } else if (vi->datalayout != EROFS_INODE_FLAT_PLAIN && + vi->datalayout != EROFS_INODE_FLAT_INLINE) { + erofs_put_metabuf(&buf); + return (EOPNOTSUPP); + } + + switch (vi->vtype) { + case VREG: + case VDIR: + case VLNK: + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + } + startblk = startblk_lo | ((uint64_t)startblk_hi << 32); + if (vi->datalayout == EROFS_INODE_FLAT_PLAIN && + ((startblk ^ EROFS_NULL_ADDR) & addrmask) == 0) + startblk = EROFS_NULL_ADDR; + vi->startblk = startblk; + vi->rdev = NODEV; + break; + case VCHR: + case VBLK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = erofs_decode_dev(raw_rdev); + break; + case VFIFO: + case VSOCK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + default: + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + error = erofs_set_data_blocks(sbi, vi, compressed_blocks); + if (error == 0) + error = erofs_validate_inline_data(sbi, vi); + if (error == 0 && vi->vtype == VLNK) + error = erofs_validate_symlink_target(sbi, vi); + if (error != 0) { + erofs_put_metabuf(&buf); + return (error); + } + + erofs_put_metabuf(&buf); + return (0); +} diff --git a/internal.h b/internal.h new file mode 100644 index 0000000..4ba3f29 --- /dev/null +++ b/internal.h @@ -0,0 +1,423 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#ifndef __EROFS_INTERNAL_H +#define __EROFS_INTERNAL_H + +#include +#include // MUST FIRST +#include +#include +#include +#include +#include +#include +#include + +#include "erofs_fs.h" + +MALLOC_DECLARE(M_EROFS); + +struct cdev; +struct g_consumer; +struct erofs_device_info; + +/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ +#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 + +typedef uint64_t erofs_nid_t; +typedef uint64_t erofs_off_t; +typedef uint64_t erofs_blk_t; + +#define EROFS_MOUNT_POSIX_ACL 0x00000020 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +struct erofs_sb_info_opts { + unsigned int mount_opt; +}; + +struct erofs_device_info { + struct vnode *devvp; + struct cdev *dev; + struct g_consumer *cp; + uint64_t mediasize; + uint32_t sectorsize; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +struct erofs_xattr_prefix_item { + uint8_t base_index; + uint8_t infix_len; + char *infix; +}; + +struct erofs_map_blocks { + erofs_off_t m_pa, m_la; + uint64_t m_plen, m_llen; + + unsigned short m_deviceid; + uint8_t m_algorithmformat; + unsigned int m_flags; +}; + +enum erofs_zextent_cache_state { + EROFS_ZCACHE_EMPTY, + EROFS_ZCACHE_INFLIGHT, + EROFS_ZCACHE_READY, + EROFS_ZCACHE_FAILED, +}; + +struct erofs_zextent_cache_metrics { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + size_t resident_bytes; +}; + +struct erofs_zextent_cache { + struct erofs_map_blocks map; + void *data; + erofs_nid_t nid; + size_t decoded_size; + size_t charged_bytes; + size_t budget_bytes; + uint64_t minimum_decode_work; + struct erofs_zextent_cache_metrics metrics[Z_EROFS_COMPRESSION_MAX]; + unsigned int waiters; + int error; + enum erofs_zextent_cache_state state; + struct cv cv; + bool closing; +}; + +enum erofs_xattr_cache_state { + EROFS_XATTR_CACHE_EMPTY, + EROFS_XATTR_CACHE_INFLIGHT, + EROFS_XATTR_CACHE_READY, + EROFS_XATTR_CACHE_FAILED, +}; + +struct erofs_xattr_cache { + struct mtx lock; + struct cv cv; + void *data; + size_t size; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_xattr_cache_state state; + bool closing; + bool initialized; +}; + +#define EROFS_STREAM_CTX_WRAPPER_SIZE 256 + +struct erofs_stream_ctx; +typedef int erofs_stream_ctx_init_t(struct erofs_stream_ctx *ctx); +typedef void erofs_stream_ctx_fini_t(struct erofs_stream_ctx *ctx); + +struct erofs_stream_ctx { + STAILQ_ENTRY(erofs_stream_ctx) link; + struct erofs_sb_info *sbi; + erofs_stream_ctx_fini_t *fini; + size_t charged_bytes; + uint8_t algorithm; + bool cached; + bool allocation_failed; +}; + +STAILQ_HEAD(erofs_stream_ctx_head, erofs_stream_ctx); + +struct erofs_stream_pool_codec { + struct erofs_stream_ctx_head idle; + unsigned int contexts; + unsigned int cached; + unsigned int borrowed; + unsigned int idle_count; +}; + +struct erofs_stream_pool { + struct erofs_stream_pool_codec codec[Z_EROFS_COMPRESSION_MAX]; + size_t resident_bytes; + struct cv cv; + bool closing; +}; + +struct erofs_sb_info { + struct mount *mnt; + struct erofs_device_info dif0; + + uint32_t block_size; + uint32_t sb_size; + uint8_t blkszbits; + uint32_t meta_blkaddr; + uint32_t xattr_blkaddr; + uint32_t xattr_prefix_start; + uint8_t xattr_prefix_count; + uint8_t xattr_filter_reserved; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + struct erofs_inode *metabox_en; + struct erofs_inode *packed_inode; + struct erofs_xattr_prefix_item *xattr_prefixes; + erofs_blk_t blocks; + uint64_t inos; + erofs_nid_t root_nid; + int64_t epoch; + uint32_t fixed_nsec; + uint32_t generation_seed; + uint32_t feature_compat; + uint32_t feature_incompat; + char volume_name[17]; + + struct erofs_sb_info_opts opt; + uint16_t available_compr_algs; + uint32_t lzma_dict_size; + uint8_t deflate_windowbits; + uint8_t zstd_windowlog; + + /* Device table */ + uint16_t extra_devices; + uint16_t device_id_mask; + bool flatdev; + erofs_blk_t total_blocks; + erofs_blk_t flatdev_blocks; + struct erofs_device_info *devs; + struct mtx z_extent_cache_lock; + struct erofs_zextent_cache z_extent_cache; + LIST_ENTRY(erofs_sb_info) z_extent_cache_link; + bool z_extent_cache_initialized; + struct erofs_stream_pool stream_pool; + LIST_ENTRY(erofs_sb_info) stream_pool_link; + bool stream_pool_initialized; + volatile u_long xattr_cache_resident; +}; + +#define MTOE(mp) ((struct erofs_sb_info *)(mp)->mnt_data) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ + static inline bool erofs_sb_has_##name(const struct erofs_sb_info *sbi) \ + { \ + return ( \ + (sbi->feature_##compat & EROFS_FEATURE_##feature) != 0); \ + } + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) +EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS) + +static inline bool +erofs_sb_has_xattr_filter_v1(const struct erofs_sb_info *sbi) +{ + return (erofs_sb_has_xattr_filter(sbi) && + sbi->xattr_filter_reserved == 0); +} + +struct erofs_inode { + erofs_nid_t nid; + uint64_t size; + uint64_t data_blocks; + /* Absolute device offset, or metabox-file offset when bit 63 is set. */ + erofs_off_t inode_off; + erofs_blk_t startblk; + uint32_t generation; + uint32_t nlink; + uid_t uid; + gid_t gid; + mode_t mode; + __enum_uint8(vtype) vtype; + dev_t rdev; + time_t mtime; + uint32_t mtime_nsec; + uint8_t datalayout; + uint8_t inode_isize; + uint32_t xattr_isize; + bool compact_inode; + bool dot_omitted; + /* Compression fields */ + uint16_t z_advise; + uint8_t z_algorithmtype[2]; + uint8_t z_lclusterbits; + uint16_t z_idata_size; + erofs_off_t z_fragmentoff; + uint64_t z_tailextent_headlcn; + uint64_t z_extents; + bool z_initialized; + /* Chunk-based fields */ + uint16_t chunkformat; + uint8_t chunkbits; + /* Fragment fields */ + bool fragment; + struct erofs_xattr_cache xattr_cache; +}; + +struct erofs_fid { + uint16_t len; + uint16_t pad; + uint32_t nid_hi; + uint32_t nid_lo; + uint32_t gen; +}; + +_Static_assert(sizeof(struct erofs_fid) == 16, + "EROFS file handle ABI must be 16 bytes"); +_Static_assert(sizeof(struct erofs_fid) <= sizeof(struct fid), + "struct erofs_fid must fit within struct fid"); + +#define VTOE(vp) ((struct erofs_inode *)(vp)->v_data) + +static inline unsigned int +erofs_inode_version(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK); +} + +static inline unsigned int +erofs_inode_datalayout(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK); +} + +static inline bool +erofs_nid_in_metabox(erofs_nid_t nid) +{ + return ((nid & EROFS_DIRENT_NID_METABOX) != 0); +} + +/* Allocated on disk at m_pa. */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata. */ +#define EROFS_MAP_META 0x0002 +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +#define EROFS_MAP_PARTIAL_REF 0x0008 +#define EROFS_MAP_FRAGMENT 0x0010 +#define EROFS_MAP_FULL(f) \ + (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) + +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +#define EROFS_GET_BLOCKS_READMORE 0x0002 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +_Static_assert(Z_EROFS_COMPRESSION_RUNTIME_MAX <= UINT8_MAX, + "algorithm format must fit in uint8_t"); + +struct erofs_map_dev { + struct erofs_device_info *m_dif; + + erofs_off_t m_pa; + unsigned int m_deviceid; + uint64_t m_plen; +}; + +struct erofs_buf { + void *data; + void (*release)(void *); +}; + +#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } + +/* Buffer and device I/O. */ +int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp); +int erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp); +void erofs_brelse(void *buf); +void erofs_put_metabuf(struct erofs_buf *buf); +int erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf); +/* Logical mapping and file data. */ +int erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int erofs_read_data_readahead(struct erofs_sb_info *sbi, + struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential, + void **bufp); +int erofs_read_file(struct vnode *vp, struct uio *uio, int ioflag); +int erofs_validate_symlink_target(struct erofs_sb_info *sbi, + struct erofs_inode *vi); +int erofs_readlink_target(struct vnode *vp, struct uio *uio); + +/* Inode and vnode lifecycle. */ +bool erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid); +bool erofs_dirent_type_matches(uint8_t file_type, + __enum_uint8(vtype) vtype); +int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi); +int erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp); +void erofs_xattr_cache_init(struct erofs_inode *vi); +void erofs_xattr_cache_fini(struct erofs_sb_info *sbi, + struct erofs_inode *vi); + +/* Directory operations. */ +int erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies); +int erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp); +int erofs_validate_dirblock(const char *blk, uint32_t blksz, uint32_t maxsize, + uint32_t *ndirentsp); +int erofs_lookup(struct vop_cachedlookup_args *ap); + +/* Compressed mapping and data. */ +int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi); +int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +void z_erofs_extent_cache_init(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_fini(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_init(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_fini(struct erofs_sb_info *sbi); +int z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp); +void z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable); +int z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes); +void *z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address); +int z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio); +int z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src, size_t srclen, + void *dst, size_t dstlen, bool partial); + +/* Compression configuration. */ +int z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb); + +/* VOP vectors. */ +extern struct vop_vector erofs_vnodeops; +extern struct vop_vector erofs_fifoops; + +#endif /* __EROFS_INTERNAL_H */ diff --git a/namei.c b/namei.c new file mode 100644 index 0000000..3079d7e --- /dev/null +++ b/namei.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct erofs_qstr { + const unsigned char *name; + const unsigned char *end; +}; + +/* + * Compare two directory entry names using an already-matched prefix. + * (Linux equivalent: erofs_dirnamecmp in Linux's namei.c) + * + * qn: search key (not necessarily null-terminated). + * qd: on-disk name range (may not be null-terminated). + * matched: in/out count of prefix characters already known to match. + * + * Returns 0 if equal, 1 if qn > qd, -1 if qn < qd. + */ +static int +erofs_dirnamecmp(const struct erofs_qstr *qn, const struct erofs_qstr *qd, + unsigned int *matched) +{ + size_t dname_span, qname_span; + unsigned int i; + + qname_span = qn->end - qn->name; + dname_span = qd->end - qd->name; + i = MIN(*matched, qname_span); + i = MIN(i, dname_span); + while (i < qname_span && i < dname_span && qd->name[i] != '\0') { + if (qn->name[i] != qd->name[i]) { + *matched = i; + return (qn->name[i] > qd->name[i] ? 1 : -1); + } + ++i; + } + *matched = i; + if (i == qname_span) + return (i == dname_span || qd->name[i] == '\0' ? 0 : -1); + return (1); +} + +static int +erofs_dirent_qstr(const char *data, uint32_t datasize, uint32_t index, + uint32_t ndirents, struct erofs_qstr *name) +{ + const struct erofs_dirent *de; + uint32_t endoff, nameoff; + size_t namelen; + int error; + + de = (const struct erofs_dirent *)data; + nameoff = le16toh(de[index].nameoff); + endoff = index + 1 < ndirents ? + le16toh(de[index + 1].nameoff) : datasize; + error = erofs_dirent_namelen(data, nameoff, endoff, + index + 1 == ndirents, &namelen); + if (error != 0) + return (error); + name->name = (const unsigned char *)data + nameoff; + name->end = name->name + namelen; + return (0); +} + +static int +erofs_read_dirblock(struct erofs_sb_info *sbi, struct erofs_inode *dir, + uint64_t block, char **datap, uint32_t *datasizep, uint32_t *ndirentsp) +{ + erofs_off_t block_off; + uint32_t datasize; + char *data; + int error; + + *datap = NULL; + if (__builtin_mul_overflow(block, (uint64_t)sbi->block_size, + &block_off) || block_off >= dir->size) + return (EINTEGRITY); + datasize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, datasize, (void **)&data); + if (error != 0) + return (error); + error = erofs_validate_dirblock(data, sbi->block_size, datasize, + ndirentsp); + if (error != 0) { + erofs_brelse(data); + return (error); + } + *datap = data; + *datasizep = datasize; + return (0); +} + +static int +erofs_dirblock_order(const char *left, uint32_t leftsize, + uint32_t leftents, const char *right, uint32_t rightsize, + uint32_t rightents) +{ + struct erofs_qstr leftname, rightname; + unsigned int matched; + int error; + + error = erofs_dirent_qstr(left, leftsize, leftents - 1, leftents, + &leftname); + if (error != 0) + return (error); + error = erofs_dirent_qstr(right, rightsize, 0, rightents, &rightname); + if (error != 0) + return (error); + matched = 0; + if (erofs_dirnamecmp(&leftname, &rightname, &matched) >= 0) + return (EINTEGRITY); + return (0); +} + +static int +erofs_validate_dirblock_neighbors(struct erofs_sb_info *sbi, + struct erofs_inode *dir, uint64_t block, const char *data, + uint32_t datasize, uint32_t ndirents) +{ + uint64_t lastblock; + uint32_t neighborsize, neighborents; + char *neighbor; + int error; + + lastblock = (dir->size - 1) / sbi->block_size; + if (block > 0) { + error = erofs_read_dirblock(sbi, dir, block - 1, &neighbor, + &neighborsize, &neighborents); + if (error != 0) + return (error); + error = erofs_dirblock_order(neighbor, neighborsize, + neighborents, data, datasize, ndirents); + erofs_brelse(neighbor); + if (error != 0) + return (error); + } + if (block < lastblock) { + error = erofs_read_dirblock(sbi, dir, block + 1, &neighbor, + &neighborsize, &neighborents); + if (error != 0) + return (error); + error = erofs_dirblock_order(data, datasize, ndirents, + neighbor, neighborsize, neighborents); + erofs_brelse(neighbor); + if (error != 0) + return (error); + } + return (0); +} + +/* + * Binary search within a directory block for the target name. + * + * Returns a pointer to the matching dirent, or NULL on miss. + */ +static struct erofs_dirent * +find_target_dirent(const struct erofs_qstr *name, char *data, + uint32_t datasize, uint32_t ndirents) +{ + uint32_t head, back; + unsigned int startprfx, endprfx; + struct erofs_dirent *const de = (struct erofs_dirent *)data; + + /* The 1st dirent has already been evaluated by the caller. */ + head = 1; + back = ndirents - 1; + startprfx = endprfx = 0; + + while (head <= back) { + const uint32_t mid = head + (back - head) / 2; + const uint32_t nameoff = le16toh(de[mid].nameoff); + unsigned int matched = MIN(startprfx, endprfx); + struct erofs_qstr dname = { + .name = (const unsigned char *)data + nameoff, + }; + + if (mid >= ndirents - 1) + dname.end = (const unsigned char *)data + datasize; + else + dname.end = (const unsigned char *)data + + le16toh(de[mid + 1].nameoff); + + /* String comparison without already matched prefix */ + int ret = erofs_dirnamecmp(name, &dname, &matched); + + if (ret == 0) + return (de + mid); + else if (ret > 0) { + head = mid + 1; + startprfx = matched; + } else { + back = mid - 1; + endprfx = matched; + } + } + + return (NULL); +} + +/* + * Find the directory block most likely to contain the target name. + * + * Uses two-level binary search: first across blocks, then within the + * candidate block via find_target_dirent(). + * + * Returns the block buffer on success (caller must erofs_brelse), + * or NULL on error. *_ndirents is set to the number of dirents in + * the returned block (0 means the first entry is the match). + * On error, *errorp is set to a positive errno. + */ +static char * +erofs_find_target_block(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, uint32_t *_ndirents, uint32_t *_datasize, + int *errorp) +{ + uint32_t bsz = sbi->block_size; + uint64_t head, back; + unsigned int startprfx = 0, endprfx = 0; + char *candidate = NULL; + int error; + + *errorp = 0; + *_ndirents = 0; + *_datasize = 0; + + if (dir->size == 0) + return (NULL); + + head = 0; + back = (dir->size - 1) / bsz; + + while (head <= back) { + const uint64_t mid = head + (back - head) / 2; + uint32_t maxsize; + const struct erofs_dirent *de; + char *blk; + int diff; + uint32_t ndirents; + uint32_t nameoff; + unsigned int matched; + struct erofs_qstr dname; + + error = erofs_read_dirblock(sbi, dir, mid, &blk, &maxsize, + &ndirents); + if (error != 0) { + *errorp = error; + goto out; + } + error = erofs_validate_dirblock_neighbors(sbi, dir, mid, blk, + maxsize, ndirents); + if (error != 0) { + erofs_brelse(blk); + *errorp = error; + goto out; + } + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[0].nameoff); + + matched = MIN(startprfx, endprfx); + dname.name = (const unsigned char *)blk + nameoff; + if (ndirents == 1) + dname.end = (const unsigned char *)blk + maxsize; + else + dname.end = (const unsigned char *)blk + + le16toh(de[1].nameoff); + + /* String comparison without already matched prefix */ + diff = erofs_dirnamecmp(name, &dname, &matched); + + if (diff < 0) { + erofs_brelse(blk); + if (mid == 0) + break; + back = mid - 1; + endprfx = matched; + continue; + } + + /* diff >= 0: this block is a candidate. */ + if (candidate != NULL) + erofs_brelse(candidate); + candidate = blk; + if (diff == 0) { + *_ndirents = 0; + *_datasize = maxsize; + return (candidate); + } + head = mid + 1; + startprfx = matched; + *_ndirents = ndirents; + *_datasize = maxsize; + } + return (candidate); +out: + if (candidate != NULL) + erofs_brelse(candidate); + return (NULL); +} + +/* + * Look up a name in a directory and return its nid and d_type. + * (Linux equivalent: erofs_namei in Linux's namei.c) + */ +static int +erofs_namei(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, erofs_nid_t *nid, uint8_t *d_type) +{ + int error; + uint32_t ndirents; + uint32_t datasize; + char *blk; + struct erofs_dirent *de; + + if (dir->size == 0) + return (ENOENT); + + blk = erofs_find_target_block(sbi, dir, name, &ndirents, + &datasize, &error); + if (blk == NULL) + return (error != 0 ? error : ENOENT); + + de = (struct erofs_dirent *)blk; + if (ndirents > 0) + de = find_target_dirent(name, blk, datasize, + ndirents); + + if (de != NULL) { + erofs_nid_t found_nid; + + found_nid = le64toh(de->nid); + if (!erofs_nid_is_valid(sbi, found_nid)) { + error = EINTEGRITY; + } else { + *nid = found_nid; + *d_type = de->file_type; + } + } + erofs_brelse(blk); + if (error != 0) + return (error); + return (de != NULL ? 0 : ENOENT); +} + +/* + * Directory name lookup (VOP_CACHEDLOOKUP entry point). + * + * FreeBSD-side API requirements: + * - "." must be returned under the caller's requested lock mode; + * - ".." must go through vn_vget_ino() to avoid holding a child lock while + * acquiring the parent directory lock in reverse; + * - Both hit and miss must correctly update the namecache. + */ +int +erofs_lookup(struct vop_cachedlookup_args *ap) +{ + struct erofs_qstr qname; + struct vnode *dvp, *vp; + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct componentname *cnp; + erofs_nid_t nid; + uint8_t dtype; + int error, ltype; + + dvp = ap->a_dvp; + cnp = ap->a_cnp; + *ap->a_vpp = NULL; + if ((cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) + return (EROFS); + if (cnp->cn_namelen < 0) + return (EINVAL); + if (cnp->cn_namelen > EROFS_NAME_LEN) + return (ENAMETOOLONG); + if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + + dir = VTOE(dvp); + sbi = MTOE(dvp->v_mount); + qname.name = (const unsigned char *)cnp->cn_nameptr; + qname.end = qname.name + cnp->cn_namelen; + error = erofs_namei(sbi, dir, &qname, &nid, &dtype); + if (error != 0) { + if (error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, NULL, cnp); + if (error == ENOENT && (cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == CREATE || cnp->cn_nameiop == RENAME)) + return (EROFS); + return (error); + } + if ((cnp->cn_flags & ISDOTDOT) == 0 && nid == dir->nid) + return (EINTEGRITY); + + if ((cnp->cn_flags & ISDOTDOT) != 0) + error = vn_vget_ino(dvp, nid, cnp->cn_lkflags, &vp); + else + error = erofs_vget(dvp->v_mount, nid, cnp->cn_lkflags, &vp); + if (error != 0) + return (error); + if ((cnp->cn_flags & ISDOTDOT) == 0 && + !erofs_dirent_type_matches(dtype, VTOE(vp)->vtype)) { + vput(vp); + return (EINTEGRITY); + } + *ap->a_vpp = vp; + if ((cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, vp, cnp); + return (0); +} diff --git a/super.c b/super.c new file mode 100644 index 0000000..8eaec97 --- /dev/null +++ b/super.c @@ -0,0 +1,1387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "xattr.h" + +MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem"); + +static const uint32_t erofs_crc32c_seed = 0x5045b54aU; + +#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024) +#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024) +#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024) +#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U) +#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U) +#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U) +#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U) + +static unsigned long erofs_stream_mount_budget = + EROFS_STREAM_MOUNT_HARD_BUDGET; +static unsigned long erofs_stream_global_budget = + EROFS_STREAM_GLOBAL_HARD_BUDGET; +static int erofs_stream_mount_contexts = + EROFS_STREAM_MOUNT_HARD_CONTEXTS; +static int erofs_stream_global_contexts_limit = + EROFS_STREAM_GLOBAL_HARD_CONTEXTS; +static int erofs_stream_mount_cached = + EROFS_STREAM_MOUNT_CACHED_PER_CODEC; +static int erofs_stream_global_cached = + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC; + +TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget", + &erofs_stream_mount_budget); +TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget", + &erofs_stream_global_budget); +TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts", + &erofs_stream_mount_contexts); +TUNABLE_INT("vfs.erofs.stream_pool.global_contexts", + &erofs_stream_global_contexts_limit); +TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec", + &erofs_stream_mount_cached); +TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec", + &erofs_stream_global_cached); + +static struct mtx erofs_stream_lock; +static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts = + LIST_HEAD_INITIALIZER(erofs_stream_mounts); +static uma_zone_t erofs_stream_zone; +static eventhandler_tag erofs_stream_lowmem_tag; +static size_t erofs_stream_global_resident; +static unsigned int erofs_stream_global_contexts; +static unsigned int erofs_stream_global_cached_by_codec[ + Z_EROFS_COMPRESSION_MAX]; +static unsigned long erofs_stream_cached_contexts; +static unsigned long erofs_stream_borrowed_contexts; +static unsigned long erofs_stream_idle_contexts; +static unsigned long erofs_stream_creations; +static unsigned long erofs_stream_reuses; +static unsigned long erofs_stream_fallbacks; +static unsigned long erofs_stream_exhaustions; +static unsigned long erofs_stream_allocation_failures; +static unsigned long erofs_stream_destroys; +static unsigned long erofs_stream_reclaims; + +MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool", + MTX_DEF); + +SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "EROFS filesystem"); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool, + CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD, + &erofs_stream_mount_budget, 0, "Effective per-mount byte budget"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD, + &erofs_stream_global_budget, 0, "Effective global byte budget"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD, + &erofs_stream_mount_contexts, 0, "Effective per-mount context limit"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit, + CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0, + "Effective global context limit"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD, + &erofs_stream_global_resident, 0, "Currently charged global bytes"); +SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD, + &erofs_stream_global_contexts, 0, "Currently allocated contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD, + &erofs_stream_cached_contexts, 0, "Currently cacheable contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD, + &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD, + &erofs_stream_idle_contexts, 0, "Currently idle contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD, + &erofs_stream_creations, 0, "Successful context creations"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD, + &erofs_stream_reuses, 0, "Idle context acquisitions"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD, + &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD, + &erofs_stream_exhaustions, 0, "Hard-limit allocation rejections"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, allocation_failures, + CTLFLAG_RD, &erofs_stream_allocation_failures, 0, + "UMA or malloc allocation failures"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD, + &erofs_stream_destroys, 0, "Destroyed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD, + &erofs_stream_reclaims, 0, "Low-memory idle context reclaims"); + +struct erofs_stream_alloc_header { + struct erofs_stream_ctx *ctx; + size_t bytes; +}; + +static bool +erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + if (bytes > erofs_stream_mount_budget || + pool->resident_bytes > erofs_stream_mount_budget - bytes || + bytes > erofs_stream_global_budget || + erofs_stream_global_resident > erofs_stream_global_budget - bytes) + return (false); + pool->resident_bytes += bytes; + erofs_stream_global_resident += bytes; + return (true); +} + +static void +erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + KASSERT(bytes <= sbi->stream_pool.resident_bytes && + bytes <= erofs_stream_global_resident, + ("erofs stream pool byte accounting underflow")); + sbi->stream_pool.resident_bytes -= bytes; + erofs_stream_global_resident -= bytes; +} + +int +z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + bool reserved; + + if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) { + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mtx_lock(&erofs_stream_lock); + reserved = ctx->charged_bytes <= SIZE_MAX - bytes && + erofs_stream_reserve_locked(ctx->sbi, bytes); + if (reserved) + ctx->charged_bytes += bytes; + else + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + if (!reserved) { + ctx->allocation_failed = true; + return (ENOMEM); + } + return (0); +} + +void +z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + + mtx_lock(&erofs_stream_lock); + KASSERT(bytes <= ctx->charged_bytes, + ("erofs stream context byte accounting underflow")); + ctx->charged_bytes -= bytes; + erofs_stream_release_locked(ctx->sbi, bytes); + mtx_unlock(&erofs_stream_lock); +} + +void * +z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_alloc_header *header; + size_t total; + + if (__builtin_add_overflow(bytes, sizeof(*header), &total) || + z_erofs_stream_ctx_charge(ctx, total) != 0) + return (NULL); + header = malloc(total, M_EROFS, M_NOWAIT | M_ZERO); + if (header == NULL) { + z_erofs_stream_ctx_uncharge(ctx, total); + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_allocation_failures; + mtx_unlock(&erofs_stream_lock); + return (NULL); + } + header->ctx = ctx; + header->bytes = total; + return (header + 1); +} + +void +z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address) +{ + struct erofs_stream_alloc_header *header; + size_t bytes; + + if (address == NULL) + return; + header = (struct erofs_stream_alloc_header *)address - 1; + KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch")); + bytes = header->bytes; + free(header, M_EROFS); + z_erofs_stream_ctx_uncharge(ctx, bytes); +} + +static void +erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_sb_info *sbi; + bool cached; + uint8_t algorithm; + + sbi = ctx->sbi; + algorithm = ctx->algorithm; + cached = ctx->cached; + ctx->fini(ctx); + KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE, + ("erofs stream backend allocation leaked")); + uma_zfree(erofs_stream_zone, ctx); + + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + KASSERT(codec->contexts != 0 && codec->borrowed != 0 && + erofs_stream_global_contexts != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream context accounting underflow")); + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + KASSERT(codec->cached != 0 && + erofs_stream_global_cached_by_codec[algorithm] != 0 && + erofs_stream_cached_contexts != 0, + ("erofs stream cache accounting underflow")); + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_destroys; + cv_broadcast(&sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); +} + +int +z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + bool cached; + int error; + + *ctxp = NULL; + if (algorithm == Z_EROFS_COMPRESSION_LZ4 || + algorithm >= Z_EROFS_COMPRESSION_MAX || + context_size < sizeof(*ctx) || + context_size > EROFS_STREAM_CTX_WRAPPER_SIZE) + return (EINVAL); + mtx_lock(&erofs_stream_lock); + if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) { + mtx_unlock(&erofs_stream_lock); + return (ENXIO); + } + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx != NULL) { + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + ++erofs_stream_reuses; + ctx->allocation_failed = false; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); + } + cached = codec->cached < (unsigned int)erofs_stream_mount_cached && + erofs_stream_global_cached_by_codec[algorithm] < + (unsigned int)erofs_stream_global_cached; + if (codec->contexts >= (unsigned int)erofs_stream_mount_contexts || + erofs_stream_global_contexts >= + (unsigned int)erofs_stream_global_contexts_limit || + !erofs_stream_reserve_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE)) { + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + ++codec->contexts; + ++codec->borrowed; + ++erofs_stream_global_contexts; + ++erofs_stream_borrowed_contexts; + if (cached) { + ++codec->cached; + ++erofs_stream_global_cached_by_codec[algorithm]; + ++erofs_stream_cached_contexts; + } else { + ++erofs_stream_fallbacks; + } + mtx_unlock(&erofs_stream_lock); + + ctx = uma_zalloc(erofs_stream_zone, M_NOWAIT | M_ZERO); + if (ctx == NULL) { + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_allocation_failures; + cv_broadcast(&sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + ctx->sbi = sbi; + ctx->fini = fini; + ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE; + ctx->algorithm = algorithm; + ctx->cached = cached; + error = init(ctx); + if (error != 0) { + erofs_stream_ctx_destroy(ctx); + return (error > 0 ? error : EIO); + } + mtx_lock(&erofs_stream_lock); + ++erofs_stream_creations; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); +} + +void +z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable) +{ + struct erofs_stream_pool_codec *codec; + + mtx_lock(&erofs_stream_lock); + codec = &ctx->sbi->stream_pool.codec[ctx->algorithm]; + if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) { + KASSERT(codec->borrowed != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream borrowed accounting underflow")); + --codec->borrowed; + --erofs_stream_borrowed_contexts; + STAILQ_INSERT_HEAD(&codec->idle, ctx, link); + ++codec->idle_count; + ++erofs_stream_idle_contexts; + cv_broadcast(&ctx->sbi->stream_pool.cv); + mtx_unlock(&erofs_stream_lock); + return; + } + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); +} + +static struct erofs_stream_ctx * +erofs_stream_take_idle_locked(struct erofs_sb_info *sbi) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx == NULL) + continue; + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + return (ctx); + } + return (NULL); +} + +static bool +erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool) +{ + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + if (pool->codec[algorithm].contexts != 0) + return (false); + } + return (true); +} + +void +z_erofs_stream_pool_init(struct erofs_sb_info *sbi) +{ + unsigned int algorithm; + + bzero(&sbi->stream_pool, sizeof(sbi->stream_pool)); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) + STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle); + cv_init(&sbi->stream_pool.cv, "erofs stream drain"); + mtx_lock(&erofs_stream_lock); + sbi->stream_pool_initialized = true; + LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link); + mtx_unlock(&erofs_stream_lock); +} + +void +z_erofs_stream_pool_fini(struct erofs_sb_info *sbi) +{ + struct erofs_stream_ctx *ctx; + + if (!sbi->stream_pool_initialized) + return; + mtx_lock(&erofs_stream_lock); + sbi->stream_pool.closing = true; + for (;;) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + } + while (!erofs_stream_pool_empty_locked(&sbi->stream_pool)) + cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock); + KASSERT(sbi->stream_pool.resident_bytes == 0, + ("erofs stream mount bytes remain at unmount")); + LIST_REMOVE(sbi, stream_pool_link); + sbi->stream_pool_initialized = false; + mtx_unlock(&erofs_stream_lock); + cv_destroy(&sbi->stream_pool.cv); +} + +static void +erofs_stream_pool_lowmem(void *arg, int howto) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *sbi; + + (void)arg; + (void)howto; + for (;;) { + ctx = NULL; + mtx_lock(&erofs_stream_lock); + LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL) { + ++erofs_stream_reclaims; + break; + } + } + mtx_unlock(&erofs_stream_lock); + if (ctx == NULL) + break; + erofs_stream_ctx_destroy(ctx); + } +} + +static void +erofs_stream_pool_global_init(void *arg) +{ + + (void)arg; + erofs_stream_mount_budget = MIN(erofs_stream_mount_budget, + EROFS_STREAM_MOUNT_HARD_BUDGET); + erofs_stream_global_budget = MIN(erofs_stream_global_budget, + EROFS_STREAM_GLOBAL_HARD_BUDGET); + erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts, + (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS)); + erofs_stream_global_contexts_limit = MAX(0, MIN( + erofs_stream_global_contexts_limit, + (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS)); + erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached, + (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC)); + erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached, + (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC)); + erofs_stream_zone = uma_zcreate("erofs stream ctx", + EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + KASSERT(erofs_stream_zone != NULL, ("cannot create erofs stream UMA zone")); + (void)uma_zone_set_max(erofs_stream_zone, + EROFS_STREAM_GLOBAL_HARD_CONTEXTS); + uma_zone_set_maxcache(erofs_stream_zone, + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3); + erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +erofs_stream_pool_global_fini(void *arg) +{ + + (void)arg; + if (erofs_stream_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag); + KASSERT(LIST_EMPTY(&erofs_stream_mounts), + ("erofs stream mounts remain at unload")); + KASSERT(erofs_stream_global_contexts == 0 && + erofs_stream_global_resident == 0 && + erofs_stream_borrowed_contexts == 0 && + erofs_stream_idle_contexts == 0 && + erofs_stream_cached_contexts == 0, + ("erofs stream resources remain at unload")); + uma_zdestroy(erofs_stream_zone); +} + +SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY, + erofs_stream_pool_global_init, NULL); +SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY, + erofs_stream_pool_global_fini, NULL); + +static const char *erofs_opts[] = { + "export", + "from", + NULL, +}; + +static vfs_mount_t erofs_mount; +static vfs_root_t erofs_root; +static vfs_statfs_t erofs_statfs; +static vfs_unmount_t erofs_unmount; +static vfs_fhtovp_t erofs_fhtovp; + +#define EROFS_DEVICE_OPT_PREFIX "device." + +struct erofs_device_arg { + uint16_t slot; + char *path; +}; + +static int +erofs_load_generation_seed(struct erofs_sb_info *sbi, uint32_t sb_size, + uint32_t *seedp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t seed; + int error; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sb_size, &buf); + if (error != 0) + return (error); + seed = fnv_32_buf(buf.data, sb_size, FNV1_32_INIT); + erofs_put_metabuf(&buf); + *seedp = seed != 0 ? seed : 1; + return (0); +} + +static void +erofs_free_device_args(struct erofs_device_arg *args, unsigned int count) +{ + unsigned int i; + + if (args == NULL) + return; + for (i = 0; i < count; ++i) + free(args[i].path, M_EROFS); + free(args, M_EROFS); +} + +static int +erofs_parse_device_slot(const char *name, uint16_t *slotp) +{ + const char *p; + unsigned int slot; + + if (strncmp(name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + return (ENOENT); + p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1; + if (*p < '1' || *p > '9') + return (EINVAL); + slot = 0; + for (; *p != '\0'; ++p) { + if (*p < '0' || *p > '9' || slot > (UINT16_MAX - (*p - '0')) / 10) + return (EINVAL); + slot = slot * 10 + (*p - '0'); + } + if (slot == 0 || slot > UINT16_MAX) + return (EINVAL); + *slotp = slot; + return (0); +} + +static int +erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp, + unsigned int *countp) +{ + struct erofs_device_arg *args; + struct vfsopt *opt; + char name[32]; + unsigned int count, i; + uint16_t slot; + int error; + + *argsp = NULL; + *countp = 0; + count = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + if (error != 0 || opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (count == UINT16_MAX) + return (E2BIG); + ++count; + } + if (count == 0) + return (0); + + args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO); + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + KASSERT(error == 0, ("validated EROFS device option changed")); + args[i].slot = slot; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + for (i = 0; i < count; ++i) { + snprintf(name, sizeof(name), EROFS_DEVICE_OPT_PREFIX "%u", + args[i].slot); + vfs_deleteopt(mp->mnt_optnew, name); + } + *argsp = args; + *countp = count; + return (0); +} + +static void +erofs_release_device_info(struct erofs_device_info *dif) +{ + if (dif->cp != NULL) { + g_topology_lock(); + g_vfs_close(dif->cp); + g_topology_unlock(); + dif->cp = NULL; + } + if (dif->devvp != NULL) { + vrele(dif->devvp); + dif->devvp = NULL; + } + if (dif->dev != NULL) { + dev_rel(dif->dev); + dif->dev = NULL; + } +} + +static bool +erofs_provider_is_duplicate(struct erofs_sb_info *sbi, struct g_provider *pp) +{ + unsigned int i; + + if (sbi == NULL) + return (false); + if (sbi->dif0.cp != NULL && sbi->dif0.cp->provider == pp) + return (true); + for (i = 0; i < sbi->extra_devices; ++i) { + if (sbi->devs[i].cp != NULL && sbi->devs[i].cp->provider == pp) + return (true); + } + return (false); +} + +static int +erofs_open_device(struct erofs_sb_info *sbi, const char *path, + struct erofs_device_info *dif) +{ + struct g_provider *pp; + struct nameidata nd; + struct vnode *devvp; + struct cdev *dev; + int error; + + bzero(dif, sizeof(*dif)); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path); + error = namei(&nd); + if (error != 0) + return (error); + devvp = nd.ni_vp; + NDFREE_PNBUF(&nd); + if (!vn_isdisk_error(devvp, &error)) { + vput(devvp); + return (error); + } + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread); + if (error != 0) + error = priv_check(curthread, PRIV_VFS_MOUNT_PERM); + if (error != 0) { + vput(devvp); + return (error); + } + dev = devvp->v_rdev; + dev_ref(dev); + g_topology_lock(); + pp = g_dev_getprovider(dev); + if (pp == NULL) + error = ENXIO; + else if (erofs_provider_is_duplicate(sbi, pp)) + error = EINVAL; + else + error = g_vfs_open(devvp, &dif->cp, "erofs", 0); + if (error == 0) { + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + } + g_topology_unlock(); + VOP_UNLOCK(devvp); + if (error != 0) { + dev_rel(dev); + vrele(devvp); + return (error); + } + dif->devvp = devvp; + dif->dev = dev; + if (dif->sectorsize == 0 || + (dif->sectorsize & (dif->sectorsize - 1)) != 0) { + erofs_release_device_info(dif); + return (EINVAL); + } + return (0); +} + +static void +erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif) +{ + u_long iosize; + + iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ? + dif->dev->si_iosize_max : MAXPHYS; + mp->mnt_iosize_max = MIN(mp->mnt_iosize_max, MIN(iosize, (u_long)MAXPHYS)); +} + +static void +erofs_free_dev_context(struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (sbi->devs != NULL) { + for (i = sbi->extra_devices; i > 0; --i) + erofs_release_device_info(&sbi->devs[i - 1]); + free(sbi->devs, M_EROFS); + } +} + +static void +erofs_drop_internal_inodes(struct erofs_sb_info *sbi) +{ + if (sbi->metabox_en != NULL) + free(sbi->metabox_en, M_EROFS); + if (sbi->packed_inode != NULL) + free(sbi->packed_inode, M_EROFS); +} + +static void +erofs_sb_free(struct erofs_sb_info *sbi) +{ + if (sbi == NULL) + return; + z_erofs_extent_cache_fini(sbi); + z_erofs_stream_pool_fini(sbi); + erofs_xattr_prefixes_cleanup(sbi); + erofs_drop_internal_inodes(sbi); + erofs_free_dev_context(sbi); + erofs_release_device_info(&sbi->dif0); + free(sbi, M_EROFS); +} + +static int +erofs_superblock_csum_verify(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t expected, crc; + size_t len; + int error; + + if (!erofs_sb_has_sb_chksum(sbi)) + return (0); + + len = 1u << dsb->blkszbits; + if (len > EROFS_SUPER_OFFSET) + len -= EROFS_SUPER_OFFSET; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, len, &buf); + if (error != 0) + return (error); + + crc = calculate_crc32c(erofs_crc32c_seed, + (const uint8_t *)buf.data + + offsetof(struct erofs_super_block, checksum) + + sizeof(dsb->checksum), + len - offsetof(struct erofs_super_block, checksum) - + sizeof(dsb->checksum)); + expected = le32toh(dsb->checksum); + erofs_put_metabuf(&buf); + + if (crc != expected) { + vfs_mount_error(sbi->mnt, + "erofs: invalid superblock checksum 0x%08x, " + "0x%08x expected", crc, expected); + return (EINTEGRITY); + } + return (0); +} + +static void +erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat, + uint64_t *blocks, erofs_nid_t *root_nid) +{ + *blocks = le32toh(dsb->blocks_lo); + if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 && + dsb->rootnid_8b != 0) { + *blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32; + *root_nid = le64toh(dsb->rootnid_8b); + } else { + *root_nid = le16toh(dsb->rb.rootnid_2b); + } +} + +static int +erofs_validate_device_size(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_blk_t blocks) +{ + uint64_t bytes; + + if (blocks == 0) + return (EINTEGRITY); + if (sbi->block_size < dif->sectorsize || + sbi->block_size % dif->sectorsize != 0) + return (EINVAL); + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + bytes = blocks << sbi->blkszbits; + if (bytes > dif->mediasize) + return (ENXIO); + return (0); +} + +static int +erofs_init_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + const char *path) +{ + struct erofs_device_info opened; + erofs_blk_t blocks, uniaddr; + int error; + + blocks = dif->blocks; + uniaddr = dif->uniaddr; + error = erofs_open_device(sbi, path, &opened); + if (error != 0) + return (error); + erofs_update_iosize_max(sbi->mnt, &opened); + opened.blocks = blocks; + opened.uniaddr = uniaddr; + *dif = opened; + return (erofs_validate_device_size(sbi, dif, dif->blocks)); +} + +static const char * +erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count, + unsigned int slot) +{ + unsigned int i; + + for (i = 0; i < count; ++i) { + if (args[i].slot == slot) + return (args[i].path); + } + return (NULL); +} + +static int +erofs_scan_devices(struct erofs_sb_info *sbi, const struct erofs_super_block *dsb, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_deviceslot *slots; + struct erofs_device_info *dif; + const char *path; + erofs_off_t devt_off; + uint64_t devt_size, image_size, end, other_end; + erofs_blk_t maxend; + unsigned int i, j, mask; + int error; + + sbi->total_blocks = sbi->dif0.blocks; + sbi->flatdev_blocks = sbi->dif0.blocks; + if (sbi->extra_devices == 0) { + if (arg_count != 0) { + vfs_mount_error(sbi->mnt, + "erofs: external devices given without a device table"); + return (EINVAL); + } + return (0); + } + devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE; + devt_size = (uint64_t)sbi->extra_devices * EROFS_DEVT_SLOT_SIZE; + if (sbi->dif0.blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + image_size = sbi->dif0.blocks << sbi->blkszbits; + if (devt_off > image_size || devt_size > image_size - devt_off || + devt_size > SIZE_MAX) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, 0, devt_off, (size_t)devt_size, &buf); + if (error != 0) + return (error); + slots = buf.data; + sbi->devs = mallocarray(sbi->extra_devices, sizeof(*sbi->devs), M_EROFS, + M_WAITOK | M_ZERO); + maxend = sbi->dif0.blocks; + for (i = 0; i < sbi->extra_devices; ++i) { + dif = &sbi->devs[i]; + dif->blocks = le32toh(slots[i].blocks_lo); + dif->uniaddr = le32toh(slots[i].uniaddr_lo); + if (erofs_sb_has_48bit(sbi)) { + dif->blocks |= (uint64_t)le16toh(slots[i].blocks_hi) << 32; + dif->uniaddr |= (uint64_t)le16toh(slots[i].uniaddr_hi) << 32; + } + if (dif->blocks == 0 || + __builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) { + error = EINTEGRITY; + goto out; + } + if (end > + (erofs_sb_has_48bit(sbi) ? (1ULL << 48) : (1ULL << 32))) { + error = EINTEGRITY; + goto out; + } + if (dif->uniaddr != 0 && dif->uniaddr < sbi->dif0.blocks) { + error = EINTEGRITY; + goto out; + } + for (j = 0; j < i; ++j) { + if (dif->uniaddr == 0 || sbi->devs[j].uniaddr == 0) + continue; + if (__builtin_add_overflow(sbi->devs[j].uniaddr, + sbi->devs[j].blocks, &other_end)) { + error = EINTEGRITY; + goto out; + } + if (dif->uniaddr < other_end && sbi->devs[j].uniaddr < end) { + error = EINTEGRITY; + goto out; + } + } + if (__builtin_add_overflow(sbi->total_blocks, dif->blocks, + &sbi->total_blocks)) { + error = EOVERFLOW; + goto out; + } + maxend = MAX(maxend, (erofs_blk_t)end); + } + erofs_put_metabuf(&buf); + sbi->flatdev_blocks = maxend; + mask = 1; + while (mask < (unsigned int)sbi->extra_devices + 1) + mask <<= 1; + sbi->device_id_mask = mask - 1; + sbi->flatdev = arg_count == 0; + if (sbi->flatdev) + return (erofs_validate_device_size(sbi, &sbi->dif0, + sbi->flatdev_blocks)); + if (arg_count != sbi->extra_devices) { + vfs_mount_error(sbi->mnt, + "erofs: external devices don't match (ondisk %u, given %u)", + sbi->extra_devices, arg_count); + return (arg_count < sbi->extra_devices ? ENXIO : EINVAL); + } + for (i = 0; i < arg_count; ++i) { + if (args[i].slot == 0 || args[i].slot > sbi->extra_devices) + return (EINVAL); + } + for (i = 0; i < sbi->extra_devices; ++i) { + path = erofs_device_arg_path(args, arg_count, i + 1); + if (path == NULL) + return (ENXIO); + error = erofs_init_device(sbi, &sbi->devs[i], path); + if (error != 0) + return (error); + } + return (0); +out: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_init_packed_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* Load the packed carrier before any fragment-backed metabox inode. */ + if (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) { + sbi->packed_inode = malloc(sizeof(*sbi->packed_inode), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->packed_nid, sbi->packed_inode); + if (error != 0) { + free(sbi->packed_inode, M_EROFS); + sbi->packed_inode = NULL; + return (error); + } + if (sbi->packed_inode->vtype != VREG || sbi->packed_inode->fragment) { + vfs_mount_error(sbi->mnt, + "erofs: packed inode nid=%ju is not a non-recursive regular file", + (uintmax_t)sbi->packed_nid); + return (EINTEGRITY); + } + } + return (0); +} + +static int +erofs_init_metabox_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* + * METABOX NIDs address inode slots in this backing inode's data. The + * packed carrier is ready first so a compressed metabox may legally end in + * a fragment pcluster without reading an uninitialized dependency. + */ + if (erofs_sb_has_metabox(sbi)) { + struct erofs_map_blocks map; + + sbi->metabox_en = malloc(sizeof(*sbi->metabox_en), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->metabox_nid, sbi->metabox_en); + if (error != 0) + return (error); + if (sbi->metabox_en->vtype != VREG) { + vfs_mount_error(sbi->mnt, + "erofs: metabox inode nid=%ju is not a regular file", + (uintmax_t)sbi->metabox_nid); + return (EINTEGRITY); + } + if (sbi->metabox_en->fragment) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == sbi->metabox_en->nid || + sbi->metabox_en->size == 0) + return (EINTEGRITY); + map = (struct erofs_map_blocks) { + .m_la = sbi->metabox_en->size - 1, + }; + error = erofs_map_blocks(sbi, sbi->metabox_en, &map); + if (error != 0 || (map.m_flags & EROFS_MAP_FRAGMENT) == 0) + return (error != 0 ? error : EINTEGRITY); + } + } + return (0); +} + +static int +erofs_read_superblock(struct erofs_sb_info *sbi, struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t unsupported; + int error; + + error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sizeof(*dsb), + &buf); + if (error != 0) + return (error); + memcpy(dsb, buf.data, sizeof(*dsb)); + erofs_put_metabuf(&buf); + + if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1) + return (EINVAL); + if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT) + return (EINVAL); + sbi->blkszbits = dsb->blkszbits; + sbi->block_size = 1u << sbi->blkszbits; + sbi->feature_compat = le32toh(dsb->feature_compat); + error = erofs_superblock_csum_verify(sbi, dsb); + if (error != 0) + return (error); + + if (dsb->dirblkbits != 0) + return (EOPNOTSUPP); + sbi->feature_incompat = le32toh(dsb->feature_incompat); + sbi->packed_nid = le64toh(dsb->packed_nid); + sbi->extra_devices = erofs_sb_has_device_table(sbi) ? + le16toh(dsb->extra_devices) : 0; + unsupported = sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT; + if (unsupported != 0) + return (EOPNOTSUPP); + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) + return (EINVAL); + sbi->meta_blkaddr = le32toh(dsb->meta_blkaddr); + sbi->xattr_blkaddr = le32toh(dsb->xattr_blkaddr); + sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start); + sbi->xattr_prefix_count = dsb->xattr_prefix_count; + if (erofs_sb_has_ishare_xattrs(sbi) && + dsb->ishare_xattr_prefix_id >= sbi->xattr_prefix_count) + return (EINTEGRITY); + /* Preserve the raw feature declaration and gate its format at use sites. */ + sbi->xattr_filter_reserved = dsb->xattr_filter_reserved; + erofs_sb_blocks_root(dsb, sbi->feature_incompat, &sbi->blocks, + &sbi->root_nid); + sbi->dif0.blocks = sbi->blocks; + error = erofs_validate_device_size(sbi, &sbi->dif0, sbi->dif0.blocks); + if (error != 0) + return (error); + sbi->inos = le64toh(dsb->inos); + sbi->epoch = (int64_t)le64toh(dsb->epoch); + sbi->fixed_nsec = le32toh(dsb->fixed_nsec); + if (sbi->fixed_nsec >= 1000000000) + return (EINTEGRITY); + error = erofs_load_generation_seed(sbi, sbi->sb_size, + &sbi->generation_seed); + if (error != 0) + return (error); + if (sbi->packed_nid != 0 && erofs_nid_in_metabox(sbi->packed_nid)) + return (EINTEGRITY); + if (erofs_sb_has_metabox(sbi)) { + if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) + return (EINTEGRITY); + sbi->metabox_nid = le64toh(dsb->metabox_nid); + if (erofs_nid_in_metabox(sbi->metabox_nid)) + return (EINTEGRITY); + } + + return (z_erofs_parse_cfgs(sbi, dsb)); +} + +static int +erofs_mountfs(struct erofs_device_info *primary, struct mount *mp, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_sb_info *sbi; + struct erofs_inode root; + struct erofs_super_block dsb; + int error; + + sbi = malloc(sizeof(*sbi), M_EROFS, M_WAITOK | M_ZERO); + sbi->mnt = mp; + z_erofs_extent_cache_init(sbi); + z_erofs_stream_pool_init(sbi); + sbi->dif0 = *primary; + bzero(primary, sizeof(*primary)); + + error = erofs_read_superblock(sbi, &dsb); + if (error != 0) + goto fail; + error = erofs_scan_devices(sbi, &dsb, args, arg_count); + if (error != 0) + goto fail; + + if (erofs_sb_has_shared_ea_in_metabox(sbi) && + !erofs_sb_has_metabox(sbi)) { + error = EINTEGRITY; + goto fail; + } + + error = erofs_init_packed_inode(sbi); + if (error != 0) + goto fail; + error = erofs_init_metabox_inode(sbi); + if (error != 0) + goto fail; + error = erofs_read_inode(sbi, sbi->root_nid, &root); + if (error != 0) + goto fail; + if (root.vtype != VDIR) { + vfs_mount_error(mp, + "erofs: root inode nid=%ju is not a directory", + (uintmax_t)sbi->root_nid); + error = EINTEGRITY; + goto fail; + } + error = erofs_xattr_prefixes_init(sbi); + if (error != 0) + goto fail; + set_opt(&sbi->opt, POSIX_ACL); + memcpy(sbi->volume_name, dsb.volume_name, 16); + sbi->volume_name[16] = '\0'; + + mp->mnt_data = sbi; + mp->mnt_stat.f_fsid.val[0] = dev2udev(sbi->dif0.devvp->v_rdev); + mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum; + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY | MNT_ACLS; + mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED | + MNTK_USES_BCACHE; + MNT_IUNLOCK(mp); + return (0); +fail: + erofs_sb_free(sbi); + return (error); +} + +static int +erofs_mount(struct mount *mp) +{ + struct erofs_device_arg *args; + struct erofs_device_info primary; + char *fspec; + unsigned int arg_count; + int error, len; + + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_RDONLY; + MNT_IUNLOCK(mp); + if (mp->mnt_flag & MNT_UPDATE) { + if (vfs_flagopt(mp->mnt_optnew, "export", NULL, 0)) + return (0); + return (EOPNOTSUPP); + } + args = NULL; + arg_count = 0; + error = erofs_parse_device_options(mp, &args, &arg_count); + if (error != 0) + return (error); + if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) { + error = EINVAL; + goto out_args; + } + fspec = NULL; + error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len); + if (error != 0 || fspec == NULL || len == 0 || + fspec[len - 1] != '\0') { + error = EINVAL; + goto out_args; + } + mp->mnt_iosize_max = MAXPHYS; + error = erofs_open_device(NULL, fspec, &primary); + if (error != 0) + goto out_args; + erofs_update_iosize_max(mp, &primary); + error = erofs_mountfs(&primary, mp, args, arg_count); +out_args: + erofs_free_device_args(args, arg_count); + if (error != 0) + return (error); + vfs_mountedfrom(mp, fspec); + return (erofs_statfs(mp, &mp->mnt_stat)); +} + +static int +erofs_root(struct mount *mp, int flags, struct vnode **vpp) +{ + int error; + + error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp); + if (error != 0) + vfs_mount_error(mp, "erofs: failed to load root nid %ju: error %d", + (uintmax_t)MTOE(mp)->root_nid, error); + return (error); +} + +static int +erofs_statfs(struct mount *mp, struct statfs *sbp) +{ + struct erofs_sb_info *sbi; + + sbi = MTOE(mp); + sbp->f_bsize = sbi->block_size; + sbp->f_iosize = sbi->block_size; + sbp->f_blocks = sbi->total_blocks; + sbp->f_bfree = 0; + sbp->f_bavail = 0; + sbp->f_files = sbi->inos; + sbp->f_ffree = 0; + return (0); +} + +static int +erofs_unmount(struct mount *mp, int mntflags) +{ + struct erofs_sb_info *sbi; + int error, flags; + + flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0; + error = vflush(mp, 0, flags, curthread); + if (error != 0) + return (error); + sbi = MTOE(mp); + mp->mnt_data = NULL; + erofs_sb_free(sbi); + return (0); +} + +/* Persistent EROFS file handle to locked vnode. */ +static int +erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + struct vnode *vp; + erofs_nid_t nid; + int error; + + *vpp = NULL; + bzero(&efid, sizeof(efid)); + memcpy(&efid, fhp, sizeof(efid)); + if (efid.len != sizeof(efid) || efid.pad != 0) + return (EINVAL); + nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo; + if (!erofs_nid_is_valid(MTOE(mp), nid)) + return (ESTALE); + error = VFS_VGET(mp, (ino_t)nid, flags, &vp); + if (error != 0) + return (error); + vi = VTOE(vp); + if (vi->mode == 0 || vi->nlink == 0 || vi->nid != nid || + vi->generation != efid.gen) { + vput(vp); + return (ESTALE); + } + *vpp = vp; + return (0); +} + +static struct vfsops erofs_vfsops = { + .vfs_fhtovp = erofs_fhtovp, + .vfs_mount = erofs_mount, + .vfs_root = erofs_root, + .vfs_statfs = erofs_statfs, + .vfs_unmount = erofs_unmount, + .vfs_vget = erofs_vget, +}; +VFS_SET(erofs_vfsops, erofs, VFCF_READONLY); +MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1); +MODULE_DEPEND(erofs, zlib, 1, 1, 1); +MODULE_VERSION(erofs, 1); diff --git a/xattr.c b/xattr.c new file mode 100644 index 0000000..37aca63 --- /dev/null +++ b/xattr.c @@ -0,0 +1,1272 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021-2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "xattr.h" + +struct posix_acl_xattr_entry { + uint16_t e_tag; + uint16_t e_perm; + uint32_t e_id; +}; + +struct posix_acl_xattr_header { + uint32_t a_version; +}; + +#define POSIX_ACL_XATTR_VERSION 0x0002 +#define EROFS_XATTR_FILTER_POSIX_ACL \ + ((1U << 21) | (1U << 30)) +#define EROFS_XATTR_CACHE_BODY_LIMIT (64UL * 1024) +#define EROFS_XATTR_CACHE_MOUNT_BUDGET (1024UL * 1024) + +struct erofs_xattr_iter { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + int attrnamespace; + const char *name; + size_t name_len; + struct uio *uio; + size_t *sizep; +}; + +static int erofs_xattr_load_body(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep); + +static uint32_t +erofs_xxh32_rotl(uint32_t value, unsigned int count) +{ + return ((value << count) | (value >> (32 - count))); +} + +static uint32_t +erofs_xxh32_round(uint32_t seed, uint32_t input) +{ + seed += input * UINT32_C(2246822519); + seed = erofs_xxh32_rotl(seed, 13); + return (seed * UINT32_C(2654435761)); +} + +static uint32_t +erofs_xxh32(const void *input, size_t length, uint32_t seed) +{ + const uint8_t *cursor, *end; + uint32_t hash; + + cursor = input; + end = cursor + length; + if (length >= 16) { + const uint8_t *limit; + uint32_t v1, v2, v3, v4; + + limit = end - 16; + v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519); + v2 = seed + UINT32_C(2246822519); + v3 = seed; + v4 = seed - UINT32_C(2654435761); + do { + v1 = erofs_xxh32_round(v1, le32dec(cursor)); + cursor += 4; + v2 = erofs_xxh32_round(v2, le32dec(cursor)); + cursor += 4; + v3 = erofs_xxh32_round(v3, le32dec(cursor)); + cursor += 4; + v4 = erofs_xxh32_round(v4, le32dec(cursor)); + cursor += 4; + } while (cursor <= limit); + hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) + + erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18); + } else { + hash = seed + UINT32_C(374761393); + } + hash += (uint32_t)length; + while (cursor + 4 <= end) { + hash += le32dec(cursor) * UINT32_C(3266489917); + hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263); + cursor += 4; + } + while (cursor < end) { + hash += *cursor++ * UINT32_C(374761393); + hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761); + } + hash ^= hash >> 15; + hash *= UINT32_C(2246822519); + hash ^= hash >> 13; + hash *= UINT32_C(3266489917); + hash ^= hash >> 16; + return (hash); +} + +static bool +erofs_xattr_filter_name(int attrnamespace, const char *name, size_t name_len, + uint8_t *indexp, const char **filter_namep, size_t *filter_name_lenp) +{ + static const char acl_access[] = "posix_acl_access"; + static const char acl_default[] = "posix_acl_default"; + static const char security[] = "security."; + static const char trusted[] = "trusted."; + + if (attrnamespace == EXTATTR_NAMESPACE_USER) { + *indexp = EROFS_XATTR_INDEX_USER; + *filter_namep = name; + *filter_name_lenp = name_len; + return (true); + } + if (attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (false); + if (name_len == sizeof(acl_access) - 1 && + memcmp(name, acl_access, sizeof(acl_access) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len == sizeof(acl_default) - 1 && + memcmp(name, acl_default, sizeof(acl_default) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len >= sizeof(trusted) - 1 && + memcmp(name, trusted, sizeof(trusted) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_TRUSTED; + *filter_namep = name + sizeof(trusted) - 1; + *filter_name_lenp = name_len - (sizeof(trusted) - 1); + return (true); + } + if (name_len >= sizeof(security) - 1 && + memcmp(name, security, sizeof(security) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_SECURITY; + *filter_namep = name + sizeof(security) - 1; + *filter_name_lenp = name_len - (sizeof(security) - 1); + return (true); + } + return (false); +} + +static int +erofs_xattr_backing_size(struct erofs_sb_info *sbi, struct erofs_inode *backing_en, + erofs_off_t *sizep) +{ + if (backing_en != NULL) { + *sizep = backing_en->size; + return (0); + } + if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + *sizep = sbi->blocks << sbi->blkszbits; + return (0); +} + +static int +erofs_xattr_read_backing(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, erofs_off_t off, size_t len, + struct erofs_buf *buf) +{ + void *data; + erofs_off_t backing_size; + int error; + + error = erofs_xattr_backing_size(sbi, backing_en, &backing_size); + if (error != 0) + return (error); + if (off > backing_size || (uint64_t)len > backing_size - off) + return (EINTEGRITY); + if (backing_en != NULL) + error = erofs_read_data(sbi, backing_en, off, len, &data); + else { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_xattr_filter_negative(struct erofs_sb_info *sbi, struct erofs_inode *vi, + int attrnamespace, const char *name, size_t name_len, bool *negativep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + const char *filter_name; + size_t filter_name_len, header_size; + uint32_t hashbit, name_filter; + uint8_t index; + int error; + + *negativep = false; + if (!erofs_sb_has_xattr_filter_v1(sbi) || + !erofs_xattr_filter_name(attrnamespace, name, name_len, &index, + &filter_name, &filter_name_len)) + return (0); + if (vi->xattr_isize < sizeof(*ih)) + return (EINTEGRITY); + if (vi->xattr_isize == sizeof(*ih)) + return (EOPNOTSUPP); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + name_filter = le32toh(ih->h_name_filter); + hashbit = erofs_xxh32(filter_name, filter_name_len, + EROFS_XATTR_FILTER_SEED + index); + hashbit &= EROFS_XATTR_FILTER_BITS - 1; + *negativep = (name_filter & (1U << hashbit)) != 0; + error = 0; + (void)header_size; + erofs_put_metabuf(&body); + return (error); +} + +/* + * Read one prefix table metadata record. + * + * When backing_en == NULL the record is in the physical metadata area; + * otherwise it lives in the selected metadata carrier's logical data stream. + */ +static int +erofs_xattr_read_metadata(struct erofs_sb_info *sbi, struct erofs_inode *backing_en, + erofs_off_t *offp, struct erofs_buf *buf, size_t *lenp) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + uint16_t raw_len; + erofs_off_t off; + size_t len; + int error; + + if (*offp > UINT64_MAX - (sizeof(struct erofs_xattr_entry) - 1)) + return (EOVERFLOW); + off = roundup2(*offp, sizeof(struct erofs_xattr_entry)); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(raw_len), + &hdrbuf); + if (error != 0) + return (error); + raw_len = le16dec(hdrbuf.data); + erofs_put_metabuf(&hdrbuf); + len = (raw_len == 0) ? (size_t)UINT16_MAX + 1 : raw_len; + if (len < sizeof(struct erofs_xattr_long_prefix) || + len > EROFS_NAME_LEN + sizeof(struct erofs_xattr_long_prefix)) + return (EINTEGRITY); + if (off > UINT64_MAX - sizeof(raw_len)) + return (EOVERFLOW); + error = erofs_xattr_read_backing(sbi, backing_en, + off + sizeof(raw_len), len, buf); + if (error != 0) + return (error); + *offp = off + sizeof(raw_len) + len; + *lenp = len; + return (0); +} + +static int +erofs_xattr_move(void *value, size_t value_size, struct uio *uio, size_t *sizep) +{ + if (sizep != NULL) + *sizep = value_size; + if (uio == NULL || value_size == 0) + return (0); + return (uiomove(value, value_size, uio)); +} + +static int +erofs_xattr_load_body_uncached(struct erofs_sb_info *sbi, + struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + char *body; + uint64_t body_off; + size_t header_size; + int error; + + if (vi->xattr_isize < sizeof(*ih)) + return (EINTEGRITY); + if (vi->inode_off > UINT64_MAX - vi->inode_isize) + return (EINTEGRITY); + body_off = vi->inode_off + vi->inode_isize; + error = erofs_xattr_read_backing(sbi, + erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL, body_off, + vi->xattr_isize, &buf); + if (error != 0) + return (error); + body = buf.data; + ih = (struct erofs_xattr_ibody_header *)body; + if (vi->xattr_isize == sizeof(*ih)) { + error = EOPNOTSUPP; + goto fail; + } + header_size = sizeof(*ih) + sizeof(uint32_t) * ih->h_shared_count; + if (header_size > vi->xattr_isize) { + error = EINTEGRITY; + goto fail; + } + *bodybuf = buf; + *ihp = ih; + *header_sizep = header_size; + return (0); +fail: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_xattr_validate_entry(struct erofs_xattr_entry *entry, size_t remaining, + size_t *entry_sizep, size_t *value_sizep) +{ + size_t entry_size, min_size, value_size; + + if (remaining < sizeof(*entry)) + return (EINTEGRITY); + value_size = le16toh(entry->e_value_size); + min_size = sizeof(*entry) + entry->e_name_len + value_size; + if (min_size > remaining) + return (EINTEGRITY); + entry_size = erofs_xattr_entry_size(entry); + if (entry_size > remaining) + return (EINTEGRITY); + if (memchr(entry->e_name, '\0', entry->e_name_len) != NULL) + return (EINTEGRITY); + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static bool +erofs_xattr_prefix(uint8_t base_index, int *namespacep, + const char **prefixp, size_t *prefix_lenp) +{ + switch (base_index) { + case EROFS_XATTR_INDEX_USER: + *namespacep = EXTATTR_NAMESPACE_USER; + *prefixp = NULL; + *prefix_lenp = 0; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_ACCESS: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_access"; + *prefix_lenp = sizeof("posix_acl_access") - 1; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_default"; + *prefix_lenp = sizeof("posix_acl_default") - 1; + return (true); + case EROFS_XATTR_INDEX_TRUSTED: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "trusted."; + *prefix_lenp = sizeof("trusted.") - 1; + return (true); + case EROFS_XATTR_INDEX_SECURITY: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "security."; + *prefix_lenp = sizeof("security.") - 1; + return (true); + case EROFS_XATTR_INDEX_LUSTRE: + default: + return (false); + } +} + +static int +erofs_xattr_namespace_prefix(int attrnamespace, uint8_t base_index, + const char **prefixp, size_t *prefix_lenp) +{ + int mapped_namespace; + + if (attrnamespace != EXTATTR_NAMESPACE_USER && + attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (EOPNOTSUPP); + if (!erofs_xattr_prefix(base_index, &mapped_namespace, prefixp, + prefix_lenp)) + return (ENOATTR); + if (mapped_namespace != attrnamespace) + return (ENOATTR); + return (0); +} + +static int +erofs_xattr_list_move(const char *namespace_prefix, size_t namespace_prefix_len, + const char *infix, size_t infix_len, const char *name, uint8_t name_len, + struct uio *uio, size_t *sizep) +{ + uint8_t total_name_len; + int error; + + if (namespace_prefix_len + infix_len + name_len > EROFS_NAME_LEN) + return (EINTEGRITY); + total_name_len = namespace_prefix_len + infix_len + name_len; + if (sizep != NULL) { + *sizep += total_name_len + 1; + return (0); + } + if (uio == NULL) + return (0); + error = uiomove(__DECONST(void *, &total_name_len), 1, uio); + if (error != 0) + return (error); + if (namespace_prefix_len != 0) { + error = uiomove(__DECONST(void *, namespace_prefix), + namespace_prefix_len, uio); + if (error != 0) + return (error); + } + if (infix_len != 0) { + error = uiomove(__DECONST(void *, infix), infix_len, uio); + if (error != 0) + return (error); + } + return (uiomove(__DECONST(void *, name), name_len, uio)); +} + +static int +erofs_xattr_resolve_name(struct erofs_sb_info *sbi, + const struct erofs_xattr_entry *entry, uint8_t *base_indexp, + const char **infixp, size_t *infix_lenp) +{ + struct erofs_xattr_prefix_item *prefix; + uint8_t prefix_id; + + if ((entry->e_name_index & EROFS_XATTR_LONG_PREFIX) == 0) { + *base_indexp = entry->e_name_index; + *infixp = NULL; + *infix_lenp = 0; + return (0); + } + if (sbi->xattr_prefixes == NULL) + return (ENOATTR); + prefix_id = entry->e_name_index & EROFS_XATTR_LONG_PREFIX_MASK; + if (prefix_id >= sbi->xattr_prefix_count) + return (ENOATTR); + prefix = &sbi->xattr_prefixes[prefix_id]; + *base_indexp = prefix->base_index; + *infixp = prefix->infix; + *infix_lenp = prefix->infix_len; + return (0); +} + +static bool +erofs_xattr_name_match(const char *namespace_prefix, + size_t namespace_prefix_len, const char *infix, size_t infix_len, + const struct erofs_xattr_entry *entry, const char *name, size_t name_len) +{ + if (name_len != namespace_prefix_len + infix_len + entry->e_name_len) + return (false); + if (namespace_prefix_len != 0 && + memcmp(name, namespace_prefix, namespace_prefix_len) != 0) + return (false); + if (infix_len != 0 && + memcmp(name + namespace_prefix_len, infix, infix_len) != 0) + return (false); + return (memcmp(name + namespace_prefix_len + infix_len, entry->e_name, + entry->e_name_len) == 0); +} + +static int +erofs_xattr_shared_entry_offset(struct erofs_sb_info *sbi, uint32_t shared_id, + erofs_off_t *phys_offp) +{ + erofs_off_t base, relative; + + if (sbi->xattr_blkaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EOVERFLOW); + base = (uint64_t)sbi->xattr_blkaddr << sbi->blkszbits; + relative = (uint64_t)shared_id * sizeof(uint32_t); + if (relative > UINT64_MAX - base) + return (EOVERFLOW); + *phys_offp = base + relative; + return (0); +} + +static int +erofs_xattr_load_shared_entry(struct erofs_sb_info *sbi, uint32_t shared_id, + struct erofs_buf *entrybuf, size_t *entry_sizep, size_t *value_sizep) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + struct erofs_inode *backing_en; + erofs_off_t off; + size_t entry_size, value_size; + int error; + + backing_en = erofs_sb_has_shared_ea_in_metabox(sbi) ? sbi->metabox_en : NULL; + if (erofs_sb_has_shared_ea_in_metabox(sbi) && backing_en == NULL) + return (EINTEGRITY); + + error = erofs_xattr_shared_entry_offset(sbi, shared_id, &off); + if (error != 0) + return (error); + + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(*entry), + &hdrbuf); + if (error != 0) + return (error); + entry = hdrbuf.data; + value_size = le16toh(entry->e_value_size); + entry_size = erofs_xattr_entry_size(entry); + + erofs_put_metabuf(&hdrbuf); + + error = erofs_xattr_read_backing(sbi, backing_en, off, entry_size, + entrybuf); + if (error != 0) + return (error); + error = erofs_xattr_validate_entry(entrybuf->data, entry_size, NULL, + value_sizep); + if (error != 0) { + erofs_put_metabuf(entrybuf); + return (error); + } + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static int +erofs_xattr_validate_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + void *body, struct erofs_xattr_ibody_header *ih, size_t header_size) +{ + struct erofs_buf entrybuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + char *cursor; + uint32_t shared_id; + size_t entry_size, remaining; + int error; + + if (header_size > vi->xattr_isize || + memchr(ih->h_reserved2, '\0', sizeof(ih->h_reserved2)) == NULL) + return (EINTEGRITY); + remaining = vi->xattr_isize - header_size; + cursor = (char *)body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + error = erofs_xattr_validate_entry(entry, remaining, &entry_size, + NULL); + if (error != 0) + return (error); + cursor += entry_size; + remaining -= entry_size; + } + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + error = erofs_xattr_load_shared_entry(sbi, shared_id, &entrybuf, + NULL, NULL); + if (error != 0) + return (error); + erofs_put_metabuf(&entrybuf); + } + return (0); +} + +static int +erofs_xattr_load_body_checked(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep) +{ + int error; + + error = erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep); + if (error != 0) + return (error); + error = erofs_xattr_validate_body(sbi, vi, bodybuf->data, *ihp, + *header_sizep); + if (error != 0) + erofs_put_metabuf(bodybuf); + return (error); +} + +static bool +erofs_xattr_cache_reserve(struct erofs_sb_info *sbi, size_t size) +{ + u_long resident; + + for (;;) { + resident = atomic_load_acq_long(&sbi->xattr_cache_resident); + if (resident > EROFS_XATTR_CACHE_MOUNT_BUDGET || + size > EROFS_XATTR_CACHE_MOUNT_BUDGET - resident) + return (false); + if (atomic_cmpset_acq_long(&sbi->xattr_cache_resident, resident, + resident + size)) + return (true); + } +} + +static void +erofs_xattr_cache_release(struct erofs_sb_info *sbi, size_t size) +{ + KASSERT(size <= EROFS_XATTR_CACHE_MOUNT_BUDGET, + ("erofs xattr cache release exceeds mount budget")); + atomic_subtract_rel_long(&sbi->xattr_cache_resident, size); +} + +static void * +erofs_xattr_cache_invalidate_locked(struct erofs_sb_info *sbi, + struct erofs_xattr_cache *cache) +{ + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + erofs_xattr_cache_release(sbi, cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + cache->size = 0; + cache->error = 0; + cache->state = EROFS_XATTR_CACHE_EMPTY; + return (data); +} + +void +erofs_xattr_cache_init(struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + + cache = &vi->xattr_cache; + bzero(cache, sizeof(*cache)); + mtx_init(&cache->lock, "erofs xattr", NULL, MTX_DEF); + cv_init(&cache->cv, "erofs xattr"); + cache->state = EROFS_XATTR_CACHE_EMPTY; + cache->initialized = true; +} + +void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + void *data; + + cache = &vi->xattr_cache; + if (!cache->initialized) + return; + mtx_lock(&cache->lock); + cache->closing = true; + while (cache->state == EROFS_XATTR_CACHE_INFLIGHT || + cache->waiters != 0) + cv_wait(&cache->cv, &cache->lock); + data = erofs_xattr_cache_invalidate_locked(sbi, cache); + cache->initialized = false; + mtx_unlock(&cache->lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&cache->lock); +} + +static int +erofs_xattr_load_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_xattr_cache *cache; + bool reserved; + int error; + + cache = &vi->xattr_cache; + if (!cache->initialized || vi->xattr_isize > EROFS_XATTR_CACHE_BODY_LIMIT) + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + mtx_lock(&cache->lock); + if (cache->closing) { + mtx_unlock(&cache->lock); + return (ENXIO); + } + if (cache->state == EROFS_XATTR_CACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs xattr ready cache has no body")); + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + mtx_unlock(&cache->lock); + return (0); + } + if (cache->state == EROFS_XATTR_CACHE_INFLIGHT) { + cache->waiters++; + do { + cv_wait(&cache->cv, &cache->lock); + } while (cache->state == EROFS_XATTR_CACHE_INFLIGHT); + if (cache->state == EROFS_XATTR_CACHE_READY) { + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + error = 0; + } else { + KASSERT(cache->state == EROFS_XATTR_CACHE_FAILED && + cache->error > 0, + ("erofs xattr failed cache has no typed error")); + error = cache->error; + } + cache->waiters--; + if (cache->state == EROFS_XATTR_CACHE_FAILED && + cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (error); + } + if (cache->state == EROFS_XATTR_CACHE_FAILED) { + if (cache->waiters != 0) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + } + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + } + reserved = erofs_xattr_cache_reserve(sbi, vi->xattr_isize); + if (!reserved) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep)); + } + cache->state = EROFS_XATTR_CACHE_INFLIGHT; + cache->error = 0; + mtx_unlock(&cache->lock); + + error = erofs_xattr_load_body_checked(sbi, vi, bodybuf, ihp, + header_sizep); + mtx_lock(&cache->lock); + if (error == 0 && !cache->closing) { + cache->data = bodybuf->data; + cache->size = vi->xattr_isize; + cache->charged_bytes = vi->xattr_isize; + cache->state = EROFS_XATTR_CACHE_READY; + bodybuf->release = NULL; + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (0); + } + if (error == 0) + error = ENXIO; + erofs_xattr_cache_release(sbi, vi->xattr_isize); + cache->error = error; + cache->state = EROFS_XATTR_CACHE_FAILED; + if (cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + erofs_put_metabuf(bodybuf); + return (error); +} + +static int +erofs_listxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error == ENOATTR) + return (0); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error == ENOATTR) + return (0); + if (error != 0) + return (error); + return (erofs_xattr_list_move(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry->e_name, entry->e_name_len, it->uio, + it->sizep)); +} + +static int +erofs_getxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry, size_t value_size) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error != 0) + return (error); + if (!erofs_xattr_name_match(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry, it->name, it->name_len)) + return (ENOATTR); + return (erofs_xattr_move(entry->e_name + entry->e_name_len, value_size, + it->uio, it->sizep)); +} + +static int +erofs_xattr_iter_inline(struct erofs_xattr_iter *it, char *body, + size_t header_size, bool get) +{ + struct erofs_xattr_entry *entry; + char *cursor; + size_t entry_size, remaining, value_size; + int error; + + remaining = it->vi->xattr_isize - header_size; + cursor = body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + error = erofs_xattr_validate_entry(entry, remaining, + &entry_size, get ? &value_size : NULL); + if (error != 0) + return (error); + if (get) + error = erofs_getxattr_foreach(it, entry, value_size); + else + error = erofs_listxattr_foreach(it, entry); + if (get) { + if (error != ENOATTR) + return (error); + } else if (error != 0) { + return (error); + } + cursor += entry_size; + remaining -= entry_size; + } + return (get ? ENOATTR : 0); +} + +static int +erofs_xattr_iter_shared(struct erofs_xattr_iter *it, + struct erofs_xattr_ibody_header *ih, bool get) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + uint32_t shared_id; + size_t value_size; + int error; + + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + error = erofs_xattr_load_shared_entry(it->sbi, shared_id, &buf, + NULL, get ? &value_size : NULL); + if (error != 0) + return (error); + entry = buf.data; + if (get) + error = erofs_getxattr_foreach(it, entry, value_size); + else + error = erofs_listxattr_foreach(it, entry); + erofs_put_metabuf(&buf); + if (get) { + if (error != ENOATTR) + return (error); + } else if (error != 0) { + return (error); + } + } + return (get ? ENOATTR : 0); +} + +/* + * Look up one inline/shared xattr by name. + * + * Name exposure rules: + * - user namespace: bare name, no "user." prefix; + * - system namespace: exposes full "trusted.*" / "security.*" names. + */ +int +erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size, name_len; + bool filter_negative; + int error; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (name == NULL || name[0] == '\0') + return (EINVAL); + name_len = strlen(name); + if (name_len > EROFS_NAME_LEN) + return (EINVAL); + if (vi->xattr_isize == 0) + return (ENOATTR); + error = erofs_xattr_filter_negative(sbi, vi, attrnamespace, name, name_len, + &filter_negative); + if (error != 0) + return (error); + if (filter_negative) + return (ENOATTR); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = name; + it.name_len = name_len; + it.uio = uio; + it.sizep = sizep; + error = erofs_xattr_iter_inline(&it, body.data, header_size, true); + if (error == ENOATTR) + error = erofs_xattr_iter_shared(&it, ih, true); + erofs_put_metabuf(&body); + return (error); +} + +/* + * Enumerate inline/shared xattr names for a given namespace. + * + * Return format: 1-byte name length followed by non-NUL-terminated name bytes. + */ +int +erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size; + int error; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (sizep != NULL) + *sizep = 0; + if (vi->xattr_isize == 0) + return (0); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = NULL; + it.name_len = 0; + it.uio = uio; + it.sizep = sizep; + error = erofs_xattr_iter_inline(&it, body.data, header_size, false); + if (error == 0) + error = erofs_xattr_iter_shared(&it, ih, false); + erofs_put_metabuf(&body); + return (error); +} + +void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi) +{ + if (sbi->xattr_prefixes == NULL) + return; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) + free(sbi->xattr_prefixes[i].infix, M_EROFS); + free(sbi->xattr_prefixes, M_EROFS); + sbi->xattr_prefixes = NULL; +} + +int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_long_prefix *prefix = NULL; + struct erofs_inode packed_en, *prefix_en; + erofs_off_t off; + size_t infix_len, len; + int error; + + if (!erofs_sb_has_xattr_prefixes(sbi) || sbi->xattr_prefix_count == 0) + return (0); + prefix_en = NULL; + if (!erofs_sb_has_plain_xattr_pfx(sbi)) { + if (erofs_sb_has_metabox(sbi)) { + if (sbi->metabox_en == NULL) + return (EINTEGRITY); + prefix_en = sbi->metabox_en; + } else if (sbi->packed_inode != NULL) { + prefix_en = sbi->packed_inode; + } else if (sbi->packed_nid != 0) { + error = erofs_read_inode(sbi, sbi->packed_nid, &packed_en); + if (error != 0) + return (error); + if (packed_en.vtype != VREG) + return (EINTEGRITY); + prefix_en = &packed_en; + } + } + sbi->xattr_prefixes = malloc(sizeof(*sbi->xattr_prefixes) * + sbi->xattr_prefix_count, + M_EROFS, M_WAITOK | M_ZERO); + off = (uint64_t)sbi->xattr_prefix_start << 2; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) { + error = erofs_xattr_read_metadata(sbi, prefix_en, &off, + &buf, &len); + if (error != 0) + goto fail; + prefix = buf.data; + infix_len = len - sizeof(*prefix); + if (memchr(prefix->infix, '\0', infix_len) != NULL) { + error = EINTEGRITY; + goto fail; + } + sbi->xattr_prefixes[i].base_index = prefix->base_index; + sbi->xattr_prefixes[i].infix_len = infix_len; + sbi->xattr_prefixes[i].infix = malloc(infix_len + 1, M_EROFS, + M_WAITOK); + memcpy(sbi->xattr_prefixes[i].infix, prefix->infix, infix_len); + sbi->xattr_prefixes[i].infix[infix_len] = '\0'; + erofs_put_metabuf(&buf); + } + return (0); +fail: + erofs_put_metabuf(&buf); + erofs_xattr_prefixes_cleanup(sbi); + return (error); +} + +static int +erofs_inode_has_noacl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + bool *noaclp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + struct erofs_inode *backing_en; + erofs_off_t body_off; + uint32_t name_filter; + int error; + + *noaclp = false; + if (vi->xattr_isize < sizeof(*ih)) { + *noaclp = true; + return (0); + } + if (!erofs_sb_has_xattr_filter(sbi) || + sbi->xattr_filter_reserved != 0) + return (0); + if (vi->inode_off > UINT64_MAX - vi->inode_isize) + return (EINTEGRITY); + body_off = vi->inode_off + vi->inode_isize; + backing_en = erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL; + error = erofs_xattr_read_backing(sbi, backing_en, body_off, sizeof(*ih), + &buf); + if (error != 0) + return (error); + ih = buf.data; + name_filter = le32toh(ih->h_name_filter); + erofs_put_metabuf(&buf); + *noaclp = (name_filter & EROFS_XATTR_FILTER_POSIX_ACL) == + EROFS_XATTR_FILTER_POSIX_ACL; + return (0); +} + +static void +erofs_acl_from_mode(mode_t mode, acl_type_t type, struct acl *aclp) +{ + if (type == ACL_TYPE_DEFAULT) { + aclp->acl_cnt = 0; + return; + } + aclp->acl_cnt = 3; + aclp->acl_entry[0].ae_tag = ACL_USER_OBJ; + aclp->acl_entry[0].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[0].ae_perm = acl_posix1e_mode_to_perm(ACL_USER_OBJ, mode); + aclp->acl_entry[1].ae_tag = ACL_GROUP_OBJ; + aclp->acl_entry[1].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[1].ae_perm = acl_posix1e_mode_to_perm(ACL_GROUP_OBJ, mode); + aclp->acl_entry[2].ae_tag = ACL_OTHER; + aclp->acl_entry[2].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[2].ae_perm = acl_posix1e_mode_to_perm(ACL_OTHER, mode); +} + +static int +erofs_posix_acl_from_xattr(const void *value, size_t size, mode_t mode, + acl_type_t type, struct acl *aclp) +{ + const uint8_t *buf; + struct posix_acl_xattr_header hdr; + struct posix_acl_xattr_entry entry; + uint32_t id; + int count, i, j, phase; + + buf = value; + if (size < sizeof(hdr) || (size - sizeof(hdr)) % sizeof(entry) != 0) + return (EINTEGRITY); + + memcpy(&hdr, buf, sizeof(hdr)); + if (le32toh(hdr.a_version) != POSIX_ACL_XATTR_VERSION) + return (EINTEGRITY); + + count = (size - sizeof(hdr)) / sizeof(entry); + if (count > ACL_MAX_ENTRIES) + return (EINTEGRITY); + if (count == 0) { + erofs_acl_from_mode(mode, type, aclp); + return (0); + } + + aclp->acl_cnt = count; + phase = 0; + for (i = 0; i < count; i++) { + uint16_t tag, perm; + + memcpy(&entry, buf + sizeof(hdr) + i * sizeof(entry), + sizeof(entry)); + tag = le16toh(entry.e_tag); + perm = le16toh(entry.e_perm); + + id = le32toh(entry.e_id); + if ((perm & ~ACL_PERM_BITS) != 0) + return (EINTEGRITY); + switch (tag) { + case ACL_USER_OBJ: + if (phase != 0 || id != UINT32_MAX) + return (EINTEGRITY); + phase = 1; + break; + case ACL_USER: + if ((phase != 1 && phase != 2) || id == UINT32_MAX) + return (EINTEGRITY); + phase = 2; + break; + case ACL_GROUP_OBJ: + if ((phase != 1 && phase != 2) || id != UINT32_MAX) + return (EINTEGRITY); + phase = 3; + break; + case ACL_GROUP: + if ((phase != 3 && phase != 4) || id == UINT32_MAX) + return (EINTEGRITY); + phase = 4; + break; + case ACL_MASK: + if ((phase != 3 && phase != 4) || id != UINT32_MAX) + return (EINTEGRITY); + phase = 5; + break; + case ACL_OTHER: + if ((phase != 3 && phase != 4 && phase != 5) || + id != UINT32_MAX) + return (EINTEGRITY); + phase = 6; + break; + default: + return (EINTEGRITY); + } + if (tag == ACL_USER || tag == ACL_GROUP) { + for (j = 0; j < i; j++) { + if (aclp->acl_entry[j].ae_tag == tag && + aclp->acl_entry[j].ae_id == id) + return (EINTEGRITY); + } + } + + aclp->acl_entry[i].ae_tag = tag; + aclp->acl_entry[i].ae_perm = perm; + aclp->acl_entry[i].ae_id = (id == UINT32_MAX) ? ACL_UNDEFINED_ID : id; + } + if (phase != 6 || acl_posix1e_check(aclp) != 0) + return (EINTEGRITY); + + return (0); +} + +int +erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + const char *xattr_name; + struct uio auio; + struct iovec aiov; + uint8_t buf[sizeof(struct posix_acl_xattr_header) + + sizeof(struct posix_acl_xattr_entry) * ACL_MAX_ENTRIES]; + size_t size; + bool noacl; + int error; + + sbi = MTOE(vp->v_mount); + if (!test_opt(&sbi->opt, POSIX_ACL)) + return (EOPNOTSUPP); + + vi = VTOE(vp); + + switch (type) { + case ACL_TYPE_ACCESS: + xattr_name = "posix_acl_access"; + break; + case ACL_TYPE_DEFAULT: + if (vp->v_type != VDIR) + return (EINVAL); + xattr_name = "posix_acl_default"; + break; + default: + return (EINVAL); + } + error = erofs_inode_has_noacl(sbi, vi, &noacl); + if (error != 0) + return (error); + if (noacl) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, NULL, + &size); + if (error == ENOATTR) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + if (error != 0) + return (error); + if (size > sizeof(buf)) + return (EINTEGRITY); + + aiov.iov_base = buf; + aiov.iov_len = size; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_offset = 0; + auio.uio_resid = size; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_rw = UIO_READ; + auio.uio_td = curthread; + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, &auio, + NULL); + if (error != 0) + return (error); + if (auio.uio_resid != 0) + return (EINTEGRITY); + + return (erofs_posix_acl_from_xattr(buf, size, vi->mode, type, aclp)); +} diff --git a/xattr.h b/xattr.h new file mode 100644 index 0000000..f60090a --- /dev/null +++ b/xattr.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_XATTR_H +#define __EROFS_XATTR_H + +#include "internal.h" + +int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi); +void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi); +int erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep); +int erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep); +int erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp); +#endif diff --git a/zdata.c b/zdata.c new file mode 100644 index 0000000..366ae0f --- /dev/null +++ b/zdata.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" + +enum z_erofs_cache_lookup { + Z_EROFS_CACHE_BYPASS, + Z_EROFS_CACHE_HIT, + Z_EROFS_CACHE_OWNER, + Z_EROFS_CACHE_ERROR, +}; + +#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024) +#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024) +#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024) + +static int z_erofs_cache_enabled = 1; +static unsigned long z_erofs_cache_mount_budget = + EROFS_ZCACHE_MOUNT_HARD_BUDGET; +static unsigned long z_erofs_cache_global_budget = + EROFS_ZCACHE_GLOBAL_HARD_BUDGET; +static unsigned long z_erofs_cache_minimum_decode_work = + EROFS_ZCACHE_MIN_DECODE_WORK; + +TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled); +TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget", + &z_erofs_cache_mount_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget", + &z_erofs_cache_global_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work", + &z_erofs_cache_minimum_decode_work); + +static struct mtx z_erofs_cache_list_lock; +static struct mtx z_erofs_cache_budget_lock; +static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts = + LIST_HEAD_INITIALIZER(z_erofs_cache_mounts); +static size_t z_erofs_cache_global_charged; +static eventhandler_tag z_erofs_cache_lowmem_tag; + +MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock, + "erofs zcache list", MTX_DEF); +MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock, + "erofs zcache budget", MTX_DEF); + +static size_t +z_erofs_extent_cache_global_limit(void) +{ + + return (MIN(z_erofs_cache_global_budget, + EROFS_ZCACHE_GLOBAL_HARD_BUDGET)); +} + +static bool +z_erofs_extent_cache_reserve(size_t bytes) +{ + size_t limit; + bool reserved; + + reserved = false; + mtx_lock(&z_erofs_cache_budget_lock); + limit = z_erofs_extent_cache_global_limit(); + if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) { + z_erofs_cache_global_charged += bytes; + reserved = true; + } + mtx_unlock(&z_erofs_cache_budget_lock); + return (reserved); +} + +static void +z_erofs_extent_cache_release(size_t bytes) +{ + + mtx_lock(&z_erofs_cache_budget_lock); + KASSERT(bytes <= z_erofs_cache_global_charged, + ("erofs decoded cache budget underflow")); + z_erofs_cache_global_charged -= bytes; + mtx_unlock(&z_erofs_cache_budget_lock); +} + +static void * +z_erofs_extent_cache_drop_locked(struct erofs_zextent_cache *cache, + bool evicted, bool reclaimed) +{ + struct erofs_zextent_cache_metrics *metrics; + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + KASSERT(cache->map.m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache charged unknown codec")); + metrics = &cache->metrics[cache->map.m_algorithmformat]; + if (cache->state == EROFS_ZCACHE_READY) { + KASSERT(metrics->resident_bytes == cache->charged_bytes, + ("erofs decoded cache codec accounting mismatch")); + metrics->resident_bytes = 0; + if (evicted) + ++metrics->evictions; + if (reclaimed) + ++metrics->reclaims; + } + z_erofs_extent_cache_release(cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + return (data); +} + +static bool +z_erofs_extent_cache_match(const struct erofs_zextent_cache *cache, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size) +{ + + return (cache->state != EROFS_ZCACHE_EMPTY && cache->nid == vi->nid && + cache->decoded_size == decoded_size && + cache->map.m_pa == map->m_pa && cache->map.m_la == map->m_la && + cache->map.m_plen == map->m_plen && + cache->map.m_llen == map->m_llen && + cache->map.m_deviceid == map->m_deviceid && + cache->map.m_algorithmformat == map->m_algorithmformat && + cache->map.m_flags == map->m_flags); +} + +static enum z_erofs_cache_lookup +z_erofs_extent_cache_claim(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + void *old; + enum z_erofs_cache_lookup result; + + KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS")); + KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff, + ("erofs extent cache copy exceeds decoded extent")); + *errorp = 0; + if (!sbi->z_extent_cache_initialized) + return (Z_EROFS_CACHE_BYPASS); + cache = &sbi->z_extent_cache; + KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache claim has unknown codec")); + old = NULL; + mtx_lock(&sbi->z_extent_cache_lock); + metrics = &cache->metrics[map->m_algorithmformat]; + if (cache->closing) { + ++metrics->bypasses; + goto bypass; + } + if (z_erofs_extent_cache_match(cache, vi, map, decoded_size)) { + switch (cache->state) { + case EROFS_ZCACHE_READY: + KASSERT(cache->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)cache->data + (size_t)mapoff, len); + ++metrics->hits; + mtx_unlock(&sbi->z_extent_cache_lock); + return (Z_EROFS_CACHE_HIT); + case EROFS_ZCACHE_INFLIGHT: + ++cache->waiters; + do { + cv_wait(&cache->cv, &sbi->z_extent_cache_lock); + } while (cache->state == EROFS_ZCACHE_INFLIGHT); + KASSERT(z_erofs_extent_cache_match(cache, vi, map, + decoded_size), ("erofs inflight cache key changed")); + if (cache->state == EROFS_ZCACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)cache->data + (size_t)mapoff, + len); + ++metrics->hits; + result = Z_EROFS_CACHE_HIT; + } else { + KASSERT(cache->state == EROFS_ZCACHE_FAILED && + cache->error > 0, + ("erofs inflight cache has no typed result")); + *errorp = cache->error; + result = Z_EROFS_CACHE_ERROR; + } + --cache->waiters; + if (cache->state == EROFS_ZCACHE_FAILED && + cache->waiters == 0) { + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + } + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&sbi->z_extent_cache_lock); + return (result); + case EROFS_ZCACHE_FAILED: + if (cache->waiters != 0) + goto bypass; + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + break; + case EROFS_ZCACHE_EMPTY: + break; + } + } + if (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) { + ++metrics->bypasses; + goto bypass; + } + if (cache->state == EROFS_ZCACHE_READY) { + old = z_erofs_extent_cache_drop_locked(cache, true, false); + cache->state = EROFS_ZCACHE_EMPTY; + } + if (!z_erofs_extent_cache_reserve(decoded_size)) { + ++metrics->bypasses; + goto bypass; + } + cache->map = *map; + cache->nid = vi->nid; + cache->decoded_size = decoded_size; + cache->charged_bytes = decoded_size; + cache->error = 0; + cache->state = EROFS_ZCACHE_INFLIGHT; + ++metrics->misses; + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_OWNER); + +bypass: + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_BYPASS); +} + +static void +z_erofs_extent_cache_complete(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, void *decoded, int error) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + + KASSERT((error == 0) == (decoded != NULL), + ("erofs extent cache completion is untyped")); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + KASSERT(cache->state == EROFS_ZCACHE_INFLIGHT && + z_erofs_extent_cache_match(cache, vi, map, decoded_size), + ("erofs extent cache owner lost its key")); + metrics = &cache->metrics[map->m_algorithmformat]; + if (error == 0) { + KASSERT(cache->charged_bytes == decoded_size && + metrics->resident_bytes == 0, + ("erofs decoded cache success accounting mismatch")); + cache->data = decoded; + metrics->resident_bytes = decoded_size; + cache->error = 0; + cache->state = EROFS_ZCACHE_READY; + } else { + KASSERT(error > 0, ("erofs extent cache published negative errno")); + z_erofs_extent_cache_release(cache->charged_bytes); + cache->charged_bytes = 0; + cache->error = error; + cache->state = EROFS_ZCACHE_FAILED; + if (cache->waiters == 0) { + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + } + } + cv_broadcast(&cache->cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_extent_cache_lowmem(void *arg, int howto) +{ + struct erofs_zextent_cache *cache; + struct erofs_sb_info *sbi; + void *data; + + (void)arg; + (void)howto; + mtx_lock(&z_erofs_cache_list_lock); + LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) { + data = NULL; + if (!mtx_trylock(&sbi->z_extent_cache_lock)) + continue; + cache = &sbi->z_extent_cache; + if (cache->state == EROFS_ZCACHE_READY && cache->waiters == 0) { + data = z_erofs_extent_cache_drop_locked(cache, true, true); + cache->state = EROFS_ZCACHE_EMPTY; + } + mtx_unlock(&sbi->z_extent_cache_lock); + free(data, M_EROFS); + } + mtx_unlock(&z_erofs_cache_list_lock); +} + +static void +z_erofs_extent_cache_global_init(void *arg) +{ + + (void)arg; + z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +z_erofs_extent_cache_global_fini(void *arg) +{ + + (void)arg; + if (z_erofs_cache_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag); + KASSERT(LIST_EMPTY(&z_erofs_cache_mounts), + ("erofs decoded cache mounts remain at unload")); + KASSERT(z_erofs_cache_global_charged == 0, + ("erofs decoded cache bytes remain at unload")); +} + +SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY, + z_erofs_extent_cache_global_init, NULL); +SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY, + z_erofs_extent_cache_global_fini, NULL); + +void +z_erofs_extent_cache_init(struct erofs_sb_info *sbi) +{ + + bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache)); + sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget, + EROFS_ZCACHE_MOUNT_HARD_BUDGET); + sbi->z_extent_cache.minimum_decode_work = + z_erofs_cache_minimum_decode_work; + mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF); + cv_init(&sbi->z_extent_cache.cv, "erofs zextent"); + sbi->z_extent_cache_initialized = true; + mtx_lock(&z_erofs_cache_list_lock); + LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); +} + +void +z_erofs_extent_cache_fini(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + void *data; + + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&z_erofs_cache_list_lock); + LIST_REMOVE(sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); + mtx_lock(&sbi->z_extent_cache_lock); + cache->closing = true; + while (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) + cv_wait(&cache->cv, &sbi->z_extent_cache_lock); + data = z_erofs_extent_cache_drop_locked(cache, false, false); + cache->error = 0; + cache->state = EROFS_ZCACHE_EMPTY; + sbi->z_extent_cache_initialized = false; + mtx_unlock(&sbi->z_extent_cache_lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&sbi->z_extent_cache_lock); +} + +static bool +z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t len) +{ + const struct erofs_zextent_cache *cache; + uint64_t decode_work; + + cache = &sbi->z_extent_cache; + if (map->m_plen > UINT64_MAX - len) + decode_work = UINT64_MAX; + else + decode_work = map->m_plen + len; + return (z_erofs_cache_enabled != 0 && len <= MAXPHYS && + len <= cache->budget_bytes && + decode_work >= cache->minimum_decode_work && + (map->m_flags & (EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT)) == 0 && + map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX && + sbi->z_extent_cache_initialized && vi != sbi->packed_inode && + vi != sbi->metabox_en); +} + +static void +z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map) +{ + struct erofs_zextent_cache *cache; + + if (!sbi->z_extent_cache_initialized || + map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + ++cache->metrics[map->m_algorithmformat].bypasses; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static int +z_erofs_decode_length(const struct erofs_map_blocks *map, + erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial) +{ + size_t end; + + if (mapoff > map->m_llen || want > map->m_llen - mapoff) + return (EINTEGRITY); +#if SIZE_MAX < UINT64_MAX + if (map->m_llen > SIZE_MAX || mapoff > SIZE_MAX) + return (EOVERFLOW); +#endif + if ((size_t)mapoff > SIZE_MAX - want) + return (EOVERFLOW); + end = (size_t)mapoff + want; + *partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0; + if (!*partial && (!z_erofs_decompress_supports_subextent(map) || + end == map->m_llen)) { + *decoded_len = (size_t)map->m_llen; + return (0); + } + *decoded_len = end; + *partial = true; + return (0); +} + +static int +z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + const void *input; + void *compressed, *decoded; + int error; + + *bufp = NULL; + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) + return (EINTEGRITY); + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (EINTEGRITY); +#if SIZE_MAX < UINT64_MAX + if (map->m_plen > SIZE_MAX || map->m_llen > SIZE_MAX) + return (EOVERFLOW); +#endif + if (decoded_len == 0 || decoded_len > map->m_llen) + return (EINTEGRITY); + if (!partial && decoded_len != map->m_llen) + return (EINTEGRITY); + + if ((map->m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata(sbi, vi->nid, map->m_pa, + (size_t)map->m_plen, &buf); + } else { + error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa, + (size_t)map->m_plen, &compressed); + } + if (error != 0) + return (error); + input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed; + + decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO); + error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen, + decoded, decoded_len, partial); + if ((map->m_flags & EROFS_MAP_META) != 0) + erofs_put_metabuf(&buf); + else + erofs_brelse(compressed); + if (error != 0) { + free(decoded, M_EROFS); + return (error); + } + *bufp = decoded; + return (0); +} + +static int +z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, char *out) +{ + struct erofs_map_blocks map; + void *decoded, *fragment; + erofs_off_t mapoff; + size_t decoded_len, done, want; + bool cache_eligible, partial; + enum z_erofs_cache_lookup cache_lookup; + int error; + + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) + return (EINTEGRITY); + mapoff = loff + done - map.m_la; +#if SIZE_MAX < UINT64_MAX + if (mapoff > SIZE_MAX) + return (EOVERFLOW); + if (map.m_llen - mapoff > SIZE_MAX) + return (EOVERFLOW); +#endif + want = MIN((size_t)(map.m_llen - mapoff), len - done); + if (want == 0) + return (EINTEGRITY); + + if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > UINT64_MAX - mapoff) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->packed_inode, + vi->z_fragmentoff + mapoff, want, &fragment); + if (error != 0) + return (error); + memcpy(out + done, fragment, want); + erofs_brelse(fragment); + } else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + error = z_erofs_decode_length(&map, mapoff, want, + &decoded_len, &partial); + if (error != 0) + return (error); + cache_eligible = !partial && + z_erofs_extent_cache_eligible(sbi, vi, &map, decoded_len); + cache_lookup = Z_EROFS_CACHE_BYPASS; + if (cache_eligible) { + cache_lookup = z_erofs_extent_cache_claim(sbi, vi, &map, + decoded_len, mapoff, want, out + done, &error); + if (cache_lookup == Z_EROFS_CACHE_HIT) { + done += want; + continue; + } + if (cache_lookup == Z_EROFS_CACHE_ERROR) + return (error); + } else + z_erofs_extent_cache_record_bypass(sbi, &map); + error = z_erofs_decode_extent(sbi, vi, &map, decoded_len, + partial, &decoded); + if (error != 0) { + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, + decoded_len, NULL, error); + return (error); + } + memcpy(out + done, (char *)decoded + (size_t)mapoff, want); + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, decoded_len, + decoded, 0); + else + free(decoded, M_EROFS); + } + done += want; + } + return (0); +} + +int +z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) + return (0); + if (loff > UINT64_MAX - len) + return (EOVERFLOW); + if (loff > vi->size || len > vi->size - loff) + return (EINTEGRITY); + + out = malloc(len, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, loff, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); +} + +int +z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + char *buf; + size_t want; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + want = MIN((size_t)uio->uio_resid, + (size_t)MIN((uint64_t)MAXPHYS, + vi->size - (uint64_t)uio->uio_offset)); + buf = malloc(want, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset, + want, buf); + if (error == 0) + error = uiomove(buf, want, uio); + free(buf, M_EROFS); + if (error != 0) + return (error); + } + return (0); +} diff --git a/zmap.c b/zmap.c new file mode 100644 index 0000000..48732f2 --- /dev/null +++ b/zmap.c @@ -0,0 +1,1108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include + +#include "internal.h" + +#define Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE (64 * 1024) + +struct z_erofs_maprecorder { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn; + uint8_t type; + uint8_t headtype; + unsigned int clusterofs; + uint16_t delta[2]; + erofs_blk_t pblk; + erofs_blk_t compressedblks; + erofs_off_t nextpackoff; + bool partialref; +}; + +_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag"); + +static int +z_erofs_index_base(const struct erofs_inode *vi, erofs_off_t *base) +{ + erofs_off_t end; + + if (__builtin_add_overflow(vi->inode_off, + (erofs_off_t)vi->inode_isize, &end) || + __builtin_add_overflow(end, (erofs_off_t)vi->xattr_isize, &end) || + __builtin_add_overflow(end, (erofs_off_t)7, &end)) + return (EINTEGRITY); + end = rounddown2(end, 8); + if (__builtin_add_overflow(end, + (erofs_off_t)sizeof(struct z_erofs_map_header), base)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_index_advance(erofs_off_t *pos, uint64_t count, uint64_t unit) +{ + erofs_off_t delta, next; + + if (__builtin_mul_overflow(count, unit, &delta) || + __builtin_add_overflow(*pos, delta, &next)) + return (EINTEGRITY); + *pos = next; + return (0); +} + +static int +z_erofs_lcluster_count(uint64_t size, unsigned int lclusterbits, + uint64_t *count) +{ + uint64_t mask; + + if (lclusterbits >= 64) + return (EINTEGRITY); + mask = (1ULL << lclusterbits) - 1; + *count = size >> lclusterbits; + if ((size & mask) != 0 && __builtin_add_overflow(*count, 1, count)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_lcluster_pos(uint64_t lcn, unsigned int lclusterbits, + uint64_t clusterofs, uint64_t *pos) +{ + uint64_t base, clustersize; + + if (lclusterbits >= 64) + return (EINTEGRITY); + clustersize = 1ULL << lclusterbits; + if (__builtin_mul_overflow(lcn, clustersize, &base) || + __builtin_add_overflow(base, clusterofs, pos)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_lcn_advance(uint64_t *lcn, uint64_t delta) +{ + uint64_t next; + + if (__builtin_add_overflow(*lcn, delta, &next)) + return (EINTEGRITY); + *lcn = next; + return (0); +} + +static int +z_erofs_compact_pblk(uint32_t base, unsigned int nblk, erofs_blk_t *pblk) +{ + if (__builtin_add_overflow((erofs_blk_t)base, (erofs_blk_t)nblk, + pblk)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_fragment_offset(uint64_t low, erofs_blk_t high, erofs_off_t *offset) +{ + if (low > UINT32_MAX || high > UINT32_MAX) + return (EINTEGRITY); + *offset = low | (high << 32); + return (0); +} + +static int +z_erofs_post_eof_len(uint64_t la, uint64_t size, uint64_t *len) +{ + if (la < size) + return (EINTEGRITY); + *len = la - size; + if (__builtin_add_overflow(*len, 1, len)) + *len = UINT64_MAX; + return (0); +} + +static int +z_erofs_physical_end(uint64_t pa, uint64_t plen, uint64_t block_size) +{ + uint64_t limit, pend; + + if (__builtin_add_overflow(pa, plen, &pend)) + return (EINTEGRITY); + if (__builtin_mul_overflow(1ULL << 48, block_size, &limit)) + return (0); + if (pend > limit) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len, + struct erofs_buf *buf) +{ + return (erofs_read_metadata(m->sbi, m->vi->nid, pos, len, buf)); +} + +static int +z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + struct z_erofs_lcluster_index *di; + erofs_off_t base, pos; + unsigned int advise; + int error; + + vi = m->vi; + if (z_erofs_index_base(vi, &base) != 0 || + z_erofs_index_advance(&base, 1, 8) != 0) + return (EINTEGRITY); + pos = base; + if (z_erofs_index_advance(&pos, lcn, sizeof(*di)) != 0) + return (EINTEGRITY); + m->nextpackoff = pos; + if (z_erofs_index_advance(&m->nextpackoff, 1, sizeof(*di)) != 0) + return (EINTEGRITY); + error = z_erofs_read_index(m, pos, sizeof(*di), &buf); + if (error != 0) + return (error); + + di = buf.data; + m->lcn = lcn; + advise = le16toh(di->di_advise); + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + m->delta[0] = le16toh(di->di_u.delta[0]); + if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->compressedblks = + m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16toh(di->di_u.delta[1]); + } else { + m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0; + m->clusterofs = le16toh(di->di_clusterofs); + m->pblk = le32toh(di->di_u.blkaddr); + } + erofs_put_metabuf(&buf); + return (0); +} + +static unsigned int +decode_compactedbits(unsigned int lobits, const uint8_t *in, + unsigned int pos, uint8_t *type) +{ + uint32_t value; + unsigned int lo; + + value = le32dec(in + pos / 8) >> (pos & 7); + lo = value & ((1U << lobits) - 1); + *type = (value >> lobits) & 3; + return (lo); +} + +static int +get_compacted_la_distance(unsigned int lobits, unsigned int encodebits, + unsigned int vcnt, const uint8_t *in, int i) +{ + unsigned int lo, distance; + uint8_t type; + + distance = 0; + do { + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + return (distance); + ++distance; + } while (++i < (int)vcnt); + + if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0) { + if (lo == 0) + return (-1); + distance += lo - 1; + } + return ((int)distance); +} + +static int +z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + erofs_off_t ebase, pos; + uint64_t compacted_2b, totalidx, original_lcn; + unsigned int compacted_4b_initial, amortizedshift; + unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize; + bool big_pcluster; + uint8_t *in, type; + int distance, error, i; + + vi = m->vi; + if (vi->z_lclusterbits > 14 || + z_erofs_index_base(vi, &ebase) != 0 || + z_erofs_lcluster_count(vi->size, vi->z_lclusterbits, + &totalidx) != 0) + return (EINTEGRITY); + if (lcn >= totalidx) + return (EINVAL); + + original_lcn = lcn; + m->lcn = lcn; + compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; + compacted_2b = 0; + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 && + compacted_4b_initial < totalidx) + compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16); + + pos = ebase; + amortizedshift = 2; + if (lcn >= compacted_4b_initial) { + if (z_erofs_index_advance(&pos, compacted_4b_initial, 4) != 0) + return (EINTEGRITY); + lcn -= compacted_4b_initial; + if (lcn < compacted_2b) { + amortizedshift = 1; + } else { + if (z_erofs_index_advance(&pos, compacted_2b, 2) != 0) + return (EINTEGRITY); + lcn -= compacted_2b; + } + } + if (z_erofs_index_advance(&pos, lcn, 1U << amortizedshift) != 0) + return (EINTEGRITY); + + if (amortizedshift == 2 && vi->z_lclusterbits <= 14) + vcnt = 2; + else if (amortizedshift == 1 && vi->z_lclusterbits <= 12) + vcnt = 16; + else + return (EOPNOTSUPP); + + packsize = vcnt << amortizedshift; + bytes = pos & (packsize - 1); + pos -= bytes; + m->nextpackoff = pos; + if (z_erofs_index_advance(&m->nextpackoff, 1, packsize) != 0) + return (EINTEGRITY); + error = z_erofs_read_index(m, pos, packsize, &buf); + if (error != 0) + return (error); + in = buf.data; + lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT)); + encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt; + i = bytes >> amortizedshift; + + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + if (lookahead) { + distance = get_compacted_la_distance(lobits, encodebits, + vcnt, in, i); + if (distance < 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->delta[1] = distance; + } + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if (!big_pcluster) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + } else { + if (i == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + lo = decode_compactedbits(lobits, in, + encodebits * (i - 1), &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + lo = 0; + else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) + lo = 1; + m->delta[0] = lo + 1; + } + erofs_put_metabuf(&buf); + return (m->delta[0] == 0 ? EINTEGRITY : 0); + } + + m->clusterofs = lo; + m->delta[0] = 0; + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) + i -= lo; + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + --i; + nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; + continue; + } + if (lo <= 1) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + error = z_erofs_compact_pblk( + le32dec(in + packsize - sizeof(uint32_t)), nblk, &m->pblk); + erofs_put_metabuf(&buf); + if (error != 0) + return (error); + m->lcn = original_lcn; + return (0); +} + +static int +z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + int error; + + if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) + error = z_erofs_load_compact_lcluster(m, lcn, lookahead); + else if (m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL) + error = z_erofs_load_full_lcluster(m, lcn); + else + return (EINTEGRITY); + if (error != 0) + return (error); + if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX) + return (EOPNOTSUPP); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->clusterofs >= (1U << m->vi->z_lclusterbits)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + uint64_t lcn; + int error; + + while (lookback_distance != 0 && m->lcn >= lookback_distance) { + lcn = m->lcn - lookback_distance; + error = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + lookback_distance = m->delta[0]; + continue; + } + m->headtype = m->type; + error = z_erofs_lcluster_pos(lcn, m->vi->z_lclusterbits, + m->clusterofs, &m->map->m_la); + if (error != 0) + return (error); + return (0); + } + return (EINTEGRITY); +} + +static int +z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, + uint64_t initial_lcn) +{ + struct erofs_inode *vi; + bool bigpcl1, bigpcl2; + uint64_t lcn, lcnpos; + int error; + + vi = m->vi; + bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0; + lcn = m->lcn; + if (z_erofs_lcn_advance(&lcn, 1) != 0 || + z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0, &lcnpos) != 0) + return (EINTEGRITY); + if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || + ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || + m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || + lcnpos >= vi->size) + m->compressedblks = 1; + if (m->compressedblks == 0) { + error = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->delta[0] != 1) + return (EINTEGRITY); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD || + m->compressedblks == 0) + m->compressedblks = 1; + } + if (__builtin_mul_overflow(m->compressedblks, + (uint64_t)m->sbi->block_size, &m->map->m_plen)) + return (EINTEGRITY); + (void)initial_lcn; + return (0); +} + +static int +z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn, headlcn, lend, lcnpos; + int error; + + vi = m->vi; + map = m->map; + lcn = m->lcn; + headlcn = map->m_la >> vi->z_lclusterbits; + for (;;) { + if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0, + &lcnpos) != 0) + return (EINTEGRITY); + if (lcnpos >= vi->size) { + if (map->m_la > vi->size) + return (EINTEGRITY); + map->m_llen = vi->size - map->m_la; + return (0); + } + error = z_erofs_load_lcluster_from_disk(m, lcn, true); + if (error != 0) + return (error); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (m->delta[1] == 0) + m->delta[1] = 1; + } else { + if (lcn != headlcn) + break; + m->delta[1] = 1; + } + if (z_erofs_lcn_advance(&lcn, m->delta[1]) != 0) + return (EINTEGRITY); + } + if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, m->clusterofs, + &lend) != 0 || lend < map->m_la) + return (EINTEGRITY); + map->m_llen = lend - map->m_la; + return (0); +} + +static int +z_erofs_extent_add(uint64_t left, uint64_t right, uint64_t *result) +{ + if (__builtin_add_overflow(left, right, result)) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_roundup(uint64_t value, unsigned int alignment, + uint64_t *result) +{ + uint64_t rounded; + + if (z_erofs_extent_add(value, alignment - 1, &rounded) != 0) + return (EINTEGRITY); + *result = rounddown2(rounded, alignment); + return (0); +} + +static int +z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz, + erofs_off_t *result) +{ + erofs_off_t pos; + + if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 || + z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 || + z_erofs_extent_roundup(pos, 8, &pos) != 0 || + z_erofs_extent_add(pos, sizeof(struct z_erofs_map_header), &pos) != 0 || + z_erofs_extent_roundup(pos, recsz, result) != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_extent_record_pos(const struct erofs_inode *vi, erofs_off_t table_pos, + unsigned int recsz, uint64_t index, erofs_off_t *result) +{ + uint64_t offset; + + if (index >= vi->z_extents || + __builtin_mul_overflow(index, recsz, &offset) || + z_erofs_extent_add(table_pos, offset, result) != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + int error; + + bzero(ext, sizeof(*ext)); + error = erofs_read_metadata(sbi, vi->nid, pos, recsz, &buf); + if (error != 0) + return (error); + memcpy(ext, buf.data, recsz); + erofs_put_metabuf(&buf); + return (0); +} + +static uint64_t +z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz) +{ + uint64_t lstart; + + lstart = le32toh(ext->lstart_lo); + if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) + lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32; + return (lstart); +} + +static int +z_erofs_validate_extent_table(struct erofs_sb_info *sbi, struct erofs_inode *vi, + unsigned int recsz) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + erofs_off_t extent_pos, record_pos, scan_pos, table_end; + uint64_t index, lstart, previous; + size_t chunk_len, offset; + int error; + + if (vi->z_extents == 0) + return (vi->size == 0 ? 0 : EINTEGRITY); + error = z_erofs_extent_table_pos(vi, recsz, &extent_pos); + if (error != 0) + return (error); + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo) && + z_erofs_extent_add(extent_pos, sizeof(uint64_t), &extent_pos) != 0) + return (EINTEGRITY); + error = z_erofs_extent_record_pos(vi, extent_pos, recsz, + vi->z_extents - 1, &record_pos); + if (error != 0 || + z_erofs_extent_add(record_pos, recsz, &table_end) != 0) + return (EINTEGRITY); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) + return (z_erofs_read_extent(sbi, vi, record_pos, recsz, &ext)); + if (vi->size == 0) + return (EINTEGRITY); + + scan_pos = extent_pos; + index = 0; + previous = 0; + while (scan_pos < table_end) { + chunk_len = (size_t)MIN(table_end - scan_pos, + (uint64_t)Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE); + error = erofs_read_metadata(sbi, vi->nid, scan_pos, chunk_len, &buf); + if (error != 0) + return (error); + for (offset = 0; offset < chunk_len; offset += recsz, index++) { + bzero(&ext, sizeof(ext)); + memcpy(&ext, (const char *)buf.data + offset, recsz); + lstart = z_erofs_extent_lstart(&ext, recsz); + if (lstart >= vi->size || + (index != 0 && lstart <= previous)) { + error = EINTEGRITY; + goto out; + } + previous = lstart; + } + erofs_put_metabuf(&buf); + scan_pos += chunk_len; + } + return (index == vi->z_extents ? 0 : EINTEGRITY); +out: + erofs_put_metabuf(&buf); + return (error); +} + +static int +z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + bool fragment, ztailpacking; + struct z_erofs_maprecorder m; + uint64_t initial_lcn, ofs, end, end_lcn; + erofs_off_t fragmentoff; + unsigned int endoff; + int error; + + fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + ztailpacking = vi->z_idata_size != 0; + bzero(&m, sizeof(m)); + m.sbi = sbi; + m.vi = vi; + m.map = map; + + if (vi->size == 0) { + map->m_la = 0; + map->m_llen = 0; + map->m_flags = 0; + return (0); + } + ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ? + vi->size - 1 : map->m_la; + if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 && + vi->z_tailextent_headlcn == 0) { + map->m_la = 0; + map->m_llen = vi->size; + map->m_flags = EROFS_MAP_FRAGMENT; + return (0); + } + initial_lcn = ofs >> vi->z_lclusterbits; + endoff = ofs & ((1U << vi->z_lclusterbits) - 1); + error = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); + if (error != 0) + return (error); + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking) + vi->z_fragmentoff = m.nextpackoff; + + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; + end_lcn = m.lcn; + if (z_erofs_lcn_advance(&end_lcn, 1) != 0 || + z_erofs_lcluster_pos(end_lcn, vi->z_lclusterbits, 0, &end) != 0) + return (EINTEGRITY); + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + endoff >= m.clusterofs) { + m.headtype = m.type; + if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits, + m.clusterofs, &map->m_la) != 0) + return (EINTEGRITY); + if (ztailpacking && end > vi->size) + end = vi->size; + } else { + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits, + m.clusterofs, &end) != 0) + return (EINTEGRITY); + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + m.delta[0] = 1; + } + error = z_erofs_extent_lookback(&m, m.delta[0]); + if (error != 0) + return (error); + } + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + if (end < map->m_la) + return (EINTEGRITY); + map->m_llen = end - map->m_la; + + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) { + vi->z_tailextent_headlcn = m.lcn; + if (fragment && + vi->datalayout == EROFS_INODE_COMPRESSED_FULL) { + error = z_erofs_fragment_offset(vi->z_fragmentoff, m.pblk, + &fragmentoff); + if (error != 0) + return (error); + vi->z_fragmentoff = fragmentoff; + } + } + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_fragmentoff; + map->m_plen = vi->z_idata_size; + if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen > + sbi->block_size) + return (EINTEGRITY); + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags = EROFS_MAP_FRAGMENT; + } else { + if (__builtin_mul_overflow(m.pblk, (erofs_blk_t)sbi->block_size, + &map->m_pa)) + return (EINTEGRITY); + error = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (error != 0) + return (error); + } + + if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { + map->m_algorithmformat = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 ? + Z_EROFS_COMPRESSION_INTERLACED : + Z_EROFS_COMPRESSION_SHIFTED; + } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { + map->m_algorithmformat = vi->z_algorithmtype[1]; + } else { + map->m_algorithmformat = vi->z_algorithmtype[0]; + } + + if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0 || + ((flags & EROFS_GET_BLOCKS_READMORE) != 0 && + (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA || + map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE || + map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) && + map->m_llen >= sbi->block_size)) { + error = z_erofs_get_extent_decompressedlen(&m); + if (error == 0) + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + return (error); + } + return (0); +} + +static int +z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + unsigned int recsz, bmask; + uint8_t fmt; + uint64_t cluster_size, extent_idx, next, rounded_lend; + erofs_off_t extent_pos, pos, table_pos; + uint64_t fragmentoff, lend, l, r, mid, pa, la, lstart; + bool interlaced, last; + int error; + + (void)flags; + interlaced = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0; + recsz = z_erofs_extent_recsize(vi->z_advise); + error = z_erofs_extent_table_pos(vi, recsz, &table_pos); + if (error != 0) + return (error); + pos = table_pos; + bmask = sbi->block_size - 1; + lend = vi->size; + cluster_size = 1ULL << vi->z_lclusterbits; + map->m_flags = 0; + + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { + error = erofs_read_metadata(sbi, vi->nid, pos, + sizeof(uint64_t), &buf); + if (error != 0) + return (error); + pa = le64dec(buf.data); + erofs_put_metabuf(&buf); + if (z_erofs_extent_add(pos, sizeof(uint64_t), &pos) != 0) + return (EINTEGRITY); + lstart = 0; + extent_idx = 0; + } else { + lstart = rounddown2(map->m_la, cluster_size); + extent_idx = lstart >> vi->z_lclusterbits; + pa = EROFS_NULL_ADDR; + } + for (;;) { + error = z_erofs_extent_record_pos(vi, pos, recsz, + extent_idx, &extent_pos); + if (error != 0) + return (error); + error = z_erofs_read_extent(sbi, vi, extent_pos, recsz, &ext); + if (error != 0) + return (error); + map->m_plen = le32toh(ext.plen); + if (pa != EROFS_NULL_ADDR) { + map->m_pa = pa; + if (z_erofs_extent_add(pa, + map->m_plen & Z_EROFS_EXTENT_PLEN_MASK, + &next) != 0) + return (EINTEGRITY); + pa = next; + } else { + map->m_pa = le32toh(ext.pstart_lo); + } + if (extent_idx == UINT64_MAX) + return (EINTEGRITY); + extent_idx++; + if (z_erofs_extent_add(lstart, cluster_size, &next) != 0) + return (EINTEGRITY); + lstart = next; + if (lstart > map->m_la) + break; + } + if (z_erofs_extent_roundup(lend, cluster_size, &rounded_lend) != 0) + return (EINTEGRITY); + last = lstart >= rounded_lend; + lend = MIN(lstart, lend); + lstart -= cluster_size; + } else { + lstart = lend; + for (l = 0, r = vi->z_extents; l < r;) { + mid = l + (r - l) / 2; + error = z_erofs_extent_record_pos(vi, table_pos, recsz, mid, + &extent_pos); + if (error != 0) + return (error); + error = z_erofs_read_extent(sbi, vi, extent_pos, + recsz, &ext); + if (error != 0) + return (error); + la = z_erofs_extent_lstart(&ext, recsz); + pa = le32toh(ext.pstart_lo) | + ((uint64_t)le32toh(ext.pstart_hi) << 32); + if (la > map->m_la) { + r = mid; + if (la > lend) + return (EINTEGRITY); + lend = la; + } else { + l = mid + 1; + if (map->m_la == la) + r = MIN(l + 1, r); + lstart = la; + map->m_plen = le32toh(ext.plen); + map->m_pa = pa; + } + } + last = l >= vi->z_extents; + } + + if (lstart < lend) { + map->m_la = lstart; + if (last && + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + map->m_flags = EROFS_MAP_FRAGMENT; + if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) { + error = z_erofs_fragment_offset(map->m_plen, map->m_pa, + &fragmentoff); + if (error != 0) + return (error); + vi->z_fragmentoff = fragmentoff; + } else { + vi->z_fragmentoff = map->m_plen; + } + } else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) { + map->m_flags = EROFS_MAP_MAPPED; + if ((map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT) > + UINT8_MAX) + return (EINTEGRITY); + fmt = (uint8_t)(map->m_plen >> + Z_EROFS_EXTENT_PLEN_FMT_BIT); + if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; + if (fmt != 0) + map->m_algorithmformat = (uint8_t)(fmt - 1); + else if (interlaced && + ((map->m_pa | map->m_plen) & bmask) == 0) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } + } + map->m_llen = lend - map->m_la; + return (0); +} + +int +z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_map_header *h; + struct erofs_map_blocks map; + uint64_t cluster_size, raw, rounded_size; + erofs_off_t pos; + unsigned int recsz; + int error; + + if (vi->z_initialized) + return (0); + if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 || + z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 || + z_erofs_extent_roundup(pos, 8, &pos) != 0) + return (EINTEGRITY); + error = erofs_read_metadata(sbi, vi->nid, pos, sizeof(*h), &buf); + if (error != 0) + return (error); + h = buf.data; + if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) { + if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + raw = le64dec(h); + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = raw ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + vi->fragment = true; + erofs_put_metabuf(&buf); + vi->z_initialized = true; + return (0); + } + + vi->z_advise = le16toh(h->h_advise); + vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15); + if (vi->z_lclusterbits >= 31) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) { + recsz = z_erofs_extent_recsize(vi->z_advise); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + cluster_size = 1ULL << vi->z_lclusterbits; + if (z_erofs_extent_roundup(vi->size, cluster_size, + &rounded_size) != 0) { + erofs_put_metabuf(&buf); + return (EINTEGRITY); + } + vi->z_extents = rounded_size >> vi->z_lclusterbits; + } else { + vi->z_extents = le32toh(h->h_extents_lo) | + ((uint64_t)le16toh(h->h_extents_hi) << 32); + } + vi->fragment = + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + erofs_put_metabuf(&buf); + if (vi->fragment && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (EINTEGRITY); + if (recsz > offsetof(struct z_erofs_extent, pstart_hi) && + vi->z_extents == 0 && vi->size != 0) + return (EINTEGRITY); + error = z_erofs_validate_extent_table(sbi, vi, recsz); + if (error != 0) + return (error); + vi->z_initialized = true; + return (0); + } + vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15); + vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4); + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) + vi->z_fragmentoff = le32toh(h->h_fragmentoff); + else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0) + vi->z_idata_size = le16toh(h->h_idata_size); + erofs_put_metabuf(&buf); + + if (!erofs_sb_has_big_pcluster(sbi) && + (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0) + return (EINTEGRITY); + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && + (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) != + ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0))) + return (EINTEGRITY); + if (vi->z_idata_size != 0 || + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (EINTEGRITY); + bzero(&map, sizeof(map)); + error = z_erofs_map_blocks_fo(sbi, vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (error != 0) + return (error); + } + vi->fragment = + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + vi->z_initialized = true; + return (0); +} + +static int +z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) { + if ((map->m_flags & (EROFS_MAP_MAPPED | EROFS_MAP_META)) != 0 || + sbi->packed_inode == NULL || sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > sbi->packed_inode->size || + map->m_llen > sbi->packed_inode->size - vi->z_fragmentoff) + return (EINTEGRITY); + return (0); + } + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (0); + if (map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX) + return (EOPNOTSUPP); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if ((sbi->available_compr_algs & + (1U << map->m_algorithmformat)) == 0) + return (EINTEGRITY); + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen) + return (EINTEGRITY); + } else if (map->m_llen > map->m_plen) { + return (EINTEGRITY); + } + if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || + map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE) + return (EOPNOTSUPP); + if ((map->m_flags & EROFS_MAP_META) != 0) + return (0); + if (z_erofs_physical_end(map->m_pa, map->m_plen, + sbi->block_size) != 0) + return (EINTEGRITY); + (void)vi; + return (0); +} + +int +z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + int error; + + if (map->m_la >= vi->size) { + error = z_erofs_post_eof_len(map->m_la, vi->size, + &map->m_llen); + if (error != 0) + return (error); + map->m_la = vi->size; + map->m_flags = 0; + return (0); + } + error = z_erofs_fill_inode(sbi, vi); + if (error == 0) { + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) + error = z_erofs_map_blocks_ext(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + else + error = z_erofs_map_blocks_fo(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + } + if (error == 0) + error = z_erofs_map_sanity_check(sbi, vi, map); + if (error != 0) + map->m_llen = 0; + return (error); +}