commit 144eec51826d4c987de0d0cd54d4bc6130e1d4f5 Author: Ruicheng Pan Date: Mon Oct 19 08:35:59 2026 +0800 update diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..3d436fa --- /dev/null +++ b/.clang-format @@ -0,0 +1,199 @@ +# Basic .clang-format +--- +BasedOnStyle: WebKit +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: AcrossEmptyLines +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: TopLevelDefinitions +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeTernaryOperators: false +# TODO: BreakStringLiterals can cause very strange formatting so turn it off? +BreakStringLiterals: false +# Prefer: +# some_var = function(arg1, +# arg2) +# over: +# some_var = +# function(arg1, arg2) +PenaltyBreakAssignment: 100 +# Prefer: +# some_long_function(arg1, arg2 +# arg3) +# over: +# some_long_function( +# arg1, arg2, arg3) +PenaltyBreakBeforeFirstCallParameter: 100 +CompactNamespaces: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: + - ARB_ARRFOREACH + - ARB_ARRFOREACH_REVWCOND + - ARB_ARRFOREACH_REVERSE + - ARB_FOREACH + - ARB_FOREACH_FROM + - ARB_FOREACH_SAFE + - ARB_FOREACH_REVERSE + - ARB_FOREACH_REVERSE_FROM + - ARB_FOREACH_REVERSE_SAFE + - BIT_FOREACH_ISCLR + - BIT_FOREACH_ISSET + - CPU_FOREACH + - CPU_FOREACH_ISCLR + - CPU_FOREACH_ISSET + - FOREACH_THREAD_IN_PROC + - FOREACH_PROC_IN_SYSTEM + - FOREACH_PRISON_CHILD + - FOREACH_PRISON_DESCENDANT + - FOREACH_PRISON_DESCENDANT_LOCKED + - FOREACH_PRISON_DESCENDANT_LOCKED_LEVEL + - MNT_VNODE_FOREACH_ALL + - MNT_VNODE_FOREACH_ACTIVE + - RB_FOREACH + - RB_FOREACH_FROM + - RB_FOREACH_SAFE + - RB_FOREACH_REVERSE + - RB_FOREACH_REVERSE_FROM + - RB_FOREACH_REVERSE_SAFE + - SLIST_FOREACH + - SLIST_FOREACH_FROM + - SLIST_FOREACH_FROM_SAFE + - SLIST_FOREACH_SAFE + - SLIST_FOREACH_PREVPTR + - SPLAY_FOREACH + - LIST_FOREACH + - LIST_FOREACH_FROM + - LIST_FOREACH_FROM_SAFE + - LIST_FOREACH_SAFE + - STAILQ_FOREACH + - STAILQ_FOREACH_FROM + - STAILQ_FOREACH_FROM_SAFE + - STAILQ_FOREACH_SAFE + - TAILQ_FOREACH + - TAILQ_FOREACH_FROM + - TAILQ_FOREACH_FROM_SAFE + - TAILQ_FOREACH_REVERSE + - TAILQ_FOREACH_REVERSE_FROM + - TAILQ_FOREACH_REVERSE_FROM_SAFE + - TAILQ_FOREACH_REVERSE_SAFE + - TAILQ_FOREACH_SAFE + - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH +SpaceBeforeParens: ControlStatementsExceptForEachMacros +IndentCaseLabels: false +IndentPPDirectives: None +Language: Cpp +NamespaceIndentation: None +PointerAlignment: Right +ContinuationIndentWidth: 4 +IndentWidth: 8 +TabWidth: 8 +ColumnLimit: 80 +UseTab: Always +SpaceAfterCStyleCast: false +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^\"opt_.*\.h\"' + Priority: 1 + SortPriority: 10 + - Regex: '^' + Priority: 2 + SortPriority: 20 + - Regex: '^' + Priority: 2 + SortPriority: 21 + - Regex: '^' + Priority: 2 + SortPriority: 22 + - Regex: '^' + Priority: 2 + SortPriority: 23 + - Regex: '^' + Priority: 3 + SortPriority: 30 + - Regex: '^ diff --git a/compress.h b/compress.h new file mode 100644 index 0000000..6e9b117 --- /dev/null +++ b/compress.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _FS_EROFS_COMPRESS_H_ +#define _FS_EROFS_COMPRESS_H_ + +#include "internal.h" + +struct z_erofs_decompress_req { + struct erofs_sb_info *sbi; + const struct erofs_map_blocks *map; + const void *in; + size_t inputsize; + void *out; + size_t outputsize; + bool partial_decoding; +}; + +struct z_erofs_decompressor { + /* Callbacks return zero or a positive FreeBSD errno. */ + int (*config)(struct erofs_sb_info *, const struct erofs_super_block *, + const void *, size_t); + int (*decompress)(const struct z_erofs_decompress_req *); +}; + +int z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq); +extern const struct z_erofs_decompressor z_erofs_lz4_decomp; +extern const struct z_erofs_decompressor z_erofs_lzma_decomp; +extern const struct z_erofs_decompressor z_erofs_deflate_decomp; +extern const struct z_erofs_decompressor z_erofs_zstd_decomp; + +#endif diff --git a/data.c b/data.c new file mode 100644 index 0000000..5e2e5fb --- /dev/null +++ b/data.c @@ -0,0 +1,878 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "internal.h" +#include "trace.h" + +#define EROFS_DIR_READAHEAD_SLOTS 256 + +void +erofs_put_metabuf(struct erofs_buf *buf) +{ + void (*release)(void *); + void *data; + + if (buf == NULL) + return; + data = buf->data; + release = buf->release; + buf->data = NULL; + buf->release = NULL; + if (data != NULL && release != NULL) + release(data); +} + +/* + * bread() on a device vnode needs no vnode lock, but breadn() reads ahead + * through breada(), whose inmem() asserts that the vnode is locked. Take the + * lock shared (the mount's device vnode allows that) only around the request. + * + * A doomed vnode has become VBAD, which the buffer cache only accepts when it + * is locked, and it no longer has a device behind it, so fail the read. + */ +static int +erofs_breadn_devvp(struct vnode *devvp, daddr_t blkno, int size, + daddr_t *rablkno, int *rabsize, int racnt, struct buf **bpp) +{ + int error; + + vn_lock(devvp, LK_SHARED | LK_RETRY); + if (VN_IS_DOOMED(devvp)) { + VOP_UNLOCK(devvp); + *bpp = NULL; + return (ENXIO); + } + error = breadn(devvp, blkno, size, rablkno, rabsize, racnt, NOCRED, + bpp); + VOP_UNLOCK(devvp); + return (error); +} + +static int +erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno, + int *rabsize, int racnt, void **bufp) +{ + struct buf *bp; + erofs_off_t device_end, device_off, end, limit; + off_t blkoff, current; + size_t blklen, done, iosize; + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (!erofs_source_is_open(dif)) + return (ENODEV); + if (__builtin_add_overflow(off, (uint64_t)len, &end)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "read of %zu bytes at %ju overflows", len, (uintmax_t)off)); + if (blocks != 0) { + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "device size of %ju blocks overflows", + (uintmax_t)blocks)); + limit = blocks << sbi->blkszbits; + if (end > limit) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "read of %zu bytes at %ju is beyond the device " + "(%ju bytes)", len, (uintmax_t)off, + (uintmax_t)limit)); + } + if (__builtin_add_overflow(dif->fsoff, off, &device_off) || + __builtin_add_overflow(device_off, (uint64_t)len, &device_end)) + return (EOVERFLOW); + if (device_end > dif->mediasize) + return (ENXIO); + if (device_off > INT64_MAX || device_end > (uint64_t)INT64_MAX + 1) + return (EOVERFLOW); + + iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize; + if (iosize == 0 || (iosize & (iosize - 1)) != 0) + return (EINVAL); + out = malloc(len, M_EROFS, M_WAITOK); + if (erofs_source_is_file(dif)) { + error = erofs_read_file_source(sbi, dif, device_off, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); + } + done = 0; + while (done < len) { + current = (off_t)(device_off + done); + blkoff = rounddown2(current, (off_t)iosize); + blklen = MIN(iosize - (size_t)(current - blkoff), len - done); + if (done == 0 && racnt != 0) + error = erofs_breadn_devvp(dif->devvp, btodb(blkoff), + iosize, rablkno, rabsize, racnt, &bp); + else + error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, + &bp); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (bp->b_data == NULL) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + if (bp->b_resid < 0 || bp->b_bcount < 0 || + (uintmax_t)bp->b_resid > (uintmax_t)bp->b_bcount || + (uintmax_t)(current - blkoff) + (uintmax_t)blklen > + (uintmax_t)bp->b_bcount - (uintmax_t)bp->b_resid) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + memcpy(out + done, (char *)bp->b_data + (current - blkoff), + blklen); + brelse(bp); + done += blklen; + } + *bufp = out; + return (0); +} + +int +erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp) +{ + return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len, + NULL, NULL, 0, bufp)); +} + +void +erofs_brelse(void *buf) +{ + free(buf, M_EROFS); +} + +/* Read inode metadata from either the primary image or the metabox file. */ +int +erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf) +{ + void *data; + int error; + + if (!erofs_nid_in_metabox(nid)) { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } else { + if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL) + return (erofs_corrupt(sbi, nid, + "metabox nid, but the image has no metabox")); + error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_inode_chunk_index *idx; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t chunk_idx, chunk_size, entry_size; + erofs_off_t loff, chunk_start, idx_off, idx_base; + uint64_t image_size, addrmask; + uint64_t blkaddr; + uint16_t raw_device_id; + int error; + + loff = map->m_la; + chunk_size = 1ULL << vi->chunkbits; + chunk_idx = loff >> vi->chunkbits; + chunk_start = chunk_idx << vi->chunkbits; + + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) + entry_size = sizeof(struct erofs_inode_chunk_index); + else + entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE; + /* + * inode_off is below 2^63 (the image fits its provider, whose size is + * an off_t; a metabox is at most OFF_MAX), isize is 32 or 64 and + * xattr_isize is below 2^19; chunk_idx is below 2^54 because + * loff < size <= OFF_MAX and a chunk has at least 512 bytes. + */ + idx_base = roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize, + entry_size); + idx_off = idx_base + chunk_idx * entry_size; + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || + idx_off > sbi->metabox_en->size || + entry_size > sbi->metabox_en->size - idx_off) + return (erofs_corrupt(sbi, vi->nid, + "chunk %ju: index entry at %ju is beyond the " + "metabox", + (uintmax_t)chunk_idx, (uintmax_t)idx_off)); + } else { + image_size = sbi->blocks << sbi->blkszbits; + if (idx_off > image_size || entry_size > image_size - idx_off) + return (erofs_corrupt(sbi, vi->nid, + "chunk %ju: index entry at %ju is beyond the image", + (uintmax_t)chunk_idx, (uintmax_t)idx_off)); + } + + error = erofs_read_metadata_range(sbi, vi->nid, idx_off, entry_size, + &buf); + if (error != 0) + return (error); + + idx = buf.data; + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) { + blkaddr = le32toh(idx->startblk_lo); + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0) + blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32; + raw_device_id = le16toh(idx->device_id); + addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ? + ((1ULL << 48) - 1) : UINT32_MAX; + } else { + blkaddr = le32dec(idx); + raw_device_id = 0; + addrmask = UINT32_MAX; + } + erofs_put_metabuf(&buf); + + if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) { + map->m_la = chunk_start; + map->m_pa = 0; + map->m_llen = MIN(chunk_size, vi->size - chunk_start); + map->m_plen = map->m_llen; + return (0); + } + + map->m_la = chunk_start; + map->m_llen = MIN(chunk_size, vi->size - chunk_start); + map->m_plen = map->m_llen; + map->m_deviceid = raw_device_id & sbi->device_id_mask; + /* blkaddr has at most 48 bits and blkszbits at most 16. */ + map->m_pa = blkaddr << sbi->blkszbits; + map->m_flags |= EROFS_MAP_MAPPED; + return (0); +} + +static erofs_off_t +erofs_inline_tail_start(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + if (vi->size == 0) + return (0); + return (roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size); +} + +static int +erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + erofs_off_t loff, tail_start; + uint64_t remain, block_rem; + + loff = map->m_la; + if (loff >= vi->size) + return (0); + + remain = vi->size - loff; + switch (vi->datalayout) { + case EROFS_INODE_CHUNK_BASED: + return (erofs_map_blocks_chunk(sbi, vi, map)); + case EROFS_INODE_FLAT_PLAIN: + map->m_llen = remain; + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff, + &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "start block %ju overflows", + (uintmax_t)vi->startblk)); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + case EROFS_INODE_FLAT_INLINE: + tail_start = erofs_inline_tail_start(sbi, vi); + if (loff < tail_start) { + map->m_llen = MIN(remain, tail_start - loff); + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow( + vi->startblk << sbi->blkszbits, loff, &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "start block %ju overflows", + (uintmax_t)vi->startblk)); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + } + block_rem = sbi->block_size - + ((loff - tail_start) & (sbi->block_size - 1)); + map->m_llen = MIN(remain, block_rem); + map->m_plen = map->m_llen; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &map->m_pa) || + __builtin_add_overflow(map->m_pa, vi->xattr_isize, + &map->m_pa) || + __builtin_add_overflow(map->m_pa, loff - tail_start, + &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "inline tail position overflows")); + map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META; + return (0); + case EROFS_INODE_COMPRESSED_FULL: + case EROFS_INODE_COMPRESSED_COMPACT: + return (EOPNOTSUPP); + default: + return (EOPNOTSUPP); + } +} + +int +erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_map_blocks next = { .m_la = map->m_la }; + int err; + + EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, 0); + if (erofs_inode_is_data_compressed(vi->datalayout)) { +#if WITH_EROFS_ZIP + err = z_erofs_map_blocks(sbi, vi, &next); +#else + err = EOPNOTSUPP; +#endif + } else { + err = erofs_map_blocks_flatmode(sbi, vi, &next); + } + *map = next; + EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, err); + return (err); +} +static int +erofs_check_device_range(const struct erofs_sb_info *sbi, + const struct erofs_device_info *dif, erofs_off_t off, uint64_t len) +{ + erofs_off_t end, limit; + + if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(off, len, &end)) + return (EINTEGRITY); + limit = dif->blocks << sbi->blkszbits; + return (end > limit ? EINTEGRITY : 0); +} + +static void +erofs_fill_from_devinfo(struct erofs_map_dev *map, + struct erofs_device_info *dif, erofs_off_t pa) +{ + map->m_dif = dif; + map->m_pa = pa; +} + +static int +erofs_map_dev_locate(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + struct erofs_device_info *dif; + uint64_t start; + unsigned int id; + int error; + + erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa); + if (map->m_deviceid != 0) { + if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL) + return (ENODEV); + dif = &sbi->devs[map->m_deviceid - 1]; + error = erofs_check_device_range(sbi, dif, map->m_pa, + map->m_plen); + if (error != 0) + return (error); + if (sbi->flatdev) { + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (__builtin_add_overflow(map->m_pa, start, + &map->m_pa)) + return (EINTEGRITY); + return (0); + } + if (!erofs_source_is_open(dif)) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa); + return (0); + } + + if (sbi->extra_devices == 0) + return (0); + if (sbi->flatdev) { + error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa, + map->m_plen); + if (error == 0) + return (0); + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0 || + dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + continue; + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa < start) + continue; + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error == 0) + return (0); + if (map->m_pa - start < + (dif->blocks << sbi->blkszbits)) + return (error); + } + return (EINTEGRITY); + } + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0) + continue; + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa >= start && + map->m_pa - start < (dif->blocks << sbi->blkszbits)) { + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error != 0) + return (error); + if (!erofs_source_is_open(dif)) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa - start); + break; + } + } + return (0); +} + +/* + * Resolve a physical range to a device. The lookup tries several devices + * and treats a miss on one as normal, so the corruption is reported here, + * once, when no device is able to serve the range. + */ +static int +erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + uint64_t pa, plen; + unsigned int deviceid; + int error; + + pa = map->m_pa; + plen = map->m_plen; + deviceid = map->m_deviceid; + error = erofs_map_dev_locate(sbi, map); + if (error == EINTEGRITY) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "range %ju+%ju of device %u is not backed by any device", + (uintmax_t)pa, (uintmax_t)plen, deviceid)); + return (error); +} + +int +erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp) +{ + struct erofs_map_dev map; + erofs_blk_t blocks; + int error; + + map = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, &map); + if (error != 0) + return (error); + blocks = map.m_dif->blocks; + if (map.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len, + NULL, NULL, 0, bufp)); +} + +static int +erofs_read_physical_readahead(struct erofs_sb_info *sbi, + unsigned int device_id, erofs_off_t off, size_t len, + unsigned int rablocks, void **bufp) +{ + struct erofs_map_dev current, future; + daddr_t *rablkno; + int *rabsize; + erofs_off_t device_pa, step; + erofs_blk_t blocks; + unsigned int count; + int error, ret; + + current = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, ¤t); + if (error != 0) + return (error); + blocks = current.m_dif->blocks; + if (current.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + if (rablocks == 0) + return (erofs_bread_device(sbi, current.m_dif, blocks, + current.m_pa, len, NULL, NULL, 0, bufp)); + /* + * 256 slots are 2 KiB, which is a large part of a kernel stack inside + * the recursive read path, so keep the run lists off the stack. + */ + rablocks = MIN(rablocks, (unsigned int)EROFS_DIR_READAHEAD_SLOTS); + rablkno = malloc(rablocks * sizeof(*rablkno), M_EROFS, M_WAITOK); + rabsize = malloc(rablocks * sizeof(*rabsize), M_EROFS, M_WAITOK); + for (count = 0; count < rablocks; count++) { + step = (erofs_off_t)(count + 1) * sbi->block_size; + if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step) + break; + future = (struct erofs_map_dev) { + .m_pa = off + step, + .m_deviceid = device_id, + .m_plen = sbi->block_size, + }; + if (erofs_map_dev(sbi, &future) != 0 || + future.m_dif != current.m_dif || + future.m_pa != current.m_pa + step) + break; + if (__builtin_add_overflow(future.m_dif->fsoff, future.m_pa, + &device_pa) || device_pa > INT64_MAX) + break; + rablkno[count] = btodb(device_pa); + rabsize[count] = sbi->block_size; + } + ret = erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len, + rablkno, rabsize, count, bufp); + free(rabsize, M_EROFS); + free(rablkno, M_EROFS); + return (ret); +} + +static int +erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char *out; + void *blk; + erofs_off_t mapoff, physical; + size_t done, want; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + EROFS_TRACE_READ_FOLIO(vi, loff, len, + !erofs_inode_is_data_compressed(vi->datalayout)); + if (loff > vi->size || (uint64_t)len > vi->size - loff) + return (erofs_corrupt(sbi, vi->nid, + "read of %zu bytes at %ju is beyond the size %ju", len, + (uintmax_t)loff, (uintmax_t)vi->size)); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) +#if WITH_EROFS_ZIP + return (z_erofs_read_data(sbi, vi, loff, len, bufp)); +#else + return (EOPNOTSUPP); +#endif + + /* Uncompressed file path */ + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) { + free(out, M_EROFS); + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju is empty or does not cover " + "it", + (uintmax_t)(loff + done))); + } + mapoff = loff + done - map.m_la; + want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS), + len - done); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + if ((map.m_flags & EROFS_MAP_META) != 0) { + if (mapoff > UINT64_MAX - map.m_pa) { + free(out, M_EROFS); + return (EOVERFLOW); + } + physical = map.m_pa + mapoff; + error = erofs_read_metadata_range(sbi, vi->nid, + physical, want, &buf); + } else if (done == 0 && mapoff == 0 && rablocks != 0 && + map.m_flags == EROFS_MAP_MAPPED) { + error = erofs_read_physical_readahead(sbi, + map.m_deviceid, map.m_pa, want, rablocks, + &blk); + } else { + if (mapoff > UINT64_MAX - map.m_pa) { + free(out, M_EROFS); + return (EOVERFLOW); + } + physical = map.m_pa + mapoff; + error = erofs_read_physical(sbi, map.m_deviceid, + physical, want, &blk); + } + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + memcpy(out + done, buf.data, want); + erofs_put_metabuf(&buf); + } else { + memcpy(out + done, blk, want); + erofs_brelse(blk); + } + } + done += want; + } + *bufp = out; + return (0); +} + +int +erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp)); +} + +int +erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, bool sequential, void **bufp) +{ + uint64_t dir_ra_bytes, total_blocks; + uint64_t remaining; + unsigned int rablocks; + int error; + + rablocks = 0; + if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN && + sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 && + len <= sbi->block_size && loff <= vi->size && + len <= vi->size - loff) { + dir_ra_bytes = erofs_dir_ra_bytes(sbi); + total_blocks = dir_ra_bytes / sbi->block_size; + if (total_blocks > 1) { + remaining = vi->size - loff - len; + rablocks = MIN(total_blocks - 1, + howmany(remaining, sbi->block_size)); + rablocks = MIN(rablocks, + (unsigned int)EROFS_DIR_READAHEAD_SLOTS); + } + } + EROFS_TRACE_READAHEAD(vi, loff >> PAGE_SHIFT, + howmany((uint64_t)len + + (uint64_t)rablocks * (uint64_t)sbi->block_size, PAGE_SIZE), + !erofs_inode_is_data_compressed(vi->datalayout)); + error = erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp); + if (error == 0 && rablocks != 0) + erofs_record_dir_readahead(sbi, rablocks); + return (error); +} + +static int +erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + void *blk; + erofs_off_t mapoff, physical; + size_t want, chunk; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + if ((uint64_t)uio->uio_offset >= vi->size) + return (0); + EROFS_TRACE_READ_FOLIO(vi, (uint64_t)uio->uio_offset, + (uint64_t)uio->uio_resid, + !erofs_inode_is_data_compressed(vi->datalayout)); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) +#if WITH_EROFS_ZIP + return (z_erofs_read_uio(sbi, vi, uio)); +#else + return (EOPNOTSUPP); +#endif + + /* Uncompressed file path */ + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + map = (struct erofs_map_blocks) { .m_la = uio->uio_offset }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > (uint64_t)uio->uio_offset || + (uint64_t)uio->uio_offset - map.m_la >= map.m_llen) + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %jd is empty or does not cover " + "it", + (intmax_t)uio->uio_offset)); + mapoff = (uint64_t)uio->uio_offset - map.m_la; + want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS), + (size_t)uio->uio_resid); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + chunk = want; + while (chunk > 0) { + size_t zlen = MIN(chunk, ZERO_REGION_SIZE); + + error = uiomove(__DECONST(void *, zero_region), + zlen, uio); + if (error != 0) + return (error); + chunk -= zlen; + } + continue; + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + if (mapoff > UINT64_MAX - map.m_pa) + return (EOVERFLOW); + physical = map.m_pa + mapoff; + error = erofs_read_metadata_range(sbi, vi->nid, + physical, want, &buf); + } else { + if (mapoff > UINT64_MAX - map.m_pa) + return (EOVERFLOW); + physical = map.m_pa + mapoff; + error = erofs_read_physical(sbi, map.m_deviceid, + physical, want, &blk); + } + if (error != 0) + return (error); + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = uiomove(buf.data, want, uio); + erofs_put_metabuf(&buf); + } else { + error = uiomove(blk, want, uio); + erofs_brelse(blk); + } + if (error != 0) + return (error); + } + return (0); +} + +/* Read regular file data. */ +int +erofs_read_file(struct vnode *vp, struct uio *uio) +{ + return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio)); +} + +int +erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks map; + erofs_off_t pos, end; + bool data; + int error; + + if (vp->v_type != VREG) + return (ENOTTY); + vi = VTOE(vp); + if (*offset < 0 || (uint64_t)*offset >= vi->size) + return (ENXIO); + + sbi = MTOE(vp->v_mount); + pos = (erofs_off_t)*offset; + while (pos < vi->size) { + map = (struct erofs_map_blocks) { .m_la = pos }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_la > pos || map.m_llen == 0 || + map.m_la > UINT64_MAX - map.m_llen) { + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju is empty or does not cover " + "it", + (uintmax_t)pos)); + } + end = map.m_la + map.m_llen; + if (pos >= end) + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju ends at %ju", (uintmax_t)pos, + (uintmax_t)end)); + data = (map.m_flags & + (EROFS_MAP_MAPPED | EROFS_MAP_FRAGMENT)) != 0; + if (data == seek_data) { + *offset = (off_t)pos; + return (0); + } + pos = MIN(end, vi->size); + } + + if (seek_data) + return (ENXIO); + if (vi->size > OFF_MAX) + return (EOVERFLOW); + *offset = (off_t)vi->size; + return (0); +} + +/* Read symlink target string. */ +int +erofs_readlink_target(struct vnode *vp, struct uio *uio) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + if (vi->size == 0) + return (erofs_corrupt(MTOE(vp->v_mount), vi->nid, + "empty symlink")); + return (erofs_read_uio(MTOE(vp->v_mount), vi, uio)); +} diff --git a/decompressor.c b/decompressor.c new file mode 100644 index 0000000..82b3721 --- /dev/null +++ b/decompressor.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ + +#include +#include +#include +#include + +#include "compress.h" + +static int +z_erofs_load_lz4_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lz4_cfgs *lz4; + uint32_t max_pclusterblks; + uint16_t distance; + + if (data != NULL) { + if (size < sizeof(*lz4)) + return (EINTEGRITY); + lz4 = data; + max_pclusterblks = le16toh(lz4->max_pclusterblks); + if (max_pclusterblks > + (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits)) + return (EOPNOTSUPP); + } else { + distance = le16toh(dsb->u1.lz4_max_distance); + if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi)) + return (0); + sbi->available_compr_algs = 1U << Z_EROFS_COMPRESSION_LZ4; + } + return (0); +} + +static int +z_erofs_transform_plain(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *src; + uint8_t *dst; + size_t first, offset, span; + + if (rq->outputsize > rq->inputsize) + return (EINTEGRITY); + src = rq->in; + dst = rq->out; + if (rq->map->m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) { + memmove(dst, src, rq->outputsize); + return (0); + } + span = rq->sbi->block_size - + (rq->map->m_la & (rq->sbi->block_size - 1)); + if (span > rq->inputsize) + return (EINTEGRITY); + first = MIN(span, rq->outputsize); + offset = rq->inputsize - span; + if (offset > rq->inputsize || first > rq->inputsize - offset) + return (EINTEGRITY); + memmove(dst, src + offset, first); + if (first < rq->outputsize) + memmove(dst + first, src, rq->outputsize - first); + return (0); +} + +static const struct z_erofs_decompressor z_erofs_shifted_decomp = { + .decompress = z_erofs_transform_plain, +}; + +static const struct z_erofs_decompressor z_erofs_interlaced_decomp = { + .decompress = z_erofs_transform_plain, +}; + +const struct z_erofs_decompressor z_erofs_lz4_decomp = { + .config = z_erofs_load_lz4_config, + .decompress = z_erofs_lz4_buffer_decompress, +}; + +static const struct z_erofs_decompressor * const z_erofs_decomp[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp, + [Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp, + [Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp, +#if WITH_EROFS_ZIP_LZMA + [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, +#endif +#if WITH_EROFS_ZIP_DEFLATE + [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, +#endif +#if WITH_EROFS_ZIP_ZSTD + [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, +#endif +}; + +static int +z_erofs_read_cfg(struct erofs_sb_info *sbi, uint64_t *offset, + struct erofs_buf *buf, + size_t *sizep) +{ + struct erofs_buf metabuf = EROFS_BUF_INITIALIZER; + uint8_t length_buf[2]; + uint64_t aligned; + uint16_t length; + int error; + + /* + * At most one configuration per algorithm, each at most 64 KiB, follows + * the superblock, so the offset stays far below the overflow limit; the + * reads are range checked. + */ + aligned = roundup2(*offset, 4); + error = erofs_read_metadata_range(sbi, 0, aligned, sizeof(length_buf), + &metabuf); + if (error != 0) + return (error); + memcpy(length_buf, metabuf.data, sizeof(length_buf)); + erofs_put_metabuf(&metabuf); + length = le16dec(length_buf); + *sizep = length != 0 ? length : UINT16_MAX + 1U; + *offset = aligned + sizeof(length_buf); + error = erofs_read_metadata_range(sbi, 0, *offset, *sizep, buf); + if (error == 0) + *offset += *sizep; + return (error); +} + +int +z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf data = EROFS_BUF_INITIALIZER; + const struct z_erofs_decompressor *decompressor; + uint64_t offset; + uint16_t algorithms; + size_t size; + int algorithm, ret; + + if (!erofs_sb_has_compr_cfgs(sbi)) + return (z_erofs_lz4_decomp.config(sbi, dsb, NULL, 0)); + algorithms = le16toh(dsb->u1.available_compr_algs); + sbi->available_compr_algs = algorithms; + if ((algorithms & ~Z_EROFS_ALL_COMPR_ALGS) != 0) + return (EOPNOTSUPP); + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; + ++algorithm) { + if ((algorithms & (1U << algorithm)) == 0) + continue; + ret = z_erofs_read_cfg(sbi, &offset, &data, &size); + if (ret != 0) + return (ret); + decompressor = z_erofs_decomp[algorithm]; + if (decompressor == NULL || decompressor->config == NULL) + ret = EOPNOTSUPP; + else + ret = decompressor->config(sbi, dsb, data.data, size); + erofs_put_metabuf(&data); + if (ret != 0) + return (ret); + } + return (0); +} + +int +z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src0, size_t srclen, + void *dst, size_t dstlen, bool partial) +{ + const struct z_erofs_decompressor *decompressor; + struct z_erofs_decompress_req rq; + const uint8_t *src; + uint8_t algorithm; + size_t padding, padding_limit; + + algorithm = map->m_algorithmformat; + if (algorithm >= nitems(z_erofs_decomp) || + z_erofs_decomp[algorithm] == NULL || + z_erofs_decomp[algorithm]->decompress == NULL) + return (EOPNOTSUPP); + decompressor = z_erofs_decomp[algorithm]; + rq = (struct z_erofs_decompress_req) { + .sbi = sbi, + .map = map, + .in = src0, + .inputsize = srclen, + .out = dst, + .outputsize = dstlen, + .partial_decoding = partial, + }; + if (algorithm == Z_EROFS_COMPRESSION_SHIFTED || + algorithm == Z_EROFS_COMPRESSION_INTERLACED) + return (decompressor->decompress(&rq)); + + src = src0; + if (map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4 || + erofs_sb_has_lz4_0padding(sbi)) { + padding_limit = MIN(srclen, sbi->block_size - + (map->m_pa & (sbi->block_size - 1))); + for (padding = 0; padding < padding_limit && src[padding] == 0; + ++padding) + ; + if (padding == padding_limit) + return (EINTEGRITY); + src += padding; + srclen -= padding; + } + rq.in = src; + rq.inputsize = srclen; + return (decompressor->decompress(&rq)); +} diff --git a/decompressor_deflate.c b/decompressor_deflate.c new file mode 100644 index 0000000..5e49d6c --- /dev/null +++ b/decompressor_deflate.c @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal DEFLATE decompressor for EROFS FreeBSD */ + +#include +#include +#include + +#include + +#include "compress.h" + +struct z_erofs_deflate_ctx { + struct erofs_stream_ctx pool; + z_stream stream; + bool initialized; +}; + +_Static_assert(sizeof(struct z_erofs_deflate_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Deflate stream wrapper exceeds UMA item"); + +static voidpf +z_erofs_deflate_alloc(voidpf opaque, uInt items, uInt size) +{ + struct erofs_stream_ctx *pool; + pool = opaque; + /* Two 32-bit factors cannot overflow a 64-bit size_t. */ + return (z_erofs_stream_ctx_alloc(pool, (size_t)items * size)); +} + +static void +z_erofs_deflate_free(voidpf opaque, voidpf address) +{ + + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_load_deflate_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_deflate_cfgs *deflate; + + if (size < sizeof(*deflate)) + return (EINTEGRITY); + deflate = data; + if (deflate->windowbits > MAX_WBITS) + return (EOPNOTSUPP); + return (0); +} + +static int +z_erofs_deflate_error(int ret) +{ + + switch (ret) { + case Z_MEM_ERROR: + return (ENOMEM); + case Z_VERSION_ERROR: + return (EOPNOTSUPP); + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_BUF_ERROR: + return (EINTEGRITY); + default: + return (EIO); + } +} + +static void +z_erofs_deflate_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + + ctx = (struct z_erofs_deflate_ctx *)pool; + if (ctx->initialized) { + (void)inflateEnd(&ctx->stream); + ctx->initialized = false; + } +} + +static int +z_erofs_deflate_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + int ret; + + ctx = (struct z_erofs_deflate_ctx *)pool; + bzero(&ctx->stream, sizeof(ctx->stream)); + ctx->stream.zalloc = z_erofs_deflate_alloc; + ctx->stream.zfree = z_erofs_deflate_free; + ctx->stream.opaque = pool; + ret = inflateInit2(&ctx->stream, -MAX_WBITS); + if (ret != Z_OK) + return (z_erofs_deflate_error(ret)); + ctx->initialized = true; + return (0); +} + +static int +z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret, + uInt avail_in) +{ + + if (rq->partial_decoding) + return (0); + if (ret != Z_STREAM_END || avail_in != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_deflate_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_deflate_ctx *ctx; + z_stream *strm; + uInt in_before, out_before; + int error, ret; + + /* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */ + EROFS_DEBUG_ASSERT(rq->inputsize <= UINT_MAX && + rq->outputsize <= UINT_MAX, ("erofs: deflate sizes")); + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_DEFLATE, + sizeof(*ctx), z_erofs_deflate_ctx_init, z_erofs_deflate_ctx_fini, + &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_deflate_ctx *)pool; + strm = &ctx->stream; + pool->allocation_failed = false; + ret = inflateReset2(strm, -MAX_WBITS); + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + z_erofs_stream_ctx_put(pool, false); + return (error); + } + strm->next_in = __DECONST(void *, rq->in); + strm->avail_in = rq->inputsize; + strm->next_out = rq->out; + strm->avail_out = rq->outputsize; + + error = 0; + ret = Z_OK; + while (strm->avail_out != 0) { + in_before = strm->avail_in; + out_before = strm->avail_out; + ret = inflate(strm, Z_SYNC_FLUSH); + if (ret == Z_STREAM_END) + break; + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + break; + } + if (strm->avail_in == in_before && + strm->avail_out == out_before) { + error = EINTEGRITY; + break; + } + } + if (error == 0 && strm->avail_out != 0) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_deflate_finish(rq, ret, strm->avail_in); + if (pool->allocation_failed) + error = ENOMEM; + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, +}; diff --git a/decompressor_lz4.c b/decompressor_lz4.c new file mode 100644 index 0000000..06549b2 --- /dev/null +++ b/decompressor_lz4.c @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Contiguous-buffer LZ4 backend for EROFS FreeBSD. */ + +#include +#include +#include + +#include "compress.h" + +#define EROFS_LZ4_TOKEN_LITERAL_SHIFT 4 +#define EROFS_LZ4_TOKEN_MATCH_MASK 0x0f +#define EROFS_LZ4_MAX_RUN 15 +#define EROFS_LZ4_EXT_SENTINEL 255 +#define EROFS_LZ4_MIN_MATCH 4 +#define EROFS_LZ4_OFFSET_BYTES 2 + +static int +lz4_finish(const uint8_t *ip, const uint8_t *iend, int partial) +{ + if (partial) + return (0); + while (ip < iend) { + if (*ip++ != 0) + return (EINTEGRITY); + } + return (0); +} + +int +z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *input; + uint8_t *output; + unsigned int token; + size_t length, copylen; + size_t offset; + size_t input_pos, output_pos; + + if (rq == NULL || (rq->inputsize != 0 && rq->in == NULL) || + (rq->outputsize != 0 && rq->out == NULL)) + return (EINTEGRITY); + if (rq->inputsize == 0) + return (rq->outputsize == 0 ? 0 : EINTEGRITY); + /* + * A length grows by at most 255 per input byte, so it cannot reach + * SIZE_MAX however long the input is. + */ + input = rq->in; + output = rq->out; + input_pos = 0; + output_pos = 0; + + while (input_pos < rq->inputsize) { + token = input[input_pos++]; + length = token >> EROFS_LZ4_TOKEN_LITERAL_SHIFT; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (input_pos >= rq->inputsize) + return (EINTEGRITY); + value = input[input_pos++]; + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > rq->inputsize - input_pos) + return (EINTEGRITY); + if (!rq->partial_decoding && + length > rq->outputsize - output_pos) + return (EINTEGRITY); + copylen = MIN(length, rq->outputsize - output_pos); + memcpy(output + output_pos, input + input_pos, copylen); + input_pos += length; + output_pos += copylen; + if (output_pos == rq->outputsize) + return (lz4_finish(input + input_pos, + input + rq->inputsize, rq->partial_decoding)); + if (input_pos >= rq->inputsize) + break; + if (EROFS_LZ4_OFFSET_BYTES > rq->inputsize - input_pos) + return (EINTEGRITY); + offset = le16dec(input + input_pos); + input_pos += EROFS_LZ4_OFFSET_BYTES; + if (offset == 0 || offset > output_pos) + return (EINTEGRITY); + length = token & EROFS_LZ4_TOKEN_MATCH_MASK; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (input_pos >= rq->inputsize) + return (EINTEGRITY); + value = input[input_pos++]; + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + length += EROFS_LZ4_MIN_MATCH; + if (!rq->partial_decoding && + length > rq->outputsize - output_pos) + return (EINTEGRITY); + copylen = MIN(length, rq->outputsize - output_pos); + while (copylen-- != 0) { + output[output_pos] = output[output_pos - offset]; + ++output_pos; + } + if (output_pos == rq->outputsize) + return (lz4_finish(input + input_pos, + input + rq->inputsize, rq->partial_decoding)); + } + return (output_pos == rq->outputsize ? + lz4_finish(input + input_pos, input + rq->inputsize, + rq->partial_decoding) : EINTEGRITY); +} diff --git a/decompressor_lzma.c b/decompressor_lzma.c new file mode 100644 index 0000000..44eded7 --- /dev/null +++ b/decompressor_lzma.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * EROFS MicroLZMA wrapper around FreeBSD's bundled XZ Embedded decoder. + * The decoder source is compiled with private symbol names because the + * stock xz.ko does not enable its optional MicroLZMA entry points. + */ + +#include +#include +#include + +#include "compress.h" + +#define XZ_DEC_MICROLZMA +#define xz_dec_lzma2_create erofs_xz_dec_lzma2_create +#define xz_dec_lzma2_reset erofs_xz_dec_lzma2_reset +#define xz_dec_lzma2_run erofs_xz_dec_lzma2_run +#define xz_dec_lzma2_end erofs_xz_dec_lzma2_end +#define xz_dec_microlzma_alloc erofs_xz_dec_microlzma_alloc +#define xz_dec_microlzma_reset erofs_xz_dec_microlzma_reset +#define xz_dec_microlzma_run erofs_xz_dec_microlzma_run +#define xz_dec_microlzma_end erofs_xz_dec_microlzma_end +#define xz_malloc erofs_xz_malloc +#define xz_free erofs_xz_free + +static void * +erofs_xz_malloc(unsigned long size) +{ + return (malloc(size, M_EROFS, M_WAITOK)); +} + +static void +erofs_xz_free(void *ptr) +{ + free(ptr, M_EROFS); +} + +#include + + +#undef bool +#undef false +#undef true +#undef min + +struct z_erofs_lzma_ctx { + struct erofs_stream_ctx pool; + struct xz_dec_microlzma *state; + size_t state_bytes; +}; + +_Static_assert(sizeof(struct z_erofs_lzma_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "LZMA stream wrapper exceeds UMA item"); + +static void +z_erofs_lzma_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + + ctx = (struct z_erofs_lzma_ctx *)pool; + if (ctx->state != NULL) { + xz_dec_microlzma_end(ctx->state); + ctx->state = NULL; + } + if (ctx->state_bytes != 0) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + } +} + +static int +z_erofs_lzma_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + int error; + + ctx = (struct z_erofs_lzma_ctx *)pool; + ctx->state_bytes = sizeof(*ctx->state); + error = z_erofs_stream_ctx_charge(pool, ctx->state_bytes); + if (error != 0) { + ctx->state_bytes = 0; + return (error); + } + ctx->state = xz_dec_microlzma_alloc(XZ_SINGLE, + pool->sbi->lzma_dict_size); + if (ctx->state == NULL) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + pool->allocation_failed = 1; + return (ENOMEM); + } + return (0); +} + +static int +z_erofs_load_lzma_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_lzma_cfgs *lzma; + uint32_t dict_size; + + if (size < sizeof(*lzma)) + return (EINTEGRITY); + lzma = data; + if (le16toh(lzma->format) != 0) + return (EOPNOTSUPP); + dict_size = le32toh(lzma->dict_size); + if (dict_size < 4096) + return (EINTEGRITY); + if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) + return (EOPNOTSUPP); + sbi->lzma_dict_size = dict_size; + return (0); +} + +static int +z_erofs_lzma_error(enum xz_ret ret) +{ + + switch (ret) { + case XZ_MEM_ERROR: + return (ENOMEM); + case XZ_MEMLIMIT_ERROR: + case XZ_OPTIONS_ERROR: + case XZ_UNSUPPORTED_CHECK: + return (EOPNOTSUPP); + default: + return (EINTEGRITY); + } +} + +static int +z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret, + size_t input_pos) +{ + + if (rq->partial_decoding && + (ret == XZ_OK || ret == XZ_STREAM_END)) + return (0); + if (!rq->partial_decoding && ret == XZ_STREAM_END && + input_pos == rq->inputsize) + return (0); + return (z_erofs_lzma_error(ret)); +} + +static int +z_erofs_lzma_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_lzma_ctx *ctx; + struct xz_buf buffer; + enum xz_ret ret; + int error; + + /* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */ + EROFS_DEBUG_ASSERT(rq->inputsize <= UINT32_MAX && + rq->outputsize <= UINT32_MAX, ("erofs: lzma sizes")); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_LZMA, + sizeof(*ctx), z_erofs_lzma_ctx_init, z_erofs_lzma_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_lzma_ctx *)pool; + bzero(&buffer, sizeof(buffer)); + buffer.in = rq->in; + buffer.in_size = rq->inputsize; + buffer.out = rq->out; + buffer.out_size = rq->outputsize; + xz_dec_microlzma_reset(ctx->state, (uint32_t)rq->inputsize, + (uint32_t)rq->outputsize, !rq->partial_decoding); + ret = xz_dec_microlzma_run(ctx->state, &buffer); + if (buffer.out_pos != rq->outputsize) + error = EINTEGRITY; + else + error = z_erofs_lzma_finish(rq, ret, buffer.in_pos); + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_lzma_decomp = { + .config = z_erofs_load_lzma_config, + .decompress = z_erofs_lzma_decompress, +}; diff --git a/decompressor_zstd.c b/decompressor_zstd.c new file mode 100644 index 0000000..01ab746 --- /dev/null +++ b/decompressor_zstd.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal zstd decompressor for EROFS FreeBSD */ +#include +#include + +#include "compress.h" + +#ifdef ZSTDIO +#define ZSTD_STATIC_LINKING_ONLY +#include + + +struct z_erofs_zstd_ctx { + struct erofs_stream_ctx pool; + ZSTD_DCtx *dctx; +}; + +_Static_assert(sizeof(struct z_erofs_zstd_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Zstd stream wrapper exceeds UMA item"); +#endif + +static bool +erofs_zstd_available(void) +{ +#ifdef ZSTDIO + return (true); +#else + return (false); +#endif +} + +static int +z_erofs_load_zstd_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_zstd_cfgs *zstd; + + if (!erofs_zstd_available()) { + vfs_mount_error(sbi->mnt, + "erofs: ZSTD compression requires ZSTDIO support"); + return (EOPNOTSUPP); + } + if (data == NULL || size < sizeof(*zstd)) + return (EINTEGRITY); + zstd = data; + if (zstd->format != 0 || zstd->windowlog > 10) + return (EOPNOTSUPP); + sbi->zstd_windowlog = zstd->windowlog; + return (0); +} + +#ifdef ZSTDIO +static void * +zstd_alloc(void *opaque, size_t size) +{ + return (z_erofs_stream_ctx_alloc(opaque, size)); +} + +static void +zstd_free(void *opaque, void *address) +{ + z_erofs_stream_ctx_free(opaque, address); +} + +static void +z_erofs_zstd_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + + ctx = (struct z_erofs_zstd_ctx *)pool; + if (ctx->dctx != NULL) { + (void)ZSTD_freeDCtx(ctx->dctx); + ctx->dctx = NULL; + } +} + +static int +z_erofs_zstd_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + ZSTD_customMem alloc; + size_t ret; + + ctx = (struct z_erofs_zstd_ctx *)pool; + alloc = (ZSTD_customMem) { + .customAlloc = zstd_alloc, + .customFree = zstd_free, + .opaque = pool, + }; + ctx->dctx = ZSTD_createDCtx_advanced(alloc); + if (ctx->dctx == NULL) + return (ENOMEM); + ret = ZSTD_DCtx_setParameter(ctx->dctx, ZSTD_d_windowLogMax, + pool->sbi->zstd_windowlog + 10); + if (ZSTD_isError(ret)) + return (pool->allocation_failed ? ENOMEM : EOPNOTSUPP); + return (0); +} + +static int +z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret, + size_t input_pos, size_t input_size) +{ + + if (rq->partial_decoding) + return (0); + if (ret != 0 || input_pos != input_size) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_zstd_ctx *ctx; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t in_before, out_before, ret; + int error; + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_ZSTD, + sizeof(*ctx), z_erofs_zstd_ctx_init, z_erofs_zstd_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_zstd_ctx *)pool; + pool->allocation_failed = false; + + input = (ZSTD_inBuffer) { + .src = rq->in, + .size = rq->inputsize, + }; + output = (ZSTD_outBuffer) { + .dst = rq->out, + .size = rq->outputsize, + }; + error = 0; + ret = 1; + while (output.pos != output.size) { + in_before = input.pos; + out_before = output.pos; + ret = ZSTD_decompressStream(ctx->dctx, &output, &input); + if (ZSTD_isError(ret)) { + error = pool->allocation_failed ? ENOMEM : EINTEGRITY; + break; + } + if (input.pos == in_before && output.pos == out_before) { + error = EINTEGRITY; + break; + } + if (ret == 0) + break; + } + if (error == 0 && output.pos != output.size) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_zstd_finish(rq, ret, input.pos, input.size); + z_erofs_stream_ctx_put(pool, error == 0 && !rq->partial_decoding); + return (error); +} +#else +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq __unused) +{ + return (EOPNOTSUPP); +} +#endif + +const struct z_erofs_decompressor z_erofs_zstd_decomp = { + .config = z_erofs_load_zstd_config, + .decompress = z_erofs_zstd_decompress, +}; diff --git a/dir.c b/dir.c new file mode 100644 index 0000000..522d373 --- /dev/null +++ b/dir.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static unsigned char +erofs_ftype_to_dtype(uint8_t ftype) +{ + switch (ftype) { + case EROFS_FT_REG_FILE: + return (DT_REG); + case EROFS_FT_DIR: + return (DT_DIR); + case EROFS_FT_CHRDEV: + return (DT_CHR); + case EROFS_FT_BLKDEV: + return (DT_BLK); + case EROFS_FT_FIFO: + return (DT_FIFO); + case EROFS_FT_SOCK: + return (DT_SOCK); + case EROFS_FT_SYMLINK: + return (DT_LNK); + default: + return (DT_UNKNOWN); + } +} + +/* + * Directory block checks. + * + * A block is trusted only as far as it is used: its header is checked once + * per block read, and each entry is bounds-checked when it is looked at. + * Nothing here requires names to be sorted, unique, or ordered across + * blocks. That is fsck.erofs's job, and a directory that violates it can + * only make a lookup miss; the binary search stays within the block either + * way. + */ + +/* + * The first name offset is also the size of the dirent array, so it must + * leave room for at least one dirent and lie inside the block. + */ +int +erofs_dirblock_ndirents(const char *blk, uint32_t maxsize, uint32_t *ndirentsp) +{ + uint32_t first_nameoff; + + if (maxsize < sizeof(struct erofs_dirent)) + return (EINTEGRITY); + first_nameoff = le16toh(((const struct erofs_dirent *)blk)->nameoff); + if (first_nameoff < sizeof(struct erofs_dirent) || + first_nameoff >= maxsize || + (first_nameoff % sizeof(struct erofs_dirent)) != 0) + return (EINTEGRITY); + *ndirentsp = first_nameoff / sizeof(struct erofs_dirent); + return (0); +} + +/* + * Bounds of the name of the idx'th entry: [*nameoffp, *endoffp) is non-empty + * and inside the block. A name ends where the next one starts; the last + * one ends at the block end and may be NUL terminated before that. + */ +int +erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents, + uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp) +{ + const struct erofs_dirent *de; + uint32_t endoff, nameoff; + + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[idx].nameoff); + endoff = idx + 1 < ndirents ? le16toh(de[idx + 1].nameoff) : maxsize; + if (endoff > maxsize || endoff <= nameoff) + return (EINTEGRITY); + *nameoffp = nameoff; + *endoffp = endoff; + return (0); +} + +/* + * The name of an entry that readdir is about to hand to userland must be a + * plain file name that fits d_name: no NUL inside it (so d_namlen matches + * strlen(d_name)), no '/', and at most EROFS_NAME_LEN bytes. Lookup does + * not need any of this. + */ +static int +erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp) +{ + size_t namelen, span; + + span = endoff - nameoff; + if (trailing) { + namelen = strnlen(blk + nameoff, span); + } else { + namelen = span; + if (memchr(blk + nameoff, '\0', span) != NULL) + return (EINTEGRITY); + } + if (namelen == 0 || namelen > EROFS_NAME_LEN) + return (EINTEGRITY); + for (size_t i = 0; i < namelen; i++) { + if (blk[nameoff + i] == '/') + return (EINTEGRITY); + } + *namelenp = namelen; + return (0); +} + +/* Extract name, nid, and type of the idx'th directory entry from a block. */ +static int +erofs_dirent_name(const char *blk, uint32_t maxsize, + uint32_t idx, uint32_t ndirents, char *name, size_t namesz __unused, + erofs_nid_t *nid, uint8_t *ftype, size_t *namelenp) +{ + const struct erofs_dirent *de; + uint32_t nameoff, endoff; + size_t namelen; + int error; + + error = erofs_dirent_span(blk, maxsize, ndirents, idx, &nameoff, + &endoff); + if (error != 0) + return (error); + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + /* erofs_dirent_namelen() caps the name at EROFS_NAME_LEN. */ + EROFS_DEBUG_ASSERT(namelen < namesz, + ("erofs: dirent name of %zu bytes exceeds buffer", namelen)); + de = (const struct erofs_dirent *)blk; + memcpy(name, blk + nameoff, namelen); + name[namelen] = '\0'; + *nid = le64toh(de[idx].nid); + *ftype = de[idx].file_type; + *namelenp = namelen; + return (0); +} + +/* Per-call state for readdir dirent/cookie output. */ +struct erofs_uiodir { + struct dirent *dirent; + uint64_t *cookies; + uint64_t last_cookie; + int ncookies; + int acookies; + int eofflag; +}; + +enum erofs_uiodir_result { + EROFS_UIODIR_BUFFER_FULL = -1, + EROFS_UIODIR_OK = 0, +}; + +/* Push a dirent and its cookie to the caller, modelled after UDF. */ +static int +erofs_uiodir(struct erofs_uiodir *uiodir, int de_size, struct uio *uio, + uint64_t cookie) +{ + int error; + + /* + * Cookies are block offsets we computed, and grow with the entry + * index whatever the block contains. + */ + EROFS_DEBUG_ASSERT(cookie > uiodir->last_cookie, + ("erofs: readdir cookie %ju not above %ju", (uintmax_t)cookie, + (uintmax_t)uiodir->last_cookie)); + if (uio->uio_resid < de_size || + (uiodir->cookies != NULL && + uiodir->acookies >= uiodir->ncookies)) { + return (EROFS_UIODIR_BUFFER_FULL); + } + error = uiomove(uiodir->dirent, de_size, uio); + if (error != 0) + return (error); + uiodir->last_cookie = cookie; + if (uiodir->cookies != NULL) + uiodir->cookies[uiodir->acookies++] = cookie; + return (EROFS_UIODIR_OK); +} + +/* + * Process directory entries within a single block and output them to uio. + * Returns 0 on success (all entries consumed), -1 if uio is full, or a + * positive error code on corruption. + */ +static int +erofs_fill_dentries(struct erofs_sb_info *sbi, erofs_nid_t dirnid, + struct erofs_uiodir *uiodir, struct uio *uio, struct dirent *d, + const char *blk, uint32_t maxsize, + uint32_t start_idx, uint32_t ndirents, erofs_off_t block_off, + uint64_t *logical_offp) +{ + char name[EROFS_NAME_LEN + 1]; + uint32_t idx; + uint64_t curpos, nextoff; + erofs_nid_t nid; + size_t namelen; + uint8_t ftype; + int error; + + for (idx = start_idx; idx < ndirents; idx++) { + curpos = block_off + idx * sizeof(struct erofs_dirent); + nextoff = (idx + 1 < ndirents) ? + (curpos + sizeof(struct erofs_dirent)) : + (block_off + maxsize); + error = erofs_dirent_name(blk, maxsize, idx, ndirents, name, + sizeof(name), &nid, &ftype, &namelen); + if (error != 0) + return (erofs_corrupt(sbi, dirnid, + "invalid name of dirent %u in the block at offset " + "%ju", + idx, (uintmax_t)block_off)); + /* + * The nid is only reported as d_fileno; lookup checks it + * (erofs_read_inode()) before it is used for anything else. + */ + bzero(d, sizeof(*d)); + d->d_fileno = erofs_nid_to_ino64(sbi, nid); + d->d_type = erofs_ftype_to_dtype(ftype); + d->d_namlen = namelen; + d->d_reclen = GENERIC_DIRSIZ(d); + d->d_off = nextoff; + strlcpy(d->d_name, name, sizeof(d->d_name)); + error = erofs_uiodir(uiodir, d->d_reclen, uio, d->d_off); + if (error != 0) + return (error); + *logical_offp = nextoff; + uio->uio_offset = *logical_offp; + } + return (0); +} + +/* + * Read directory contents and output a FreeBSD dirent stream to uio. + * + * Key points: + * - On-disk entries use their logical file offsets as cookies; + * - A dot_omitted directory appends a synthetic "." at i_size, matching + * Linux, so existing on-disk cookies are not shifted; + * - The dirent array occupies only the front portion of a block, so after + * scanning all entries offset must jump to maxsize (the block end), + * otherwise the loop would get stuck on the same block; + * - Supports a_ncookies / a_cookies for NFS and other callers that need + * resumable iteration. + */ +int +erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies) +{ + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct erofs_uiodir uiodir; + struct dirent d; + uint64_t *cookiebuf; + size_t cookie_count; + char *blk; + erofs_off_t block_off; + uint64_t logical_off; + uint32_t block_pos, blksz, ndirents, start_idx, maxsize; + bool sequential; + int error; + + dir = VTOE(vp); + sbi = MTOE(vp->v_mount); + blksz = sbi->block_size; + error = 0; + cookiebuf = NULL; + cookie_count = 0; + uiodir.eofflag = 0; + uiodir.acookies = 0; + uiodir.dirent = &d; + uiodir.cookies = NULL; + uiodir.ncookies = 0; + if (cookies != NULL && ncookies != NULL) { + *cookies = NULL; + *ncookies = 0; + if (uio->uio_resid > 0) { + cookie_count = (size_t)uio->uio_resid / + GENERIC_MINDIRSIZ; + cookie_count = MIN(cookie_count, (size_t)INT_MAX); + } + uiodir.ncookies = (int)cookie_count; + if (cookie_count != 0) + cookiebuf = malloc(sizeof(*cookiebuf) * cookie_count, + M_TEMP, M_WAITOK); + uiodir.cookies = cookiebuf; + } + + if (uio->uio_offset < 0) { + error = EINVAL; + goto out; + } + if (dir->dot_omitted && dir->size == (uint64_t)OFF_MAX) { + error = erofs_corrupt(sbi, dir->nid, + "directory size leaves no room for '.'"); + goto out; + } + + logical_off = uio->uio_offset; + sequential = logical_off == 0; + uiodir.last_cookie = logical_off; + uio->uio_offset = logical_off; + + while (logical_off < dir->size) { + block_off = rounddown2(logical_off, (uint64_t)blksz); + maxsize = MIN((uint64_t)blksz, dir->size - block_off); + block_pos = logical_off - block_off; + if ((block_pos % sizeof(struct erofs_dirent)) != 0) { + block_pos = roundup(block_pos, + sizeof(struct erofs_dirent)); + logical_off = block_off + block_pos; + uio->uio_offset = logical_off; + } + error = erofs_read_data_readahead(sbi, dir, block_off, + maxsize, sequential, (void **)&blk); + if (error != 0) + goto out; + error = erofs_dirblock_ndirents(blk, maxsize, &ndirents); + if (error != 0) { + erofs_brelse(blk); + error = erofs_corrupt(sbi, dir->nid, + "invalid dirent array in the block at offset %ju", + (uintmax_t)block_off); + goto out; + } + start_idx = block_pos / sizeof(struct erofs_dirent); + if (start_idx >= ndirents) { + logical_off = block_off + maxsize; + uio->uio_offset = logical_off; + erofs_brelse(blk); + continue; + } + error = erofs_fill_dentries(sbi, dir->nid, &uiodir, uio, &d, + blk, maxsize, start_idx, ndirents, block_off, &logical_off); + erofs_brelse(blk); + if (error != 0) + goto out; + } + if (dir->dot_omitted && logical_off == dir->size) { + bzero(&d, sizeof(d)); + d.d_fileno = erofs_nid_to_ino64(sbi, dir->nid); + d.d_type = DT_DIR; + d.d_namlen = 1; + d.d_reclen = GENERIC_DIRSIZ(&d); + d.d_off = dir->size + 1; + d.d_name[0] = '.'; + d.d_name[1] = '\0'; + error = erofs_uiodir(&uiodir, d.d_reclen, uio, d.d_off); + if (error != 0) + goto out; + logical_off++; + uio->uio_offset = logical_off; + } + uiodir.eofflag = 1; +out: + if (error == EROFS_UIODIR_BUFFER_FULL) + error = 0; + if (eofflag != NULL && error == 0) + *eofflag = uiodir.eofflag; + if (cookies != NULL && ncookies != NULL) { + if (error != 0) { + if (cookiebuf != NULL) + free(cookiebuf, M_TEMP); + } else { + *ncookies = uiodir.acookies; + *cookies = cookiebuf; + } + } + return (error); +} diff --git a/erofs_compat.h b/erofs_compat.h new file mode 100644 index 0000000..f302244 --- /dev/null +++ b/erofs_compat.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * What erofs_fs.h, which is kept as it is in Linux, takes from Linux headers: + * the integer and little-endian types, a few macros, and the file type + * and magic definitions that Linux keeps elsewhere. + */ +#ifndef _FS_EROFS_COMPAT_H_ +#define _FS_EROFS_COMPAT_H_ + +#include +#include + +typedef uint8_t u8; +typedef uint8_t __u8; +typedef uint32_t __u32; +typedef uint16_t __le16; +typedef uint32_t __le32; +typedef uint64_t __le64; + +#define le16_to_cpu(x) le16toh(x) +#define cpu_to_le64(x) htole64(x) + +#define BIT(n) (1UL << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define round_up(x, y) roundup2((x), (y)) + +/* Fails to compile when the condition is true, as in Linux. */ +#define BUILD_BUG_ON(cond) ((void)sizeof(char[1 - 2 * !!(cond)])) + +/* include/uapi/linux/magic.h */ +#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 + +/* + * File types in directory entries; Linux uses the generic FT_* values from + * include/linux/fs_types.h. + */ +#define EROFS_FT_UNKNOWN 0 +#define EROFS_FT_REG_FILE 1 +#define EROFS_FT_DIR 2 +#define EROFS_FT_CHRDEV 3 +#define EROFS_FT_BLKDEV 4 +#define EROFS_FT_FIFO 5 +#define EROFS_FT_SOCK 6 +#define EROFS_FT_SYMLINK 7 + +#endif diff --git a/erofs_fs.h b/erofs_fs.h new file mode 100644 index 0000000..7871b16 --- /dev/null +++ b/erofs_fs.h @@ -0,0 +1,467 @@ +/* SPDX-License-Identifier: MIT */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +/* to allow for x86 boot sectors and other oddities. */ +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 +#define EROFS_FEATURE_COMPAT_XATTR_FILTER 0x00000004 +#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX 0x00000008 +#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX 0x00000010 +#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS 0x00000020 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 +#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 +#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES 0x00000040 +#define EROFS_FEATURE_INCOMPAT_48BIT 0x00000080 +#define EROFS_FEATURE_INCOMPAT_METABOX 0x00000100 +#define EROFS_ALL_FEATURE_INCOMPAT \ + ((EROFS_FEATURE_INCOMPAT_METABOX << 1) - 1) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +struct erofs_deviceslot { + u8 tag[64]; /* digest(sha256), etc. */ + __le32 blocks_lo; /* total blocks count of this device */ + __le32 uniaddr_lo; /* unified starting block of this device */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 uniaddr_hi; /* unified starting block MSB */ + u8 reserved[52]; +}; +#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) + +/* erofs on-disk super block (currently 144 bytes at maximum) */ +struct erofs_super_block { + __le32 magic; /* file system magic number */ + __le32 checksum; /* crc32c to avoid unexpected on-disk overlap */ + __le32 feature_compat; + __u8 blkszbits; /* filesystem block size in bit shift */ + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */ + union { + __le16 rootnid_2b; /* nid of root directory */ + __le16 blocks_hi; /* (48BIT on) blocks count MSB */ + } __packed rb; + __le64 inos; /* total valid ino # (== f_files - f_favail) */ + __le64 epoch; /* base seconds used for compact inodes */ + __le32 fixed_nsec; /* fixed nanoseconds for compact inodes */ + __le32 blocks_lo; /* blocks count LSB */ + __le32 meta_blkaddr; /* start block address of metadata area */ + __le32 xattr_blkaddr; /* start block address of shared xattr area */ + __u8 uuid[16]; /* 128-bit uuid for volume */ + __u8 volume_name[16]; /* volume name */ + __le32 feature_incompat; + union { + /* bitmap for available compression algorithms */ + __le16 available_compr_algs; + /* customized sliding window size instead of 64k by default */ + __le16 lz4_max_distance; + } __packed u1; + __le16 extra_devices; /* # of devices besides the primary device */ + __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */ + __u8 dirblkbits; /* directory block size in bit shift */ + __u8 xattr_prefix_count; /* # of long xattr name prefixes */ + __le32 xattr_prefix_start; /* start of long xattr prefixes */ + __le64 packed_nid; /* nid of the special packed inode */ + __u8 xattr_filter_reserved; /* reserved for xattr name filter */ + __u8 ishare_xattr_prefix_id; + __u8 reserved[2]; + __le32 build_time; /* seconds added to epoch for mkfs time */ + __le64 rootnid_8b; /* (48BIT on) nid of root directory */ + __le64 reserved2; + __le64 metabox_nid; /* (METABOX on) nid of the metabox inode */ + __le64 reserved3; /* [align to extslot 1] */ +}; + +/* + * EROFS inode datalayout (i_format in on-disk inode): + * 0 - uncompressed flat inode without tail-packing inline data: + * 1 - compressed inode with non-compact indexes: + * 2 - uncompressed flat inode with tail-packing inline data: + * 3 - compressed inode with compact indexes: + * 4 - chunk-based inode with (optional) multi-device support: + * 5~7 - reserved + */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_COMPRESSED_FULL = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_COMPRESSED_COMPACT = 3, + EROFS_INODE_CHUNK_BASED = 4, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool erofs_inode_is_data_compressed(unsigned int datamode) +{ + return datamode == EROFS_INODE_COMPRESSED_COMPACT || + datamode == EROFS_INODE_COMPRESSED_FULL; +} + +/* bit definitions of inode i_format */ +#define EROFS_I_VERSION_MASK 0x01 +#define EROFS_I_DATALAYOUT_MASK 0x07 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_NLINK_1_BIT 4 /* non-directory compact inodes only */ +#define EROFS_I_DOT_OMITTED_BIT 4 /* (directories) omit the `.` dirent */ +#define EROFS_I_ALL ((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1) + +/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS_MASK 0x001F +/* with chunk indexes or just a 4-byte block array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 +#define EROFS_CHUNK_FORMAT_48BIT 0x0040 + +#define EROFS_CHUNK_FORMAT_ALL ((EROFS_CHUNK_FORMAT_48BIT << 1) - 1) + +/* 32-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +/* 64-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_EXTENDED 1 + +struct erofs_inode_chunk_info { + __le16 format; /* chunk blkbits, etc. */ + __le16 reserved; +}; + +union erofs_inode_i_u { + __le32 blocks_lo; /* total blocks count (if compressed inodes) */ + __le32 startblk_lo; /* starting block number (if flat inodes) */ + __le32 rdev; /* device ID (if special inodes) */ + struct erofs_inode_chunk_info c; +}; + +union erofs_inode_i_nb { + __le16 nlink; /* if EROFS_I_NLINK_1_BIT is unset */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 startblk_hi; /* starting block number MSB */ +} __packed; + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le32 i_size; + __le32 i_mtime; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved; +}; + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le64 i_size; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le32 i_uid; + __le32 i_gid; + __le64 i_mtime; + __le32 i_mtime_nsec; + __le32 i_nlink; + __u8 i_reserved2[16]; +}; + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_name_filter; /* bit value 1 indicates not-present */ + __u8 h_shared_count; + __u8 h_reserved2[7]; + __le32 h_shared_xattrs[]; /* shared xattr id array */ +}; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* + * bit 7 of e_name_index is set when it refers to a long xattr name prefix, + * while the remained lower bits represent the index of the prefix. + */ +#define EROFS_XATTR_LONG_PREFIX 0x80 +#define EROFS_XATTR_LONG_PREFIX_MASK 0x7f + +#define EROFS_XATTR_FILTER_BITS 32 +#define EROFS_XATTR_FILTER_DEFAULT UINT32_MAX +#define EROFS_XATTR_FILTER_SEED 0x25BBE08F + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_size; /* size of attribute value */ + /* followed by e_name and e_value */ + char e_name[]; /* attribute name */ +}; + +/* long xattr name prefix */ +struct erofs_xattr_long_prefix { + __u8 base_index; /* short xattr name prefix index */ + char infix[]; /* infix apart from short prefix */ +}; + +static inline unsigned int erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + return sizeof(struct erofs_xattr_ibody_header) + + sizeof(__u32) * (le16_to_cpu(i_xattr_icount) - 1); +} + +#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry)) + +static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) +{ + return EROFS_XATTR_ALIGN(sizeof(struct erofs_xattr_entry) + + e->e_name_len + le16_to_cpu(e->e_value_size)); +} + +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR -1 + +/* 4-byte block address array */ +#define EROFS_BLOCK_MAP_ENTRY_SIZE sizeof(__le32) + +/* 8-byte inode chunk index */ +struct erofs_inode_chunk_index { + __le16 startblk_hi; /* starting block number MSB */ + __le16 device_id; /* back-end storage id (with bits masked) */ + __le32 startblk_lo; /* starting block number of this chunk */ +}; + +#define EROFS_DIRENT_NID_METABOX_BIT 63 +#define EROFS_DIRENT_NID_MASK (BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT) - 1) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; /* node number */ + __le16 nameoff; /* start offset of file name */ + __u8 file_type; /* file type */ + __u8 reserved; /* reserved */ +} __packed; + +/* + * EROFS file types should match generic FT_* types and + * it seems no need to add BUILD_BUG_ONs since potential + * unmatchness will break other fses as well... + */ + +#define EROFS_NAME_LEN 255 + +/* maximum supported encoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) + +/* maximum supported decoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) + +/* available compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_LZMA = 1, + Z_EROFS_COMPRESSION_DEFLATE = 2, + Z_EROFS_COMPRESSION_ZSTD = 3, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS ((1 << Z_EROFS_COMPRESSION_MAX) - 1) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + u8 reserved[10]; +} __packed; + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lzma_cfgs { + __le32 dict_size; + __le16 format; + u8 reserved[8]; +} __packed; + +#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE) + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_deflate_cfgs { + u8 windowbits; /* 8..15 for DEFLATE */ + u8 reserved[5]; +} __packed; + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_zstd_cfgs { + u8 format; + u8 windowlog; /* windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN(10) */ + u8 reserved[4]; +} __packed; + +#define Z_EROFS_ZSTD_MAX_DICT_SIZE Z_EROFS_PCLUSTER_MAX_SIZE + +/* + * Enable COMPACTED_2B for EROFS_INODE_COMPRESSED_COMPACT inodes: + * 4B (disabled) vs 4B+2B+4B (enabled) + */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +/* Enable extent metadata for EROFS_INODE_COMPRESSED_FULL inodes */ +#define Z_EROFS_ADVISE_EXTENTS 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +/* Indicate the record size for each extent if extent metadata is used */ +#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 +#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 + +#define Z_EROFS_FRAGMENT_INODE_BIT 7 +struct z_erofs_map_header { + union { + /* fragment data offset in the packed inode */ + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + /* indicates the encoded size of tailpacking data */ + __le16 h_idata_size; + }; + __le32 h_extents_lo; /* extent count LSB */ + }; + __le16 h_advise; + union { + struct { + /* algorithm type (bit 0-3: HEAD1; bit 4-7: HEAD2) */ + __u8 h_algorithmtype; + /* + * bit 0-3 : logical cluster bits - blkszbits + * bit 4-6 : reserved + * bit 7 : pack the whole file into packed inode + */ + __u8 h_clusterbits; + } __packed; + __le16 h_extents_hi; /* extent count MSB */ + } __packed; +}; + +enum { + Z_EROFS_LCLUSTER_TYPE_PLAIN = 0, + Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1, + Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3, + Z_EROFS_LCLUSTER_TYPE_MAX +}; + +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) + +/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ +#define Z_EROFS_LI_PARTIAL_REF (1 << 15) + +/* Set on 1st non-head lcluster to store compressed block counti (in blocks) */ +#define Z_EROFS_LI_D0_CBLKCNT (1 << 11) + +struct z_erofs_lcluster_index { + __le16 di_advise; + /* where to decompress in the head lcluster */ + __le16 di_clusterofs; + + union { + __le32 blkaddr; /* for the HEAD lclusters */ + /* + * [0] - distance to its HEAD lcluster + * [1] - distance to the next HEAD lcluster + */ + __le16 delta[2]; /* for the NONHEAD lclusters */ + } di_u; +}; + +#define Z_EROFS_MAP_HEADER_END(end) \ + (ALIGN(end, 8) + sizeof(struct z_erofs_map_header)) +#define Z_EROFS_FULL_INDEX_START(end) (Z_EROFS_MAP_HEADER_END(end) + 8) + +#define Z_EROFS_EXTENT_PLEN_PARTIAL BIT(27) +#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 +#define Z_EROFS_EXTENT_PLEN_MASK ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) +struct z_erofs_extent { + __le32 plen; /* encoded length */ + __le32 pstart_lo; /* physical offset */ + __le32 pstart_hi; /* physical offset MSB */ + __le32 lstart_lo; /* logical offset */ + __le32 lstart_hi; /* logical offset MSB (>= 4GiB inodes) */ + __u8 reserved[12]; /* for future use */ +}; + +static inline int z_erofs_extent_recsize(unsigned int advise) +{ + return 4 << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & + Z_EROFS_ADVISE_EXTRECSZ_MASK); +} + +/* check the EROFS on-disk layout strictly at compile time */ +static inline void erofs_check_ondisk_layout_definitions(void) +{ + const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) { + .h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT + }; + + BUILD_BUG_ON(sizeof(struct erofs_super_block) != 144); + BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32); + BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64); + BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12); + BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_lcluster_index) != 8); + BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12); + /* keep in sync between 2 index structures for better extendibility */ + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != + sizeof(struct z_erofs_lcluster_index)); + BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128); + + /* exclude old compiler versions like gcc 7.5.0 */ + BUILD_BUG_ON(__builtin_constant_p(fmh) ? + fmh != cpu_to_le64(1ULL << 63) : 0); +} + +#endif diff --git a/erofs_vnops.c b/erofs_vnops.c new file mode 100644 index 0000000..e7b0552 --- /dev/null +++ b/erofs_vnops.c @@ -0,0 +1,681 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if WITH_EROFS_POSIX_ACL +#include +#endif + +#include +#include + +#include "internal.h" +#include "xattr.h" + +static vop_inactive_t erofs_inactive; +static vop_reclaim_t erofs_reclaim; + +static vop_readdir_t erofs_readdir; +static vop_readlink_t erofs_readlink; + +static vop_open_t erofs_open; +static vop_read_t erofs_read; +static vop_bmap_t erofs_bmap; +static vop_ioctl_t erofs_ioctl; + +static vop_getattr_t erofs_getattr; +static vop_setattr_t erofs_setattr; +static vop_access_t erofs_access; +static vop_pathconf_t erofs_pathconf; + +static vop_getextattr_t erofs_getextattr; +static vop_listextattr_t erofs_listextattr; +static vop_deleteextattr_t erofs_deleteextattr; +static vop_setextattr_t erofs_setextattr; +#if WITH_EROFS_POSIX_ACL +static vop_getacl_t erofs_vop_getacl; +static vop_aclcheck_t erofs_aclcheck; +static vop_setacl_t erofs_setacl; +#endif + +/* vop_fhtovp removed in FreeBSD 15.0 */ +static vop_vptofh_t erofs_vptofh; + +/* + * Access check: data nodes are read-only, but device/FIFO nodes are not + * denied writes. + */ +static int +erofs_access(struct vop_access_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; +#if WITH_EROFS_POSIX_ACL + struct acl *acl; +#endif + accmode_t accmode; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + accmode = ap->a_accmode; + if ((accmode & VMODIFY_PERMS) != 0) { + switch (vp->v_type) { + case VDIR: + case VLNK: + case VREG: + return (EROFS); + default: + break; + } + } + error = vfs_unixify_accmode(&accmode); + if (error != 0) + return (error); +#if WITH_EROFS_POSIX_ACL + if ((vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); + + acl = acl_alloc(M_WAITOK); + error = erofs_get_acl(vp, ACL_TYPE_ACCESS, acl); + if (error == 0) + error = vaccess_acl_posix1e(vp->v_type, vi->uid, vi->gid, acl, + accmode, ap->a_cred); + acl_free(acl); + return (error); +#else + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); +#endif +} + +/* + * Tell the generic pager that EROFS does not provide block-level bmap. + * + * Returning EOPNOTSUPP prevents the pager from assuming a bufobj/strategy is + * available, avoiding "No strategy for buffer" errors. VM will correctly + * fall back to the VOP_READ-based page-in path. The default async VOP wrapper + * must remain in use so synchronous fallback completion still calls iodone. + */ +static int +erofs_bmap(struct vop_bmap_args *ap) +{ + if (ap->a_bop != NULL) + *ap->a_bop = NULL; + if (ap->a_bnp != NULL) + *ap->a_bnp = -1; + if (ap->a_runp != NULL) + *ap->a_runp = 0; + if (ap->a_runb != NULL) + *ap->a_runb = 0; + return (EOPNOTSUPP); +} + +/* No dirty writeback on last ref release, so inactive is a no-op. */ +static int +erofs_inactive(struct vop_inactive_args *ap __unused) +{ + return (0); +} + +static int +erofs_open(struct vop_open_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + if (VN_ISDEV(vp)) + return (EOPNOTSUPP); + if (vp->v_type == VREG) { + error = vnode_create_vobject(vp, vi->size, ap->a_td); + return (error); + } + return (0); +} + +static int +erofs_getattr(struct vop_getattr_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct erofs_sb_info *sbi; + struct vattr *vap; + + vp = ap->a_vp; + vi = VTOE(vp); + sbi = MTOE(vp->v_mount); + vap = ap->a_vap; + VATTR_NULL(vap); + /* The image has no creation time; keep the vop_stat() default. */ + vap->va_birthtime.tv_sec = -1; + vap->va_birthtime.tv_nsec = 0; + vap->va_type = vp->v_type; + vap->va_mode = vi->mode & ALLPERMS; + vap->va_nlink = vi->nlink; + vap->va_uid = vi->uid; + vap->va_gid = vi->gid; + vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; + vap->va_fileid = erofs_nid_to_ino64(sbi, vi->nid); + vap->va_size = vi->size; + vap->va_blocksize = sbi->block_size; + vap->va_atime.tv_sec = vi->mtime; + vap->va_mtime.tv_sec = vi->mtime; + vap->va_ctime.tv_sec = vi->mtime; + vap->va_atime.tv_nsec = vi->mtime_nsec; + vap->va_mtime.tv_nsec = vi->mtime_nsec; + vap->va_ctime.tv_nsec = vi->mtime_nsec; + vap->va_gen = vi->generation; + vap->va_flags = 0; + vap->va_rdev = VN_ISDEV(vp) ? vi->rdev : NODEV; + /* A 48-bit block count of 64 KiB blocks, or about size bytes. */ + vap->va_bytes = vi->data_blocks << sbi->blkszbits; + vap->va_filerev = 0; + return (0); +} + +/* + * Read-only xattr get entry point. + * + * Delegates to erofs_getxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes) + */ +static int +erofs_getextattr(struct vop_getextattr_args *ap __unused) +{ +#if WITH_EROFS_XATTR + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER && + !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + if (ap->a_name == NULL || ap->a_name[0] == '\0') + return (EINVAL); + if (strlen(ap->a_name) > EXTATTR_MAXNAMELEN) + return (EINVAL); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_getxattr(ap->a_vp, ap->a_attrnamespace, ap->a_name, + ap->a_uio, ap->a_size)); +#else + return (EOPNOTSUPP); +#endif +} + +/* + * Read-only xattr list entry point. + * + * Delegates to erofs_listxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes) + */ +static int +erofs_listextattr(struct vop_listextattr_args *ap __unused) +{ +#if WITH_EROFS_XATTR + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER && + !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_listxattr(ap->a_vp, ap->a_attrnamespace, ap->a_uio, + ap->a_size)); +#else + return (EOPNOTSUPP); +#endif +} + +static int +erofs_deleteextattr(struct vop_deleteextattr_args *ap __unused) +{ +#if !WITH_EROFS_XATTR + return (EOPNOTSUPP); +#else + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +#endif +} + +static int +erofs_setextattr(struct vop_setextattr_args *ap __unused) +{ +#if !WITH_EROFS_XATTR + return (EOPNOTSUPP); +#else + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +#endif +} + +/* + * EROFS is read-only; mutations on regular files/dirs/symlinks are denied, size + * changes on special vnodes are treated as no-ops per read-only convention. + */ +static int +erofs_setattr(struct vop_setattr_args *ap) +{ + struct vnode *vp; + struct vattr *vap; + + vp = ap->a_vp; + vap = ap->a_vap; + if (vap->va_mode != (mode_t)VNOVAL || vap->va_uid != (uid_t)VNOVAL || + vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL || + vap->va_atime.tv_nsec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || + vap->va_mtime.tv_nsec != VNOVAL || vap->va_flags != VNOVAL) + return (EROFS); + if (vap->va_size != VNOVAL) { + switch (vp->v_type) { + case VDIR: + return (EISDIR); + case VLNK: + case VREG: + return (EROFS); + case VCHR: + case VBLK: + case VSOCK: + case VFIFO: + case VNON: + case VBAD: + case VMARKER: + return (0); + } + } + return (0); +} + +static int +erofs_read(struct vop_read_args *ap) +{ + switch (ap->a_vp->v_type) { + case VREG: + return (erofs_read_file(ap->a_vp, ap->a_uio)); + case VDIR: + return (EISDIR); + default: + return (EINVAL); + } +} + +static int +erofs_ioctl(struct vop_ioctl_args *ap) +{ + struct vnode *vp; + bool seek_data; + int error; + + switch (ap->a_command) { + case FIOSEEKDATA: + seek_data = true; + break; + case FIOSEEKHOLE: + seek_data = false; + break; + default: + return (ENOTTY); + } + + vp = ap->a_vp; + error = vn_lock(vp, LK_SHARED); + if (error != 0) + return (EBADF); + error = erofs_file_llseek(vp, (off_t *)ap->a_data, seek_data); + VOP_UNLOCK(vp); + return (error); +} + +static int +erofs_readdir(struct vop_readdir_args *ap) +{ + if (ap->a_vp->v_type != VDIR) + return (ENOTDIR); + return (erofs_readdir_block(ap->a_vp, ap->a_uio, ap->a_eofflag, + ap->a_ncookies, ap->a_cookies)); +} + +static int +erofs_readlink(struct vop_readlink_args *ap) +{ + if (ap->a_vp->v_type != VLNK) + return (EINVAL); + return (erofs_readlink_target(ap->a_vp, ap->a_uio)); +} + +static int +erofs_pathconf(struct vop_pathconf_args *ap) +{ + switch (ap->a_name) { + case _PC_NAME_MAX: + *ap->a_retval = EROFS_NAME_LEN; + return (0); + case _PC_PATH_MAX: + *ap->a_retval = PATH_MAX; + return (0); + case _PC_FILESIZEBITS: + *ap->a_retval = 64; + return (0); + case _PC_LINK_MAX: + *ap->a_retval = INT_MAX; + return (0); + case _PC_CHOWN_RESTRICTED: + case _PC_NO_TRUNC: + *ap->a_retval = 1; + return (0); +#if WITH_EROFS_POSIX_ACL + case _PC_ACL_EXTENDED: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? 1 : 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? + ACL_MAX_ENTRIES : 3; + return (0); + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); +#else + case _PC_ACL_EXTENDED: + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = 3; + return (0); +#endif + default: + return (vop_stdpathconf(ap)); + } +} + +#if WITH_EROFS_POSIX_ACL +static int +erofs_vop_getacl(struct vop_getacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + return (erofs_get_acl(ap->a_vp, ap->a_type, ap->a_aclp)); +} + +static int +erofs_aclcheck(struct vop_aclcheck_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + if (ap->a_aclp == NULL) + return (EINVAL); + switch (ap->a_type) { + case ACL_TYPE_ACCESS: + break; + case ACL_TYPE_DEFAULT: + if (ap->a_vp->v_type != VDIR) + return (EINVAL); + break; + default: + return (EINVAL); + } + return (acl_posix1e_check(ap->a_aclp)); +} + +static int +erofs_setacl(struct vop_setacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} +#endif + +static u_int +erofs_vfs_hash(erofs_nid_t nid) +{ + + return (fnv_32_buf(&nid, sizeof(nid), FNV1_32_INIT)); +} + +static int +erofs_vfs_hash_cmp(struct vnode *vp, void *pnid) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + return (vi == NULL || vi->nid != *(erofs_nid_t *)pnid); +} + +static void +erofs_fill_vnode(struct erofs_sb_info *sbi, struct vnode *vp, + const struct erofs_inode *vi) +{ + vp->v_type = vi->vtype; + if (vp->v_type == VFIFO) + vp->v_op = &erofs_fifoops; + else + VN_LOCK_ASHARE(vp); + if (vi->nid == sbi->root_nid) + vp->v_vflag |= VV_ROOT; +} + +int +erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags, + struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct thread *td; + struct vnode *vp; + u_int hash; + bool shared; + int error; + + *vpp = NULL; + td = curthread; + shared = (flags & LK_TYPE_MASK) == LK_SHARED; + hash = erofs_vfs_hash(nid); + error = vfs_hash_get(mp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + sbi = MTOE(mp); + vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO); + error = getnewvnode("erofs", mp, &erofs_vnodeops, &vp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + vp->v_data = vi; + vi->nid = nid; + lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); + error = insmntque(vp, mp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + error = vfs_hash_insert(vp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + error = erofs_read_inode(sbi, nid, vi); + if (error != 0) { + *vpp = NULL; + vgone(vp); + vput(vp); + return (error); + } + erofs_xattr_cache_init(vi); + erofs_fill_vnode(sbi, vp, vi); + vn_set_state(vp, VSTATE_CONSTRUCTED); + if (shared) + VOP_LOCK(vp, LK_DOWNGRADE); + *vpp = vp; + return (0); +} + +int +erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags, + struct vnode **vpp) +{ + return (erofs_vget_raw(mp, *(erofs_nid_t *)arg, flags, vpp)); +} + +int +erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + erofs_nid_t nid; + int error; + + *vpp = NULL; + sbi = MTOE(mp); + error = erofs_ino_to_nid(sbi, ino, &nid); + if (error != 0) + return (error); + if (!erofs_nid_is_valid(sbi, nid)) + return (ESTALE); + return (erofs_vget_raw(mp, nid, flags, vpp)); +} + +static int +erofs_reclaim(struct vop_reclaim_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (vi != NULL) { + vfs_hash_remove(vp); + erofs_xattr_cache_fini(MTOE(vp->v_mount), vi); + free(vi, M_EROFS); + vp->v_data = NULL; + } + return (0); +} + +/* Vnode pointer to persistent EROFS file handle. */ +static int +erofs_vptofh(struct vop_vptofh_args *ap) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + + vi = VTOE(ap->a_vp); + bzero(&efid, sizeof(efid)); + efid.len = sizeof(efid); + efid.nid_hi = vi->nid >> 32; + efid.nid_lo = vi->nid; + efid.gen = vi->generation; + memcpy(ap->a_fhp, &efid, sizeof(efid)); + return (0); +} + +struct vop_vector erofs_vnodeops = { + .vop_default = &default_vnodeops, + .vop_inactive = erofs_inactive, + .vop_reclaim = erofs_reclaim, + + .vop_lookup = vfs_cache_lookup, + .vop_cachedlookup = erofs_lookup, + .vop_readdir = erofs_readdir, + .vop_readlink = erofs_readlink, + + .vop_open = erofs_open, + .vop_read = erofs_read, + .vop_bmap = erofs_bmap, + .vop_ioctl = erofs_ioctl, + .vop_getpages = vnode_pager_local_getpages, + + .vop_getattr = erofs_getattr, + .vop_setattr = erofs_setattr, + .vop_access = erofs_access, + .vop_pathconf = erofs_pathconf, + + .vop_getextattr = erofs_getextattr, + .vop_listextattr = erofs_listextattr, + .vop_deleteextattr = erofs_deleteextattr, + .vop_setextattr = erofs_setextattr, +#if WITH_EROFS_POSIX_ACL + .vop_getacl = erofs_vop_getacl, + .vop_aclcheck = erofs_aclcheck, + .vop_setacl = erofs_setacl, +#endif + + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_vnodeops); + +struct vop_vector erofs_fifoops = { + .vop_default = &fifo_specops, + .vop_access = erofs_access, +#if WITH_EROFS_POSIX_ACL + .vop_aclcheck = erofs_aclcheck, +#endif + .vop_deleteextattr = erofs_deleteextattr, +#if WITH_EROFS_POSIX_ACL + .vop_getacl = erofs_vop_getacl, +#endif + .vop_getextattr = erofs_getextattr, + .vop_getattr = erofs_getattr, + .vop_listextattr = erofs_listextattr, + .vop_pathconf = erofs_pathconf, + .vop_reclaim = erofs_reclaim, +#if WITH_EROFS_POSIX_ACL + .vop_setacl = erofs_setacl, +#endif + .vop_setattr = erofs_setattr, + .vop_setextattr = erofs_setextattr, + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_fifoops); diff --git a/fileio.c b/fileio.c new file mode 100644 index 0000000..999b637 --- /dev/null +++ b/fileio.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2024, Alibaba Cloud + */ + +#include +#include +#include + +#include "internal.h" + +#if WITH_EROFS_BACKED_BY_FILE +static int +erofs_read_file_source_vnrdwr(struct erofs_device_info *dif, erofs_off_t off, + size_t len, void *out, int ioflag) +{ + size_t done, want; + ssize_t resid; + int error; + + /* The credential is taken when the file source is opened. */ + EROFS_DEBUG_ASSERT(dif->source_cred != NULL, + ("erofs: file source without a credential")); + done = 0; + while (done < len) { + want = MIN(len - done, (size_t)MAXPHYS); + /* erofs_bread_device() keeps off + len within INT64_MAX + 1. */ + error = vn_rdwr(UIO_READ, dif->devvp, (char *)out + done, + (int)want, (off_t)(off + done), UIO_SYSSPACE, ioflag, + dif->source_cred, NOCRED, &resid, curthread); + if (error != 0) + return (error); + if (resid != 0) + return (EIO); + done += want; + } + return (0); +} + +static int +erofs_read_file_source_direct(struct erofs_device_info *dif, erofs_off_t off, + size_t len, void *out) +{ + return (erofs_read_file_source_vnrdwr(dif, off, len, out, IO_DIRECT)); +} + +int +erofs_read_file_source(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out) +{ + if (test_opt(&sbi->opt, DIRECT_IO)) + return (erofs_read_file_source_direct(dif, off, len, out)); + return (erofs_read_file_source_vnrdwr(dif, off, len, out, 0)); +} +#endif diff --git a/inode.c b/inode.c new file mode 100644 index 0000000..c19d913 --- /dev/null +++ b/inode.c @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "xattr.h" + +static uint64_t +erofs_addrmask(const struct erofs_sb_info *sbi) +{ + if (erofs_sb_has_48bit(sbi)) + return ((1ULL << 48) - 1); + return (UINT32_MAX); +} + +static dev_t +erofs_decode_dev(uint32_t dev) +{ + unsigned int major, minor; + + major = (dev & 0xfff00) >> 8; + minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); + return (makedev(major, minor)); +} + +static uint32_t +erofs_inode_generation(const struct erofs_sb_info *sbi, erofs_nid_t nid, + const void *inode, size_t inode_size) +{ + uint8_t encoded_nid[sizeof(nid)]; + uint32_t generation; + + le64enc(encoded_nid, nid); + generation = fnv_32_buf(encoded_nid, sizeof(encoded_nid), + sbi->generation_seed); + generation = fnv_32_buf(inode, inode_size, generation); + return (generation != 0 ? generation : 1); +} + +static int +erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds, + uint32_t nanoseconds) +{ + /* Keep va_mtime a valid timespec; mkfs.erofs never writes this. */ + if (nanoseconds >= 1000000000) + return (EINTEGRITY); + vi->mtime = seconds; + vi->mtime_nsec = nanoseconds; + return (0); +} + +static void +erofs_set_data_blocks(const struct erofs_sb_info *sbi, struct erofs_inode *vi, + uint64_t compressed_blocks) +{ + if (erofs_inode_is_data_compressed(vi->datalayout)) + vi->data_blocks = compressed_blocks; + else if (vi->size == 0) + vi->data_blocks = 0; + else + /* size is at most OFF_MAX (checked by the caller). */ + vi->data_blocks = roundup2(vi->size, + (uint64_t)sbi->block_size) >> sbi->blkszbits; +} + +static int +erofs_validate_inline_data(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + uint64_t image_size, inline_end, inline_off, inline_size, tail_start; + + if (vi->datalayout != EROFS_INODE_FLAT_INLINE || vi->size == 0) + return (0); + tail_start = roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size; + inline_size = vi->size - tail_start; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &inline_off) || + __builtin_add_overflow(inline_off, vi->xattr_isize, &inline_off) || + __builtin_add_overflow(inline_off, inline_size, &inline_end)) + return (erofs_corrupt(sbi, vi->nid, + "inline tail offset overflows")); + if ((inline_off & (sbi->block_size - 1)) + inline_size > + sbi->block_size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail of %ju bytes crosses its block", + (uintmax_t)inline_size)); + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || + inline_end > sbi->metabox_en->size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail ends at %ju, beyond the metabox", + (uintmax_t)inline_end)); + return (0); + } + /* The image fits its provider; see erofs_nid_is_valid(). */ + image_size = sbi->blocks << sbi->blkszbits; + if (inline_end > image_size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail ends at %ju, beyond the image", + (uintmax_t)inline_end)); + return (0); +} + +/* + * Convert a logical nid to its inode-table byte offset. Normal NIDs are + * relative to the primary metadata area. For metabox NIDs, bit 63 selects + * the metabox backing inode and the remaining bits are relative to its data. + * EROFS_NULL_ADDR is returned when the address cannot be represented. + */ +static erofs_off_t +erofs_iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t meta_offset; + erofs_nid_t nid_lo; + erofs_off_t result; + bool in_metabox; + + in_metabox = erofs_nid_in_metabox(nid); + if (in_metabox && !erofs_sb_has_metabox(sbi)) + return (EROFS_NULL_ADDR); + nid_lo = nid & EROFS_DIRENT_NID_MASK; + if (nid_lo > (UINT64_MAX >> 5)) + return (EROFS_NULL_ADDR); + result = nid_lo << 5; + if (in_metabox) + return (result); + + meta_offset = (uint64_t)sbi->meta_blkaddr << sbi->blkszbits; + if (result > UINT64_MAX - meta_offset) + return (EROFS_NULL_ADDR); + + return (meta_offset + result); +} + +/* + * Check that a NID can address at least one compact inode slot without + * crossing the declared primary image or metabox backing-file boundary. + */ +bool +erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t image_size, off; + + off = erofs_iloc(sbi, nid); + if (off == EROFS_NULL_ADDR) + return (false); + if (erofs_nid_in_metabox(nid)) { + if (sbi->metabox_en == NULL || off > sbi->metabox_en->size) + return (false); + return (sizeof(struct erofs_inode_compact) <= + sbi->metabox_en->size - off); + } + /* + * The image fits its provider (erofs_validate_device_size()), and a + * block count has at most 48 bits, so neither the shift nor the + * provider size needs a check here. + */ + image_size = sbi->blocks << sbi->blkszbits; + return (off <= image_size && + sizeof(struct erofs_inode_compact) <= image_size - off); +} + +int +erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi) +{ + struct erofs_inode_compact *dic; + struct erofs_inode_extended *die; + struct erofs_inode_chunk_info chunk_info; + union erofs_inode_i_nb inode_nb; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t addrmask; + int64_t mtime; + erofs_off_t off; + erofs_blk_t startblk; + uint64_t compressed_blocks; + uint32_t raw_rdev, startblk_lo; + uint16_t ifmt, startblk_hi; + int err; + + if (!erofs_nid_is_valid(sbi, nid)) + return (erofs_corrupt(sbi, nid, "invalid nid")); + off = erofs_iloc(sbi, nid); + EROFS_TRACE_FILL_INODE(nid, off >> sbi->blkszbits, + off & (sbi->block_size - 1)); + err = erofs_read_metadata_range(sbi, nid, off, + sizeof(struct erofs_inode_compact), &buf); + if (err != 0) + return (err); + + bzero(&vi->size, sizeof(*vi) - offsetof(struct erofs_inode, size)); + vi->nid = nid; + vi->inode_off = off; + ifmt = le16dec(buf.data); + if ((ifmt & ~EROFS_I_ALL) != 0) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported inode format %#x @ nid %ju", ifmt, + (uintmax_t)nid); + return (EOPNOTSUPP); + } + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported data layout %u @ nid %ju", + vi->datalayout, (uintmax_t)nid); + return (EOPNOTSUPP); + } + vi->compact_inode = (erofs_inode_version(ifmt) == 0); + if (!vi->compact_inode) { + erofs_put_metabuf(&buf); + err = erofs_read_metadata_range(sbi, nid, off, + sizeof(struct erofs_inode_extended), &buf); + if (err != 0) + return (err); + } + addrmask = erofs_addrmask(sbi); + startblk = EROFS_NULL_ADDR; + startblk_lo = 0; + startblk_hi = 0; + compressed_blocks = 0; + raw_rdev = 0; + bzero(&inode_nb, sizeof(inode_nb)); + dic = buf.data; + if (vi->compact_inode) { + vi->inode_isize = sizeof(struct erofs_inode_compact); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(dic->i_mode); + vi->size = le32toh(dic->i_size); + vi->uid = le16toh(dic->i_uid); + vi->gid = le16toh(dic->i_gid); + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + if (__builtin_add_overflow(sbi->epoch, + (int64_t)le32toh(dic->i_mtime), &mtime)) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "mtime overflows the superblock epoch")); + } + err = erofs_set_timestamp(vi, mtime, sbi->fixed_nsec); + if (err != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "invalid mtime nsec %u", sbi->fixed_nsec)); + } + startblk_lo = le32toh(dic->i_u.startblk_lo); + compressed_blocks = le32toh(dic->i_u.blocks_lo); + raw_rdev = le32toh(dic->i_u.rdev); + if (!S_ISDIR(vi->mode) && + ((ifmt >> EROFS_I_NLINK_1_BIT) & 0x1) != 0) { + vi->nlink = 1; + inode_nb = dic->i_nb; + } else { + vi->nlink = le16toh(dic->i_nb.nlink); + addrmask = UINT32_MAX; + } + } else { + die = buf.data; + vi->inode_isize = sizeof(struct erofs_inode_extended); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(die->i_mode); + vi->size = le64toh(die->i_size); + vi->uid = le32toh(die->i_uid); + vi->gid = le32toh(die->i_gid); + vi->nlink = le32toh(die->i_nlink); + inode_nb = die->i_nb; + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + err = erofs_set_timestamp(vi, + (int64_t)le64toh(die->i_mtime), + le32toh(die->i_mtime_nsec)); + if (err != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "invalid mtime nsec %u", + le32toh(die->i_mtime_nsec))); + } + startblk_lo = le32toh(die->i_u.startblk_lo); + compressed_blocks = le32toh(die->i_u.blocks_lo); + raw_rdev = le32toh(die->i_u.rdev); + } + startblk_hi = le16toh(inode_nb.startblk_hi); + compressed_blocks |= (uint64_t)le16toh(inode_nb.blocks_hi) << 32; + if (vi->size > (uint64_t)OFF_MAX) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "size %ju is too large", + (uintmax_t)vi->size)); + } + + vi->vtype = IFTOVT(vi->mode); + /* A mode with an unknown file type cannot be given a vnode type. */ + if (vi->mode != 0 && vi->vtype == VNON) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "invalid mode %#o", vi->mode)); + } + if (vi->vtype == VLNK) { + if (vi->size == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "empty symlink")); + } + } + vi->dot_omitted = (vi->vtype == VDIR) && + (((ifmt >> EROFS_I_DOT_OMITTED_BIT) & 0x1) != 0); + + if (erofs_inode_is_data_compressed(vi->datalayout)) { +#if WITH_EROFS_ZIP + err = z_erofs_fill_inode(sbi, vi); +#else + err = EOPNOTSUPP; +#endif + if (err != 0) { + erofs_put_metabuf(&buf); + return (err); + } + } else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + if (vi->compact_inode) + chunk_info = dic->i_u.c; + else + chunk_info = die->i_u.c; + if (le16toh(chunk_info.reserved) != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "reserved chunk field %#x is set", + le16toh(chunk_info.reserved))); + } + vi->chunkformat = le16toh(chunk_info.format); + if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported chunk format %#x @ nid %ju", + vi->chunkformat, (uintmax_t)nid); + return (EOPNOTSUPP); + } + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 && + (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "chunk format %#x: 48-bit without chunk indexes", + vi->chunkformat)); + } + vi->chunkbits = sbi->blkszbits + + (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); + } else if (vi->datalayout != EROFS_INODE_FLAT_PLAIN && + vi->datalayout != EROFS_INODE_FLAT_INLINE) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported data layout %u @ nid %ju", + vi->datalayout, (uintmax_t)nid); + return (EOPNOTSUPP); + } + + switch (vi->vtype) { + case VREG: + case VDIR: + case VLNK: + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + } + startblk = startblk_lo | ((uint64_t)startblk_hi << 32); + if (vi->datalayout == EROFS_INODE_FLAT_PLAIN && + ((startblk ^ EROFS_NULL_ADDR) & addrmask) == 0) + startblk = EROFS_NULL_ADDR; + vi->startblk = startblk; + vi->rdev = NODEV; + break; + case VCHR: + case VBLK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = erofs_decode_dev(raw_rdev); + break; + case VFIFO: + case VSOCK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + default: + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "unknown file type, mode %#o", + vi->mode)); + } + erofs_set_data_blocks(sbi, vi, compressed_blocks); + err = erofs_validate_inline_data(sbi, vi); + if (err != 0) { + erofs_put_metabuf(&buf); + return (err); + } + + erofs_put_metabuf(&buf); + return (0); +} diff --git a/internal.h b/internal.h new file mode 100644 index 0000000..aef896f --- /dev/null +++ b/internal.h @@ -0,0 +1,721 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#ifndef _FS_EROFS_INTERNAL_H_ +#define _FS_EROFS_INTERNAL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "erofs_compat.h" +#include "erofs_fs.h" + +/* + * Linux checks the sizes of the on-disk structures in + * erofs_check_ondisk_layout_definitions(); these also check the offsets. + */ +_Static_assert(sizeof(struct erofs_deviceslot) == 128, + "EROFS device slot ABI size"); +_Static_assert(sizeof(struct erofs_super_block) == 144, + "EROFS super block ABI size"); +_Static_assert(sizeof(struct erofs_inode_compact) == 32, + "EROFS compact inode ABI size"); +_Static_assert(sizeof(struct erofs_inode_extended) == 64, + "EROFS extended inode ABI size"); +_Static_assert(sizeof(struct erofs_xattr_ibody_header) == 12, + "EROFS xattr ibody header ABI size"); +_Static_assert(sizeof(struct erofs_xattr_entry) == 4, + "EROFS xattr entry ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_info) == 4, + "EROFS chunk info ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_index) == 8, + "EROFS chunk index ABI size"); +_Static_assert(sizeof(struct z_erofs_map_header) == 8, + "EROFS zmap header ABI size"); +_Static_assert(sizeof(struct z_erofs_lcluster_index) == 8, + "EROFS lcluster index ABI size"); +_Static_assert(sizeof(struct z_erofs_extent) == 32, + "EROFS compression extent ABI size"); +_Static_assert(sizeof(struct erofs_dirent) == 12, + "EROFS dirent ABI size"); +_Static_assert(sizeof(struct erofs_deviceslot) == EROFS_DEVT_SLOT_SIZE, + "EROFS device slot ABI macro"); +_Static_assert( + __builtin_offsetof(struct erofs_super_block, extra_devices) == 86, + "EROFS extra device count ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, devt_slotoff) == 88, + "EROFS device table slot offset ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, rootnid_8b) == 112, + "EROFS 48-bit root nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, metabox_nid) == 128, + "EROFS metabox nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_compact, i_u) == 16, + "EROFS compact inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_u) == 16, + "EROFS extended inode union ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_extended, i_reserved2) == 48, + "EROFS extended inode reserved ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_chunk_index, device_id) == 2, + "EROFS chunk device id ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_chunk_index, startblk_lo) == 4, + "EROFS chunk start block ABI offset"); +_Static_assert(__builtin_offsetof(struct z_erofs_map_header, h_advise) == 4, + "EROFS zmap advise ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_lo) == 64, + "EROFS device blocks ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_lo) == 68, + "EROFS device unified address ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_hi) == 72, + "EROFS device blocks high ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_hi) == 74, + "EROFS device unified address high ABI offset"); + +MALLOC_DECLARE(M_EROFS); + +#ifndef WITH_EROFS_DEBUG +#define WITH_EROFS_DEBUG 1 +#endif + +#if WITH_EROFS_DEBUG +#define EROFS_DEBUG_ASSERT(exp, msg) KASSERT((exp), msg) +#else +#define EROFS_DEBUG_ASSERT(exp, msg) do { } while (0) +#endif + +/* + * Diagnostics. erofs_corrupt() goes where bad on-disk data is first + * noticed, not in the callers that pass the error up. It logs one + * rate-limited line (mount point, function, what was wrong, inode), fires + * the erofs:::corrupt DTrace probe (arguments: nid, source line, error) + * and returns EINTEGRITY, so a check reads + * + * return (erofs_corrupt(sbi, nid, "bad foo %u", foo)); + * + * erofs_err() is the same without the probe, for errors that are not + * corruption, such as an unsupported feature. + */ +struct erofs_sb_info; +/* The error is not about an inode. */ +#define EROFS_NID_NONE ((erofs_nid_t)-1) +void erofs_log_err(const struct erofs_sb_info *sbi, const char *func, + const char *fmt, ...) __printflike(3, 4); +int erofs_log_corrupt(const struct erofs_sb_info *sbi, uint64_t nid, + const char *func, int line, const char *fmt, ...) __printflike(5, 6); +#define erofs_err(sbi, fmt, ...) \ + erofs_log_err((sbi), __func__, fmt, ##__VA_ARGS__) +#define erofs_corrupt(sbi, nid, fmt, ...) \ + erofs_log_corrupt((sbi), (nid), __func__, __LINE__, fmt, ##__VA_ARGS__) + +struct cdev; +struct g_consumer; +struct erofs_device_info; + +typedef uint64_t erofs_nid_t; +typedef uint64_t erofs_off_t; +typedef uint64_t erofs_blk_t; + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) +#error EROFS requires compiler byte-order definitions +#elif __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error EROFS requires a little-endian FreeBSD kernel ABI +#endif +/* + * The module is LP64-only (see also the architecture list in the Makefile), + * so size_t, off_t, ino_t, time_t and erofs_blk_t are all 64 bits wide. + * Code below relies on this: do not add 32-bit overflow guards. + */ +_Static_assert(sizeof(uintptr_t) == sizeof(uint64_t), + "EROFS requires a 64-bit pointer ABI"); +_Static_assert(sizeof(ino_t) >= sizeof(uint64_t), + "EROFS requires a 64-bit inode number ABI"); +_Static_assert(sizeof(off_t) >= sizeof(uint64_t), + "EROFS requires a 64-bit file offset ABI"); +_Static_assert(sizeof(time_t) >= sizeof(int64_t), + "EROFS requires a 64-bit timestamp ABI"); +_Static_assert(sizeof(u_long) == sizeof(uint64_t), + "EROFS requires 64-bit FreeBSD atomic long operations"); +/* + * The block count of the image and of each device has at most 48 bits, so + * (blocks << blkszbits) fits in 64 bits. This does not hold for + * flatdev_blocks, a sum of two such counts: erofs_bread_device() checks it. + */ +_Static_assert(PAGE_SHIFT <= 16, "EROFS block sizes are at most 64 KiB"); + +#define EROFS_MOUNT_POSIX_ACL 0x00000020 +#define EROFS_MOUNT_XATTR_USER 0x00000080 +#define EROFS_MOUNT_DIRECT_IO 0x00000100 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +struct erofs_sb_info_opts { + unsigned int mount_opt; + uint8_t cache_strategy; + uint8_t worker_queue_priority; +}; + +enum erofs_sync_decompress { + EROFS_SYNC_DECOMPRESS_AUTO, + EROFS_SYNC_DECOMPRESS_FORCE_ON, + EROFS_SYNC_DECOMPRESS_FORCE_OFF, +}; + +enum erofs_zip_cache_strategy { + EROFS_ZIP_CACHE_DISABLED, + EROFS_ZIP_CACHE_READAHEAD, + EROFS_ZIP_CACHE_READAROUND, +}; + +struct erofs_device_handle; + +enum erofs_source_kind { + EROFS_SOURCE_NONE, + EROFS_SOURCE_BLOCK, + EROFS_SOURCE_FILE, +}; + +struct erofs_device_info { + struct vnode *devvp; + struct cdev *dev; + struct g_consumer *cp; + struct erofs_device_handle *handle; + struct ucred *source_cred; + enum erofs_source_kind source_kind; + uint64_t mediasize; + uint32_t sectorsize; + uint64_t fsoff; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +static inline bool +erofs_source_is_open(const struct erofs_device_info *dif) +{ + return (dif != NULL && dif->devvp != NULL && dif->handle != NULL && + (dif->source_kind == EROFS_SOURCE_BLOCK || + dif->source_kind == EROFS_SOURCE_FILE)); +} + +static inline bool +erofs_source_is_file(const struct erofs_device_info *dif) +{ + return (erofs_source_is_open(dif) && + dif->source_kind == EROFS_SOURCE_FILE); +} + +struct erofs_xattr_prefix_item { + uint8_t base_index; + uint8_t infix_len; + char *infix; +}; + +struct erofs_map_blocks { + erofs_off_t m_pa, m_la; + uint64_t m_plen, m_llen; + + unsigned short m_deviceid; + uint8_t m_algorithmformat; + unsigned int m_flags; +}; + +enum erofs_zextent_cache_state { + EROFS_ZCACHE_EMPTY, + EROFS_ZCACHE_INFLIGHT, + EROFS_ZCACHE_READY, + EROFS_ZCACHE_FAILED, +}; + +#define EROFS_ZCACHE_MAX_ENTRIES 8 + +struct erofs_zextent_key { + struct erofs_map_blocks map; + erofs_nid_t nid; + size_t decoded_size; +}; + +struct erofs_zextent_cache_metrics { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + uint64_t waiter_merges; + size_t resident_bytes; +}; + +struct erofs_cache_stats { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + uint64_t waiter_merges; + uint64_t entries; + uint64_t bytes; +}; + +struct erofs_zextent_cache_entry { + struct erofs_zextent_key key; + void *data; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_zextent_cache_state state; + struct cv cv; + uint64_t lru; + bool active; +}; + +struct erofs_zextent_cache { + struct erofs_zextent_cache_entry entries[EROFS_ZCACHE_MAX_ENTRIES]; + size_t charged_bytes; + size_t budget_bytes; + uint64_t minimum_decode_work; + struct erofs_zextent_cache_metrics metrics[Z_EROFS_COMPRESSION_MAX]; + uint64_t lru_clock; + unsigned int entry_count; + bool closing; +}; + +struct erofs_decompress_task { + struct task task; + struct mtx lock; + struct cv cv; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks map; + size_t decoded_len; + bool partial; + void *decoded; + int error; + bool done; +}; + +enum erofs_xattr_cache_state { + EROFS_XATTR_CACHE_EMPTY, + EROFS_XATTR_CACHE_INFLIGHT, + EROFS_XATTR_CACHE_READY, + EROFS_XATTR_CACHE_FAILED, +}; + +struct erofs_xattr_cache { + struct mtx lock; + struct cv cv; + void *data; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_xattr_cache_state state; + bool closing; + bool initialized; +}; + +#define EROFS_STREAM_CTX_WRAPPER_SIZE 256 + +struct erofs_stream_ctx; +typedef int erofs_stream_ctx_init_t(struct erofs_stream_ctx *ctx); +typedef void erofs_stream_ctx_fini_t(struct erofs_stream_ctx *ctx); + +struct erofs_stream_ctx { + STAILQ_ENTRY(erofs_stream_ctx) link; + struct erofs_sb_info *sbi; + erofs_stream_ctx_fini_t *fini; + size_t charged_bytes; + uint8_t algorithm; + bool cached; + bool allocation_failed; +}; + +STAILQ_HEAD(erofs_stream_ctx_head, erofs_stream_ctx); + +struct erofs_stream_pool_codec { + struct erofs_stream_ctx_head idle; + unsigned int contexts; + unsigned int cached; + unsigned int borrowed; + unsigned int idle_count; +}; + +struct erofs_stream_pool { + struct erofs_stream_pool_codec codec[Z_EROFS_COMPRESSION_MAX]; + size_t resident_bytes; + unsigned int waiters; + struct cv cv; + bool closing; +}; + +struct erofs_sb_info { + struct mount *mnt; + struct erofs_device_info dif0; + struct mtx control_lock; + struct sysctl_ctx_list sysctl_ctx; + uint64_t control_id; + uint64_t dir_ra_bytes; + uint64_t dir_ra_max_bytes; + uint64_t dir_ra_requests; + uint64_t dir_ra_blocks; + bool control_initialized; + bool sysctl_registered; + bool control_closing; + + uint32_t block_size; + uint32_t sb_size; + uint8_t blkszbits; + uint32_t meta_blkaddr; + uint32_t xattr_blkaddr; + uint32_t xattr_prefix_start; + uint8_t xattr_prefix_count; + uint8_t xattr_filter_reserved; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + struct erofs_inode *metabox_en; + struct erofs_inode *packed_inode; + struct erofs_xattr_prefix_item *xattr_prefixes; + erofs_blk_t blocks; + uint64_t inos; + erofs_nid_t root_nid; + int64_t epoch; + uint32_t fixed_nsec; + uint32_t generation_seed; + uint32_t feature_compat; + uint32_t feature_incompat; + + struct erofs_sb_info_opts opt; + uint16_t available_compr_algs; + uint32_t lzma_dict_size; + uint8_t zstd_windowlog; + unsigned int sync_decompress; + + /* Device table */ + uint16_t extra_devices; + uint16_t device_id_mask; + bool flatdev; + erofs_blk_t total_blocks; + erofs_blk_t flatdev_blocks; + struct erofs_device_info *devs; + struct mtx z_extent_cache_lock; + struct erofs_zextent_cache z_extent_cache; + LIST_ENTRY(erofs_sb_info) z_extent_cache_link; + bool z_extent_cache_initialized; + struct taskqueue *decompress_taskqueue; + struct cv decompress_taskqueue_cv; + unsigned int decompress_taskqueue_users; + unsigned int decompress_worker_count; + bool decompress_worker_cpu_bound; + uint8_t decompress_queue_priority; + uint64_t decompress_submissions; + uint64_t decompress_worker_runs; + uint64_t decompress_queue_fallbacks; + struct erofs_stream_pool stream_pool; + LIST_ENTRY(erofs_sb_info) stream_pool_link; + bool stream_pool_initialized; + volatile u_long xattr_cache_resident; +}; + +#define MTOE(mp) ((struct erofs_sb_info *)(mp)->mnt_data) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ + static inline bool \ + erofs_sb_has_##name(const struct erofs_sb_info *sbi) \ + { \ + return ( \ + (sbi->feature_##compat & EROFS_FEATURE_##feature) != 0); \ + } + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) + +static inline bool +erofs_sb_has_xattr_filter_v1(const struct erofs_sb_info *sbi) +{ + return (erofs_sb_has_xattr_filter(sbi) && + sbi->xattr_filter_reserved == 0); +} + +struct erofs_inode { + erofs_nid_t nid; + uint64_t size; + uint64_t data_blocks; + /* Byte offset in the device, or in the metabox for a metabox nid. */ + erofs_off_t inode_off; + erofs_blk_t startblk; + uint32_t generation; + uint32_t nlink; + uid_t uid; + gid_t gid; + mode_t mode; + __enum_uint8(vtype) vtype; + dev_t rdev; + time_t mtime; + uint32_t mtime_nsec; + uint8_t datalayout; + uint8_t inode_isize; + uint32_t xattr_isize; + bool compact_inode; + bool dot_omitted; + /* Compression fields */ + uint16_t z_advise; + uint8_t z_algorithmtype[2]; + uint8_t z_lclusterbits; + uint16_t z_idata_size; + erofs_off_t z_fragmentoff; + uint64_t z_tailextent_headlcn; + uint64_t z_extents; + bool z_initialized; + /* Chunk-based fields */ + uint16_t chunkformat; + uint8_t chunkbits; + /* Fragment fields */ + struct erofs_xattr_cache xattr_cache; +}; + +struct erofs_fid { + uint16_t len; + uint16_t pad; + uint32_t nid_hi; + uint32_t nid_lo; + uint32_t gen; +}; + +_Static_assert(sizeof(struct erofs_fid) == 16, + "EROFS file handle ABI must be 16 bytes"); +_Static_assert(sizeof(struct erofs_fid) <= sizeof(struct fid), + "struct erofs_fid must fit within struct fid"); + +#define VTOE(vp) ((struct erofs_inode *)(vp)->v_data) + +static inline unsigned int +erofs_inode_version(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK); +} + +static inline unsigned int +erofs_inode_datalayout(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK); +} + +static inline bool +erofs_nid_in_metabox(erofs_nid_t nid) +{ + return ((nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) != 0); +} + +static inline uint64_t +erofs_nid_to_ino64(const struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + if (!erofs_sb_has_metabox(sbi)) + return (nid); + return ((nid << 1) & 0xffffffff00000000ULL) | + (nid & 0x000000007fffffffULL) | + ((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31); +} + +static inline int +erofs_ino_to_nid(const struct erofs_sb_info *sbi, ino_t ino, + erofs_nid_t *nidp) +{ + uint64_t ino64; + erofs_nid_t nid; + + ino64 = (uint64_t)ino; + if (!erofs_sb_has_metabox(sbi)) + nid = ino64; + else + nid = ((ino64 >> 1) & UINT64_C(0x7fffffff80000000)) | + (ino64 & UINT64_C(0x000000007fffffff)) | + ((ino64 & UINT64_C(0x0000000080000000)) << 32); + if (erofs_nid_to_ino64(sbi, nid) != ino64) + return (EOVERFLOW); + *nidp = nid; + return (0); +} + +/* Allocated on disk at m_pa. */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata. */ +#define EROFS_MAP_META 0x0002 +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +#define EROFS_MAP_PARTIAL_REF 0x0008 +#define EROFS_MAP_FRAGMENT 0x0010 +#define EROFS_MAP_FULL(f) \ + (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) + +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +_Static_assert(Z_EROFS_COMPRESSION_RUNTIME_MAX <= UINT8_MAX, + "algorithm format must fit in uint8_t"); + +struct erofs_map_dev { + struct erofs_device_info *m_dif; + + erofs_off_t m_pa; + unsigned int m_deviceid; + uint64_t m_plen; +}; + +struct erofs_buf { + void *data; + void (*release)(void *); +}; + +#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } + +/* Buffer and device I/O. */ +void erofs_put_metabuf(struct erofs_buf *buf); +int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, + void **bufp); +int erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp); +void erofs_brelse(void *buf); +int erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf); +/* Logical mapping and file data. */ +int erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int erofs_read_data_readahead(struct erofs_sb_info *sbi, + struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential, + void **bufp); +uint64_t erofs_dir_ra_bytes(struct erofs_sb_info *sbi); +int erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value); +void erofs_record_dir_readahead(struct erofs_sb_info *sbi, + unsigned int blocks); +int erofs_read_file(struct vnode *vp, struct uio *uio); +int erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data); +int erofs_readlink_target(struct vnode *vp, struct uio *uio); + +/* File-backed source I/O is owned by fileio.c. */ +#if WITH_EROFS_BACKED_BY_FILE +int erofs_read_file_source(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out); +#else +static inline int +erofs_read_file_source(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_off_t off, size_t len, void *out) +{ + (void)sbi; + (void)dif; + (void)off; + (void)len; + (void)out; + return (EOPNOTSUPP); +} +#endif + +/* Inode and vnode lifecycle. */ +bool erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid); +int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi); +int erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags, + struct vnode **vpp); +int erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags, + struct vnode **vpp); +int erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp); + +/* Directory operations. */ +int erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies); +int erofs_dirblock_ndirents(const char *blk, uint32_t maxsize, + uint32_t *ndirentsp); +int erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents, + uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp); +int erofs_lookup(struct vop_cachedlookup_args *ap); + +/* Compressed mapping and data. */ +int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi); +int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +void z_erofs_extent_cache_init(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_fini(struct erofs_sb_info *sbi); +int z_erofs_extent_cache_drop(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_stats(struct erofs_sb_info *sbi, + struct erofs_cache_stats *stats); +void z_erofs_stream_pool_init(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_fini(struct erofs_sb_info *sbi); +int z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp); +void z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable); +int z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes); +void *z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address); +int z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio); +int z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src, size_t srclen, + void *dst, size_t dstlen, bool partial); +int z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, + void **bufp); +int z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len, + bool partial, void **bufp); + +/* Compression configuration. */ +int z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb); + +#if WITH_EROFS_ZIP +int erofs_parse_zip_options(struct mount *mp, + struct erofs_sb_info_opts *opts, unsigned int *syncp, + bool *cache_presentp, bool *sync_presentp); +#endif + +int erofs_parse_dir_ra_option(struct erofs_sb_info *sbi, + struct vfsoptlist *opts); +int erofs_parse_dir_ra_value(struct erofs_sb_info *sbi, + struct vfsoptlist *opts, uint64_t *valuep, bool *presentp); +void erofs_management_init(struct erofs_sb_info *sbi); +void erofs_management_fini(struct erofs_sb_info *sbi); +int erofs_register_sysfs(struct erofs_sb_info *sbi); +int erofs_unregister_sysfs(struct erofs_sb_info *sbi); + +/* VOP vectors. */ +extern struct vop_vector erofs_vnodeops; +extern struct vop_vector erofs_fifoops; + +#endif /* _FS_EROFS_INTERNAL_H_ */ diff --git a/namei.c b/namei.c new file mode 100644 index 0000000..ac08e2b --- /dev/null +++ b/namei.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "xattr.h" + +struct erofs_qstr { + const unsigned char *name; + const unsigned char *end; +}; + +/* + * Compare two directory entry names using an already-matched prefix. + * qn: search key (not necessarily null-terminated). + * qd: on-disk name range (may not be null-terminated). + * matched: in/out count of prefix characters already known to match. + * + * Returns 0 if equal, 1 if qn > qd, -1 if qn < qd. + */ +static int +erofs_dirnamecmp(const struct erofs_qstr *qn, const struct erofs_qstr *qd, + unsigned int *matched) +{ + size_t dname_span, qname_span; + unsigned int i; + + qname_span = qn->end - qn->name; + dname_span = qd->end - qd->name; + i = MIN(*matched, qname_span); + i = MIN(i, dname_span); + while (i < qname_span && i < dname_span && qd->name[i] != '\0') { + if (qn->name[i] != qd->name[i]) { + *matched = i; + return (qn->name[i] > qd->name[i] ? 1 : -1); + } + ++i; + } + *matched = i; + if (i == qname_span) + return (i == dname_span || qd->name[i] == '\0' ? 0 : -1); + return (1); +} + +/* + * Name of the idx'th dirent of a block, as a range inside the block. Only + * the bounds are checked; see erofs_dirent_span(). + */ +static int +erofs_dirent_qstr(const char *data, uint32_t datasize, uint32_t ndirents, + uint32_t idx, struct erofs_qstr *name) +{ + uint32_t endoff, nameoff; + int error; + + error = erofs_dirent_span(data, datasize, ndirents, idx, &nameoff, + &endoff); + if (error != 0) + return (error); + name->name = (const unsigned char *)data + nameoff; + name->end = (const unsigned char *)data + endoff; + return (0); +} + +/* + * Binary search within a directory block for the target name. + * + * Sets *dep to the matching dirent, or NULL on a miss. Only the entries + * the search visits are looked at, and each is bounds-checked as it is + * visited, so an unsorted or otherwise odd block makes the search miss but + * cannot take it outside the block. + */ +static int +find_target_dirent(const struct erofs_qstr *name, char *data, + uint32_t datasize, uint32_t ndirents, struct erofs_dirent **dep) +{ + uint32_t head, back; + unsigned int startprfx, endprfx; + struct erofs_dirent *const de = (struct erofs_dirent *)data; + int error; + + *dep = NULL; + /* The 1st dirent has already been evaluated by the caller. */ + head = 1; + back = ndirents - 1; + startprfx = endprfx = 0; + + while (head <= back) { + const uint32_t mid = head + (back - head) / 2; + unsigned int matched = MIN(startprfx, endprfx); + struct erofs_qstr dname; + int ret; + + error = erofs_dirent_qstr(data, datasize, ndirents, mid, + &dname); + if (error != 0) + return (error); + + /* String comparison without already matched prefix */ + ret = erofs_dirnamecmp(name, &dname, &matched); + + if (ret == 0) { + *dep = de + mid; + return (0); + } else if (ret > 0) { + head = mid + 1; + startprfx = matched; + } else { + back = mid - 1; + endprfx = matched; + } + } + + return (0); +} + +static int +erofs_read_dirblock(struct erofs_sb_info *sbi, struct erofs_inode *dir, + uint64_t block, char **datap, uint32_t *datasizep, uint32_t *ndirentsp) +{ + erofs_off_t block_off; + uint32_t datasize; + char *data; + int error; + + *datap = NULL; + /* The only caller searches the blocks that hold dir->size bytes. */ + block_off = block * sbi->block_size; + EROFS_DEBUG_ASSERT(block_off < dir->size, + ("erofs: directory block %ju beyond the size", (uintmax_t)block)); + datasize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, datasize, (void **)&data); + if (error != 0) + return (error); + error = erofs_dirblock_ndirents(data, datasize, ndirentsp); + if (error != 0) { + erofs_brelse(data); + return (erofs_corrupt(sbi, dir->nid, + "invalid dirent array in block %ju", (uintmax_t)block)); + } + *datap = data; + *datasizep = datasize; + return (0); +} + +/* + * Find the directory block most likely to contain the target name. + * + * Uses two-level binary search: first across blocks, then within the + * candidate block via find_target_dirent(). + * + * Returns the block buffer on success (caller must erofs_brelse), + * or NULL if no block matches, in which case *errorp is 0, or on error, + * when *errorp is set to a positive errno. *_ndirents is set to the number + * of dirents in the returned block (0 means the first entry is the match). + */ +static char * +erofs_find_target_block(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, uint32_t *_ndirents, uint32_t *_datasize, + int *errorp) +{ + uint32_t bsz = sbi->block_size; + uint64_t head, back; + unsigned int startprfx = 0, endprfx = 0; + char *candidate = NULL; + int error; + + *errorp = 0; + *_ndirents = 0; + *_datasize = 0; + + if (dir->size == 0) + return (NULL); + + head = 0; + back = (dir->size - 1) / bsz; + + while (head <= back) { + const uint64_t mid = head + (back - head) / 2; + uint32_t maxsize; + char *blk; + int diff; + uint32_t ndirents; + unsigned int matched; + struct erofs_qstr dname; + + error = erofs_read_dirblock(sbi, dir, mid, &blk, &maxsize, + &ndirents); + if (error != 0) { + *errorp = error; + goto out; + } + error = erofs_dirent_qstr(blk, maxsize, ndirents, 0, &dname); + if (error != 0) { + erofs_brelse(blk); + *errorp = erofs_corrupt(sbi, dir->nid, + "invalid name span of the first dirent in block " + "%ju", + (uintmax_t)mid); + goto out; + } + + matched = MIN(startprfx, endprfx); + /* String comparison without already matched prefix */ + diff = erofs_dirnamecmp(name, &dname, &matched); + + if (diff < 0) { + erofs_brelse(blk); + if (mid == 0) + break; + back = mid - 1; + endprfx = matched; + continue; + } + + /* diff >= 0: this block is a candidate. */ + if (candidate != NULL) + erofs_brelse(candidate); + candidate = blk; + if (diff == 0) { + *_ndirents = 0; + *_datasize = maxsize; + return (candidate); + } + head = mid + 1; + startprfx = matched; + *_ndirents = ndirents; + *_datasize = maxsize; + } + return (candidate); +out: + if (candidate != NULL) + erofs_brelse(candidate); + return (NULL); +} + +/* + * Look up a name in a directory and return its nid. + */ +static int +erofs_namei(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, erofs_nid_t *nid) +{ + int error; + uint32_t ndirents; + uint32_t datasize; + char *blk; + struct erofs_dirent *de; + + blk = erofs_find_target_block(sbi, dir, name, &ndirents, + &datasize, &error); + if (blk == NULL) + return (error != 0 ? error : ENOENT); + + de = (struct erofs_dirent *)blk; + if (ndirents > 0) { + error = find_target_dirent(name, blk, datasize, ndirents, &de); + if (error != 0) + error = erofs_corrupt(sbi, dir->nid, + "invalid name span in a dirent block"); + } + + if (error == 0 && de != NULL) { + erofs_nid_t found_nid; + + found_nid = le64toh(de->nid); + if (!erofs_nid_is_valid(sbi, found_nid)) { + error = erofs_corrupt(sbi, dir->nid, + "dirent points to invalid nid %ju", + (uintmax_t)found_nid); + } else { + *nid = found_nid; + } + } + erofs_brelse(blk); + if (error != 0) + return (error); + return (de != NULL ? 0 : ENOENT); +} + +/* + * Directory name lookup (VOP_CACHEDLOOKUP entry point). + * + * FreeBSD-side API requirements: + * - "." must be returned under the caller's requested lock mode; + * - ".." must go through vn_vget_ino() to avoid holding a child lock while + * acquiring the parent directory lock in reverse; + * - Both hit and miss must correctly update the namecache. + */ +int +erofs_lookup(struct vop_cachedlookup_args *ap) +{ + struct erofs_qstr qname; + struct vnode *dvp, *vp; + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct componentname *cnp; + erofs_nid_t nid; + int error, ltype; + + dvp = ap->a_dvp; + cnp = ap->a_cnp; + *ap->a_vpp = NULL; + if ((cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) + return (EROFS); + if (cnp->cn_namelen > EROFS_NAME_LEN) + return (ENAMETOOLONG); + erofs_trace_lookup(VTOE(dvp), cnp->cn_nameptr, + (size_t)cnp->cn_namelen, (uint32_t)cnp->cn_flags); + if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + + dir = VTOE(dvp); + sbi = MTOE(dvp->v_mount); + qname.name = (const unsigned char *)cnp->cn_nameptr; + qname.end = qname.name + cnp->cn_namelen; + error = erofs_namei(sbi, dir, &qname, &nid); + if (error != 0) { + if (error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, NULL, cnp); + if (error == ENOENT && (cnp->cn_flags & ISLASTCN) != 0 && + cnp->cn_nameiop == CREATE) + return (EROFS); + return (error); + } + if ((cnp->cn_flags & ISDOTDOT) != 0 && nid == dir->nid) { + if (nid != sbi->root_nid) + return (erofs_corrupt(sbi, dir->nid, + "'..' of a non-root directory points to itself")); + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + if ((cnp->cn_flags & ISDOTDOT) == 0 && nid == dir->nid) + return (erofs_corrupt(sbi, dir->nid, + "directory contains an entry pointing to itself")); + + if ((cnp->cn_flags & ISDOTDOT) != 0) + error = vn_vget_ino_gen(dvp, erofs_vget_raw_alloc, &nid, + cnp->cn_lkflags, &vp); + else + error = erofs_vget_raw(dvp->v_mount, nid, cnp->cn_lkflags, &vp); + if (error != 0) + return (error); + if ((cnp->cn_flags & ISDOTDOT) != 0 && VTOE(vp)->vtype != VDIR) { + vput(vp); + return (erofs_corrupt(sbi, dir->nid, + "'..' does not lead to a directory (nid %ju)", + (uintmax_t)nid)); + } + *ap->a_vpp = vp; + if ((cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, vp, cnp); + return (0); +} diff --git a/super.c b/super.c new file mode 100644 index 0000000..10cfa7d --- /dev/null +++ b/super.c @@ -0,0 +1,1643 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "trace.h" +#include "xattr.h" + +MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem"); + +static const uint32_t erofs_crc32c_seed = 0x5045b54aU; + +/* + * At most EROFS_ERR_MAXPPS lines a second for the whole module, so a bad + * directory cannot flood the log. The DTrace probe is not rate limited. + */ +#define EROFS_ERR_MAXPPS 5 +static struct timeval erofs_err_lasttime; +static int erofs_err_curpps; + +static const char * +erofs_mntonname(const struct erofs_sb_info *sbi) +{ + if (sbi == NULL || sbi->mnt == NULL) + return ("?"); + return (sbi->mnt->mnt_stat.f_mntonname); +} + +void +erofs_log_err(const struct erofs_sb_info *sbi, const char *func, + const char *fmt, ...) +{ + char msg[128]; + va_list ap; + + if (ppsratecheck(&erofs_err_lasttime, &erofs_err_curpps, + EROFS_ERR_MAXPPS) == 0) + return; + va_start(ap, fmt); + vsnprintf(msg, sizeof(msg), fmt, ap); + va_end(ap); + log(LOG_ERR, "erofs (%s): %s: %s\n", erofs_mntonname(sbi), func, msg); +} + +int +erofs_log_corrupt(const struct erofs_sb_info *sbi, uint64_t nid, + const char *func, int line, const char *fmt, ...) +{ + char msg[128]; + va_list ap; + + EROFS_TRACE_CORRUPT(nid, line, EINTEGRITY); + if (ppsratecheck(&erofs_err_lasttime, &erofs_err_curpps, + EROFS_ERR_MAXPPS) != 0) { + va_start(ap, fmt); + vsnprintf(msg, sizeof(msg), fmt, ap); + va_end(ap); + if (nid == EROFS_NID_NONE) + log(LOG_ERR, "erofs (%s): %s: %s\n", + erofs_mntonname(sbi), func, msg); + else + log(LOG_ERR, "erofs (%s): %s: %s @ nid %ju\n", + erofs_mntonname(sbi), func, msg, (uintmax_t)nid); + } + return (EINTEGRITY); +} + +static int +erofs_superblock_csum_verify(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t expected, crc; + size_t len; + int error; + + if (!erofs_sb_has_sb_chksum(sbi)) + return (0); + + len = 1u << dsb->blkszbits; + if (len > EROFS_SUPER_OFFSET) + len -= EROFS_SUPER_OFFSET; + + error = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, len, + &buf); + if (error != 0) + return (error); + + crc = calculate_crc32c(erofs_crc32c_seed, + (const uint8_t *)buf.data + + offsetof(struct erofs_super_block, checksum) + + sizeof(dsb->checksum), + len - offsetof(struct erofs_super_block, checksum) - + sizeof(dsb->checksum)); + expected = le32toh(dsb->checksum); + erofs_put_metabuf(&buf); + + if (crc != expected) { + vfs_mount_error(sbi->mnt, + "erofs: invalid superblock checksum 0x%08x, " + "0x%08x expected", crc, expected); + return (EBADMSG); + } + return (0); +} + + +static const char *erofs_opts[] = { + "acl", + "device", + "export", + "from", + "dir_ra_bytes", + "directio", + "cache_strategy", + "sync_decompress", + "worker_queue_priority", + "fsoffset", + "nodirectio", + "user_xattr", + /* Generic VFS options that other file systems accept. */ + "async", + "noatime", + "noclusterr", + "noclusterw", + "nosymfollow", + "sync", + "union", + NULL, +}; + +static vfs_mount_t erofs_mount; +static vfs_root_t erofs_root; +static vfs_statfs_t erofs_statfs; +static vfs_unmount_t erofs_unmount; +static vfs_fhtovp_t erofs_fhtovp; + +static int +erofs_parse_fsoffset(struct mount *mp, uint64_t *fsoffsetp, bool *presentp) +{ + struct vfsopt *opt; + const unsigned char *value; + uint64_t fsoffset; + size_t i; + + *fsoffsetp = 0; + *presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "fsoffset") != 0) + continue; + *presentp = true; + if (opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') + goto invalid; + value = opt->value; + fsoffset = 0; + for (i = 0; i < (size_t)opt->len - 1; ++i) { + if (value[i] < '0' || value[i] > '9') + goto invalid; + if (fsoffset > (UINT64_MAX - (value[i] - '0')) / 10) + goto overflow; + fsoffset = fsoffset * 10 + (value[i] - '0'); + } + *fsoffsetp = fsoffset; + } + return (0); + +invalid: + vfs_mount_error(mp, "erofs: fsoffset requires an unsigned byte offset"); + return (EINVAL); +overflow: + vfs_mount_error(mp, "erofs: fsoffset value is too large"); + return (ERANGE); +} + +static int +erofs_parse_directio_options(struct mount *mp, bool *enabledp, bool *presentp) +{ + struct vfsopt *opt; + + *enabledp = false; + *presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "directio") != 0 && + strcmp(opt->name, "nodirectio") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, + "erofs: directio/nodirectio take no value"); + return (EINVAL); + } +#if !WITH_EROFS_BACKED_BY_FILE + vfs_mount_error(mp, + "erofs: directio support is not built"); + return (EOPNOTSUPP); +#endif + *presentp = true; + *enabledp = strcmp(opt->name, "directio") == 0; + } + return (0); +} + +static int +erofs_parse_zip_mount_options(struct mount *mp, + struct erofs_sb_info_opts *opts __unused, unsigned int *syncp __unused, + bool *cache_presentp, bool *sync_presentp, bool *priority_presentp) +{ +#if WITH_EROFS_ZIP + struct vfsopt *opt; + int error; + + *priority_presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "worker_queue_priority") == 0) + *priority_presentp = true; + } + error = erofs_parse_zip_options(mp, opts, syncp, + cache_presentp, sync_presentp); + if (error != 0) { + vfs_mount_error(mp, + "erofs: invalid cache_strategy, sync_decompress, or " + "worker_queue_priority option"); + return (error); + } + return (0); +#else + struct vfsopt *opt; + + *cache_presentp = false; + *sync_presentp = false; + *priority_presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "cache_strategy") != 0 && + strcmp(opt->name, "sync_decompress") != 0 && + strcmp(opt->name, "worker_queue_priority") != 0) + continue; + vfs_mount_error(mp, + "erofs: compression options require ZIP support"); + return (EOPNOTSUPP); + } + return (0); +#endif +} + +#define EROFS_DEVICE_OPT_PREFIX "device." + +struct erofs_device_arg { + uint16_t slot; + char *path; +}; + +static int +erofs_load_generation_seed(struct erofs_sb_info *sbi, uint32_t sb_size, + uint32_t *seedp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t seed; + int error; + + error = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, sb_size, + &buf); + if (error != 0) + return (error); + seed = fnv_32_buf(buf.data, sb_size, FNV1_32_INIT); + erofs_put_metabuf(&buf); + *seedp = seed != 0 ? seed : 1; + return (0); +} + +static void +erofs_free_device_args(struct erofs_device_arg *args, unsigned int count) +{ + unsigned int i; + + if (args == NULL) + return; + for (i = 0; i < count; ++i) + free(args[i].path, M_EROFS); + free(args, M_EROFS); +} + +static int +erofs_parse_device_slot(const char *name, uint16_t *slotp) +{ + const char *p; + unsigned int slot; + + if (strncmp(name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + return (ENOENT); + p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1; + if (*p < '1' || *p > '9') + return (EINVAL); + slot = 0; + for (; *p != '\0'; ++p) { + if (*p < '0' || *p > '9' || + slot > (unsigned int)(UINT16_MAX - (*p - '0')) / 10) + return (EINVAL); + slot = slot * 10 + (*p - '0'); + } + *slotp = slot; + return (0); +} + +static int +erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp, + unsigned int *countp) +{ + struct erofs_device_arg *args; + struct vfsopt *opt; + char name[32]; + unsigned int count, device_count, slot_count, i; + uint16_t slot; + int error; + + *argsp = NULL; + *countp = 0; + count = 0; + device_count = 0; + slot_count = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") == 0) { + if (opt->value == NULL || opt->len == 0 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, + "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (device_count == UINT16_MAX) + return (E2BIG); + ++device_count; + continue; + } + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + if (error != 0 || opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, + "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (slot_count == UINT16_MAX) + return (E2BIG); + ++slot_count; + } + if (device_count != 0 && slot_count != 0) { + vfs_mount_error(mp, + "erofs: device and device.N options cannot be combined"); + return (EINVAL); + } + count = device_count != 0 ? device_count : slot_count; + if (count == 0) + return (0); + + args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO); + if (device_count != 0) { + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") != 0) + continue; + args[i].slot = i + 1; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + vfs_deleteopt(mp->mnt_optnew, "device"); + } else { + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + EROFS_DEBUG_ASSERT(error == 0, + ("validated EROFS device option changed")); + args[i].slot = slot; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + for (i = 0; i < count; ++i) { + unsigned int j; + + for (j = 0; j < i; ++j) { + if (args[i].slot != args[j].slot) + continue; + vfs_mount_error(mp, + "erofs: external device slot %u was " + "specified more than once", + args[i].slot); + erofs_free_device_args(args, count); + return (EINVAL); + } + } + for (i = 0; i < count; ++i) { + snprintf(name, sizeof(name), + EROFS_DEVICE_OPT_PREFIX "%u", args[i].slot); + vfs_deleteopt(mp->mnt_optnew, name); + } + } + *argsp = args; + *countp = count; + return (0); +} + +static int +erofs_reject_device_update(struct mount *mp) +{ + struct vfsopt *opt; + + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") != 0 && + strncmp(opt->name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + continue; + vfs_mount_error(mp, + "erofs: external devices cannot be changed on update"); + return (EOPNOTSUPP); + } + return (0); +} + +static int +erofs_parse_acl_options(struct mount *mp, bool default_enabled, bool *enabledp) +{ + struct vfsopt *opt; + bool enabled; + + enabled = default_enabled; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "acl") != 0 && + strcmp(opt->name, "noacl") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, "erofs: acl/noacl take no value"); + return (EINVAL); + } +#if WITH_EROFS_POSIX_ACL + enabled = strcmp(opt->name, "acl") == 0; +#else + vfs_mount_error(mp, "erofs: POSIX ACL support is not built"); + return (EOPNOTSUPP); +#endif + } +#if !WITH_EROFS_POSIX_ACL + enabled = false; +#endif + *enabledp = enabled; + return (0); +} + +static int +erofs_parse_xattr_options(struct mount *mp, bool default_enabled, + bool *enabledp) +{ + struct vfsopt *opt; + bool enabled; + + enabled = default_enabled; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "user_xattr") != 0 && + strcmp(opt->name, "nouser_xattr") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, + "erofs: user_xattr/nouser_xattr take no value"); + return (EINVAL); + } +#if !WITH_EROFS_XATTR + vfs_mount_error(mp, + "erofs: extended attribute support is not built"); + return (EOPNOTSUPP); +#endif + enabled = strcmp(opt->name, "user_xattr") == 0; + } +#if !WITH_EROFS_XATTR + enabled = false; +#endif + *enabledp = enabled; + return (0); +} + +static void +erofs_set_xattr_state(struct erofs_sb_info *sbi, bool enabled) +{ + if (enabled) + set_opt(&sbi->opt, XATTR_USER); + else + clear_opt(&sbi->opt, XATTR_USER); +} + +static void +erofs_set_acl_state(struct erofs_sb_info *sbi, bool enabled) +{ + struct mount *mp; + + mp = sbi->mnt; + if (enabled) { + set_opt(&sbi->opt, POSIX_ACL); + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_ACLS; + MNT_IUNLOCK(mp); + } else { + MNT_ILOCK(mp); + mp->mnt_flag &= ~MNT_ACLS; + MNT_IUNLOCK(mp); + clear_opt(&sbi->opt, POSIX_ACL); + } +} + +struct erofs_device_handle { + struct vnode *devvp; /* block: private mntfs vnode; file: the file */ + struct vnode *odevvp; /* block: the devfs vnode, reference only */ + struct cdev *dev; + struct g_consumer *cp; + struct mount *source_mp; + struct mount_upper_node source_upper; + volatile u_int refs; +}; + +/* + * A mount reads through VOP_READ on another mount whenever any of its devices + * is file-backed, not just the primary one: FreeBSD allows a block-backed + * primary device together with file-backed extra devices. + */ +static bool +erofs_mount_has_file_source(const struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (erofs_source_is_file(&sbi->dif0)) + return (true); + for (i = 0; i < sbi->extra_devices; ++i) { + if (erofs_source_is_file(&sbi->devs[i])) + return (true); + } + return (false); +} + +/* + * The registration pins source_mp. Judge the mount that really holds the + * data: nullfs forwards VOP_READ to the vnode below it, and + * VOP_GETLOWVNODE() looks through it. unionfs does not implement that, so it + * is refused by name, as Linux refuses any stacked filesystem. + */ +static int +erofs_check_source_stacking(struct erofs_sb_info *sbi, struct vnode *devvp) +{ + struct erofs_sb_info *lower; + struct vnode *lowvp; + struct mount *lowmp; + int error; + + error = VOP_GETLOWVNODE(devvp, &lowvp, FREAD); + if (error != 0) + return (error); + lowmp = lowvp->v_mount; + error = 0; + if (lowmp == NULL) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source filesystem is being unmounted"); + error = EBUSY; + } else if (strcmp(lowmp->mnt_vfc->vfc_name, "unionfs") == 0) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed mounts cannot be applied to stacked " + "fses"); + error = ENOTBLK; + } else if (lowmp->mnt_vfc == sbi->mnt->mnt_vfc) { + lower = lowmp->mnt_data; + if (lower != NULL && erofs_mount_has_file_source(lower)) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed mounts cannot be applied to " + "file-backed EROFS mounts"); + error = ENOTBLK; + } + } + vrele(lowvp); + return (error); +} + +static int +erofs_register_source_upper(struct erofs_sb_info *sbi, + struct erofs_device_info *dif) +{ + struct mount *source_mnt, *covered_mnt; + struct mount *source_mp; + int error; + + if (!erofs_source_is_file(dif)) + return (0); + if (sbi == NULL || sbi->mnt == NULL || dif->handle == NULL || + dif->devvp == NULL) + return (EINVAL); + /* + * These run before registration because vfs_register_upper_from_vp() + * asserts that the lower mount is not the one being built, so the + * self-referential case has to be rejected first. + */ + source_mnt = dif->devvp->v_mount; + if (source_mnt == sbi->mnt) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source resolves to the mount being " + "built"); + return (EDEADLK); + } + covered_mnt = sbi->mnt->mnt_vnodecovered != NULL ? + sbi->mnt->mnt_vnodecovered->v_mount : NULL; + if (covered_mnt != NULL && source_mnt == covered_mnt && + source_mnt->mnt_vfc == sbi->mnt->mnt_vfc) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source and mount target are on the " + "same mount"); + return (ELOOP); + } + source_mp = vfs_register_upper_from_vp(dif->devvp, sbi->mnt, + &dif->handle->source_upper); + if (source_mp == NULL) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source filesystem is being unmounted"); + return (EBUSY); + } + dif->handle->source_mp = source_mp; + /* + * Registering as an upper mount pins source_mp until this mount is torn + * down, so the mounts below it (and their erofs_sb_info) stay valid for + * the check. + * + * Reading a file-backed source re-enters VOP_READ on the lower mount + * while this frame is still live. Refuse another file-backed EROFS + * level, matching Linux: a block-backed EROFS mount may still carry one + * file-backed mount, so the depth stays 2. + */ + error = erofs_check_source_stacking(sbi, dif->devvp); + if (error != 0) { + vfs_unregister_upper(source_mp, &dif->handle->source_upper); + dif->handle->source_mp = NULL; + return (error); + } + return (0); +} + +static void +erofs_destroy_device_handle(struct erofs_device_handle *handle) +{ + if (handle == NULL) + return; + if (handle->source_mp != NULL) { + vfs_unregister_upper(handle->source_mp, &handle->source_upper); + handle->source_mp = NULL; + } + if (handle->odevvp != NULL) { + /* + * Block device: close the consumer and free the private vnode + * with the vnode locked, then drop the devfs vnode reference. + */ + if (handle->devvp != NULL) { + vn_lock(handle->devvp, LK_EXCLUSIVE | LK_RETRY); + if (handle->cp != NULL) { + g_topology_lock(); + g_vfs_close(handle->cp); + g_topology_unlock(); + } + mntfs_freevp(handle->devvp); + } + vrele(handle->odevvp); + } else if (handle->devvp != NULL) { + vrele(handle->devvp); + } + if (handle->dev != NULL) + dev_rel(handle->dev); + free(handle, M_EROFS); +} + +static void +erofs_release_device_info(struct erofs_device_info *dif) +{ + struct erofs_device_handle *handle; + + handle = dif->handle; + dif->handle = NULL; + dif->cp = NULL; + dif->devvp = NULL; + dif->dev = NULL; + if (dif->source_cred != NULL) { + crfree(dif->source_cred); + dif->source_cred = NULL; + } + dif->source_kind = EROFS_SOURCE_NONE; + if (handle == NULL || !refcount_release(&handle->refs)) + return; + erofs_destroy_device_handle(handle); +} + +static struct erofs_device_handle * +erofs_find_device_handle(struct erofs_sb_info *sbi, struct g_provider *pp) +{ + unsigned int i; + + if (sbi == NULL) + return (NULL); + if (sbi->dif0.cp != NULL && sbi->dif0.cp->provider == pp) + return (sbi->dif0.handle); + for (i = 0; i < sbi->extra_devices; ++i) { + if (sbi->devs[i].cp != NULL && sbi->devs[i].cp->provider == pp) + return (sbi->devs[i].handle); + } + return (NULL); +} + +static int +erofs_open_device(struct mount *mp, struct erofs_sb_info *sbi, + const char *path, struct erofs_device_info *dif) +{ + struct erofs_device_handle *handle, *shared; + struct g_provider *pp; + struct nameidata nd; + struct vnode *devvp, *mdevvp; + struct cdev *dev; + bool reused, need_open; + int error; + + bzero(dif, sizeof(*dif)); + handle = malloc(sizeof(*handle), M_EROFS, M_WAITOK | M_ZERO); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path); + error = namei(&nd); + if (error != 0) { + free(handle, M_EROFS); + return (error); + } + devvp = nd.ni_vp; + NDFREE_PNBUF(&nd); + if (devvp->v_type == VREG) { +#if !WITH_EROFS_BACKED_BY_FILE + vput(devvp); + free(handle, M_EROFS); + return (EOPNOTSUPP); +#else + struct vattr va; + + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, + curthread); + if (error == 0) + error = VOP_GETATTR(devvp, &va, curthread->td_ucred); + if (error != 0) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + dif->source_cred = crhold(curthread->td_ucred); + handle->devvp = devvp; + refcount_init(&handle->refs, 1); + dif->handle = handle; + dif->devvp = devvp; + dif->source_kind = EROFS_SOURCE_FILE; + dif->mediasize = (uint64_t)va.va_size; + dif->sectorsize = 1; + VOP_UNLOCK(devvp); + return (0); +#endif + } + if (devvp->v_type != VCHR) { + vput(devvp); + free(handle, M_EROFS); + return (EINVAL); + } + if (!vn_isdisk_error(devvp, &error)) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread); + if (error != 0) + error = priv_check(curthread, PRIV_VFS_MOUNT_PERM); + if (error != 0) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + dev = devvp->v_rdev; + dev_ref(dev); + handle->odevvp = devvp; + handle->dev = dev; + refcount_init(&handle->refs, 1); + reused = false; + need_open = false; + g_topology_lock(); + pp = g_dev_getprovider(dev); + if (pp == NULL) + error = ENXIO; + else if ((shared = erofs_find_device_handle(sbi, pp)) != NULL) { + refcount_acquire(&shared->refs); + dif->handle = shared; + dif->cp = shared->cp; + dif->devvp = shared->devvp; + dif->dev = shared->dev; + dif->source_kind = EROFS_SOURCE_BLOCK; + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + reused = true; + error = 0; + } else { + need_open = true; + error = 0; + } + g_topology_unlock(); + if (need_open) { + /* + * Use a private vnode for the device, not the devfs one. When + * the device goes away devfs vgone()s its own vnode, and with + * it the buffer cache state that this mount is still using. + * mntfs_allocvp() unlocks the devfs vnode and returns the new + * one locked. + */ + mdevvp = mntfs_allocvp(mp, devvp); + /* + * Readahead takes this lock around breadn(), which waits for + * I/O. devfs vnodes allow shared locking, so keep that. + */ + VN_LOCK_ASHARE(mdevvp); + g_topology_lock(); + error = g_vfs_open(mdevvp, &handle->cp, "erofs", 0); + if (error == 0) { + handle->devvp = mdevvp; + dif->handle = handle; + dif->cp = handle->cp; + dif->devvp = mdevvp; + dif->dev = handle->dev; + dif->source_kind = EROFS_SOURCE_BLOCK; + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + } + g_topology_unlock(); + if (error == 0) + VOP_UNLOCK(mdevvp); + else + mntfs_freevp(mdevvp); + } else { + VOP_UNLOCK(devvp); + } + if (error != 0 || reused) { + erofs_destroy_device_handle(handle); + } + if (error != 0) { + return (error); + } + if (dif->sectorsize == 0 || + (dif->sectorsize & (dif->sectorsize - 1)) != 0) { + erofs_release_device_info(dif); + return (EINVAL); + } + return (0); +} + +static void +erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif) +{ + u_long iosize; + + iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ? + dif->dev->si_iosize_max : MAXPHYS; + mp->mnt_iosize_max = (int)MIN((u_long)mp->mnt_iosize_max, + MIN(iosize, (u_long)MAXPHYS)); +} + +static void +erofs_free_dev_context(struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (sbi->devs != NULL) { + for (i = sbi->extra_devices; i > 0; --i) + erofs_release_device_info(&sbi->devs[i - 1]); + free(sbi->devs, M_EROFS); + } +} + +static void +erofs_drop_internal_inodes(struct erofs_sb_info *sbi) +{ + if (sbi->metabox_en != NULL) + free(sbi->metabox_en, M_EROFS); + if (sbi->packed_inode != NULL) + free(sbi->packed_inode, M_EROFS); +} + +static int +erofs_sb_free(struct erofs_sb_info *sbi) +{ + int error; + + if (sbi == NULL) + return (0); + error = erofs_unregister_sysfs(sbi); + if (error != 0) + return (error); +#if WITH_EROFS_ZIP + z_erofs_extent_cache_fini(sbi); + z_erofs_stream_pool_fini(sbi); +#endif + erofs_xattr_prefixes_cleanup(sbi); + erofs_drop_internal_inodes(sbi); + erofs_free_dev_context(sbi); + erofs_release_device_info(&sbi->dif0); + erofs_management_fini(sbi); + free(sbi, M_EROFS); + return (0); +} + +static void +erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat, + uint64_t *blocks, erofs_nid_t *root_nid) +{ + *blocks = le32toh(dsb->blocks_lo); + if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 && + dsb->rootnid_8b != 0) { + *blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32; + *root_nid = le64toh(dsb->rootnid_8b); + } else { + *root_nid = le16toh(dsb->rb.rootnid_2b); + } +} + +static int +erofs_validate_device_size(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_blk_t blocks) +{ + uint64_t bytes, end; + + /* + * The sector size is a power of two (erofs_open_device()), so a block + * that is not smaller is also a multiple of it. A block count has at + * most 48 bits and blkszbits is at most 16 (internal.h), so the shift + * cannot overflow. + */ + if (sbi->block_size < dif->sectorsize) + return (EINVAL); + bytes = blocks << sbi->blkszbits; + if (__builtin_add_overflow(dif->fsoff, bytes, &end)) + return (EOVERFLOW); + if (end > dif->mediasize) + return (ENXIO); + return (0); +} + +static int +erofs_init_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + const char *path) +{ + struct erofs_device_info opened; + erofs_blk_t blocks, uniaddr; + int error; + + blocks = dif->blocks; + uniaddr = dif->uniaddr; + error = erofs_open_device(sbi->mnt, sbi, path, &opened); + if (error != 0) + return (error); + erofs_update_iosize_max(sbi->mnt, &opened); + opened.blocks = blocks; + opened.uniaddr = uniaddr; + error = erofs_validate_device_size(sbi, &opened, opened.blocks); + if (error != 0) { + erofs_release_device_info(&opened); + return (error); + } + error = erofs_register_source_upper(sbi, &opened); + if (error != 0) { + erofs_release_device_info(&opened); + return (error); + } + *dif = opened; + return (0); +} + +static const char * +erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count, + unsigned int slot) +{ + unsigned int i; + + for (i = 0; i < count; ++i) { + if (args[i].slot == slot) + return (args[i].path); + } + return (NULL); +} + +static int +erofs_scan_devices(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_deviceslot *slots; + struct erofs_device_info *dif; + const char *path; + erofs_off_t devt_off; + uint64_t devt_size, image_size, end; + erofs_blk_t maxend; + unsigned int i, mask; + int err; + + sbi->total_blocks = sbi->dif0.blocks; + sbi->flatdev_blocks = sbi->dif0.blocks; + if (sbi->extra_devices == 0) { + if (arg_count != 0) { + vfs_mount_error(sbi->mnt, + "erofs: external devices given without a device " + "table"); + return (EINVAL); + } + return (0); + } + devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE; + devt_size = (uint64_t)sbi->extra_devices * EROFS_DEVT_SLOT_SIZE; + if (sbi->dif0.blocks > (UINT64_MAX >> sbi->blkszbits)) { + vfs_mount_error(sbi->mnt, "erofs: image size overflows"); + return (EINTEGRITY); + } + image_size = sbi->dif0.blocks << sbi->blkszbits; + if (devt_off > image_size || devt_size > image_size - devt_off) { + vfs_mount_error(sbi->mnt, + "erofs: device table of %ju bytes at %ju is outside the " + "image", (uintmax_t)devt_size, (uintmax_t)devt_off); + return (EINTEGRITY); + } + err = erofs_read_metadata_range(sbi, 0, devt_off, (size_t)devt_size, + &buf); + if (err != 0) + return (err); + slots = buf.data; + sbi->devs = mallocarray(sbi->extra_devices, sizeof(*sbi->devs), M_EROFS, + M_WAITOK | M_ZERO); + maxend = sbi->dif0.blocks; + for (i = 0; i < sbi->extra_devices; ++i) { + dif = &sbi->devs[i]; + dif->blocks = le32toh(slots[i].blocks_lo); + dif->uniaddr = le32toh(slots[i].uniaddr_lo); + if (erofs_sb_has_48bit(sbi)) { + dif->blocks |= + (uint64_t)le16toh(slots[i].blocks_hi) << 32; + dif->uniaddr |= + (uint64_t)le16toh(slots[i].uniaddr_hi) << 32; + } + /* + * blocks == 0 is valid: mkfs.erofs --blobdev leaves an empty + * device when no chunk data goes to it. Every read from such a + * device fails the range check in erofs_check_device_range(). + */ + if (__builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) { + vfs_mount_error(sbi->mnt, + "erofs: device %u: unified address plus size " + "overflows", i + 1); + err = EINTEGRITY; + goto out; + } + /* + * The slots are not checked against each other or against the + * primary image: overlapping or out-of-range slots only make + * the address lookup ambiguous, and every resolved range is + * checked against its device in erofs_check_device_range() and + * erofs_bread_device(). A pairwise scan would cost O(n^2) at + * mount for up to 65535 slots. + */ + if (__builtin_add_overflow(sbi->total_blocks, dif->blocks, + &sbi->total_blocks)) { + err = EOVERFLOW; + goto out; + } + maxend = MAX(maxend, (erofs_blk_t)end); + } + erofs_put_metabuf(&buf); + sbi->flatdev_blocks = maxend; + mask = 1; + while (mask < (unsigned int)sbi->extra_devices + 1) + mask <<= 1; + sbi->device_id_mask = mask - 1; + sbi->flatdev = arg_count == 0; + if (sbi->flatdev) { + /* + * The device table addresses a unified space that is larger + * than the primary image whenever the other devices are not + * part of it, e.g. a split image mounted without device + * options. Such an image still mounts, as on Linux; a read + * that needs a range beyond the provider fails with ENXIO in + * erofs_bread_device(). + */ + return (0); + } + if (arg_count != sbi->extra_devices) { + vfs_mount_error(sbi->mnt, + "erofs: external devices don't match (ondisk %u, given %u)", + sbi->extra_devices, arg_count); + return (arg_count < sbi->extra_devices ? ENXIO : EINVAL); + } + for (i = 0; i < arg_count; ++i) { + if (args[i].slot == 0 || args[i].slot > sbi->extra_devices) + return (EINVAL); + } + for (i = 0; i < sbi->extra_devices; ++i) { + path = erofs_device_arg_path(args, arg_count, i + 1); + if (path == NULL) + return (ENXIO); + err = erofs_init_device(sbi, &sbi->devs[i], path); + if (err != 0) + return (err); + } + return (0); +out: + erofs_put_metabuf(&buf); + return (err); +} + +static int +erofs_init_packed_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* Load the packed carrier before any fragment-backed metabox inode. */ + if (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) { + sbi->packed_inode = malloc(sizeof(*sbi->packed_inode), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->packed_nid, + sbi->packed_inode); + if (error != 0) { + free(sbi->packed_inode, M_EROFS); + sbi->packed_inode = NULL; + return (error); + } + /* + * The type of the packed inode is not checked, and neither is + * whether it has fragments of its own: every read of it is + * bounded, and z_erofs_do_read() refuses a fragment that points + * into the packed inode itself. + */ + } + return (0); +} + +static int +erofs_init_metabox_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* + * METABOX NIDs address inode slots in this backing inode's data. The + * packed carrier is ready first so a compressed metabox may legally end + * in a fragment pcluster without reading an uninitialized dependency. + */ + if (erofs_sb_has_metabox(sbi)) { + sbi->metabox_en = malloc(sizeof(*sbi->metabox_en), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->metabox_nid, + sbi->metabox_en); + if (error != 0) + return (error); + } + return (0); +} + +static int +erofs_read_superblock(struct erofs_sb_info *sbi, struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t unsupported; + int ret; + + ret = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, + sizeof(*dsb), &buf); + if (ret != 0) + return (ret); + memcpy(dsb, buf.data, sizeof(*dsb)); + erofs_put_metabuf(&buf); + + if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1) + return (EINVAL); + if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT) + return (EINVAL); + sbi->blkszbits = dsb->blkszbits; + sbi->block_size = 1u << sbi->blkszbits; + if ((sbi->dif0.fsoff & (sbi->block_size - 1)) != 0) { + vfs_mount_error(sbi->mnt, + "erofs: fsoffset %ju is not aligned to block size %u", + (uintmax_t)sbi->dif0.fsoff, sbi->block_size); + return (EINVAL); + } + if (dsb->dirblkbits != 0) { + vfs_mount_error(sbi->mnt, + "erofs: dirblkbits %u isn't supported", dsb->dirblkbits); + return (EINVAL); + } + sbi->feature_compat = le32toh(dsb->feature_compat); + ret = erofs_superblock_csum_verify(sbi, dsb); + if (ret != 0) + return (ret); + + sbi->feature_incompat = le32toh(dsb->feature_incompat); + sbi->packed_nid = le64toh(dsb->packed_nid); + sbi->extra_devices = erofs_sb_has_device_table(sbi) ? + le16toh(dsb->extra_devices) : 0; + unsupported = sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT; + if (unsupported != 0) { + vfs_mount_error(sbi->mnt, + "erofs: unidentified incompatible feature %x, please " + "upgrade kernel", + unsupported); + return (EINVAL); + } + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) + return (EINVAL); + sbi->meta_blkaddr = le32toh(dsb->meta_blkaddr); +#if WITH_EROFS_XATTR + sbi->xattr_blkaddr = le32toh(dsb->xattr_blkaddr); + sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start); + sbi->xattr_prefix_count = dsb->xattr_prefix_count; + sbi->xattr_filter_reserved = dsb->xattr_filter_reserved; +#endif + erofs_sb_blocks_root(dsb, sbi->feature_incompat, &sbi->blocks, + &sbi->root_nid); + sbi->dif0.blocks = sbi->blocks; + if (sbi->dif0.blocks == 0) { + vfs_mount_error(sbi->mnt, "erofs: the image declares 0 blocks"); + return (EINTEGRITY); + } + ret = erofs_validate_device_size(sbi, &sbi->dif0, sbi->dif0.blocks); + if (ret != 0) + return (ret); + sbi->inos = le64toh(dsb->inos); + sbi->epoch = (int64_t)le64toh(dsb->epoch); + /* Not checked here: erofs_set_timestamp() checks it for each inode. */ + sbi->fixed_nsec = le32toh(dsb->fixed_nsec); + ret = erofs_load_generation_seed(sbi, sbi->sb_size, + &sbi->generation_seed); + if (ret != 0) + return (ret); + if (erofs_sb_has_metabox(sbi)) { + if (sbi->sb_size <= offsetof(struct erofs_super_block, + metabox_nid)) { + vfs_mount_error(sbi->mnt, + "erofs: the superblock has no room for " + "metabox_nid"); + return (EINTEGRITY); + } + sbi->metabox_nid = le64toh(dsb->metabox_nid); + if (erofs_nid_in_metabox(sbi->metabox_nid)) { + vfs_mount_error(sbi->mnt, + "erofs: metabox nid %ju is in the metabox", + (uintmax_t)sbi->metabox_nid); + return (EINTEGRITY); + } + } + +#if WITH_EROFS_ZIP + return (z_erofs_parse_cfgs(sbi, dsb)); +#else + if (dsb->u1.available_compr_algs != 0 || + erofs_sb_has_lz4_0padding(sbi)) { + vfs_mount_error(sbi->mnt, + "erofs: compression disabled, unable to mount compressed " + "EROFS"); + return (EOPNOTSUPP); + } + return (0); +#endif +} + +static int +erofs_mountfs(struct erofs_device_info *primary, struct mount *mp, + const struct erofs_device_arg *args, unsigned int arg_count, + bool xattr_user_enabled, bool acl_enabled, bool directio_enabled) +{ + struct erofs_sb_info *sbi; + struct erofs_inode root; + struct erofs_super_block dsb; + bool cache_present, sync_present, priority_present; + int cleanup_error, error; + + sbi = malloc(sizeof(*sbi), M_EROFS, M_WAITOK | M_ZERO); + sbi->mnt = mp; + erofs_management_init(sbi); + sbi->opt.cache_strategy = EROFS_ZIP_CACHE_READAROUND; + sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_AUTO; + error = erofs_parse_zip_mount_options(mp, &sbi->opt, + &sbi->sync_decompress, &cache_present, &sync_present, + &priority_present); + if (error != 0) + goto fail; + mtx_lock(&sbi->control_lock); + sbi->decompress_queue_priority = sbi->opt.worker_queue_priority; + mtx_unlock(&sbi->control_lock); +#if WITH_EROFS_ZIP + z_erofs_extent_cache_init(sbi); + z_erofs_stream_pool_init(sbi); +#endif + sbi->dif0 = *primary; + bzero(primary, sizeof(*primary)); + if (directio_enabled) + set_opt(&sbi->opt, DIRECT_IO); + error = erofs_register_source_upper(sbi, &sbi->dif0); + if (error != 0) + goto fail; + + error = erofs_read_superblock(sbi, &dsb); + if (error != 0) + goto fail; + error = erofs_parse_dir_ra_option(sbi, mp->mnt_optnew); + if (error != 0) { + vfs_mount_error(mp, "erofs: invalid dir_ra_bytes option"); + goto fail; + } + vfs_deleteopt(mp->mnt_optnew, "dir_ra_bytes"); + error = erofs_scan_devices(sbi, &dsb, args, arg_count); + if (error != 0) + goto fail; + + error = erofs_init_packed_inode(sbi); + if (error != 0) + goto fail; + error = erofs_init_metabox_inode(sbi); + if (error != 0) + goto fail; + error = erofs_read_inode(sbi, sbi->root_nid, &root); + if (error != 0) + goto fail; + if (root.vtype != VDIR) { + vfs_mount_error(mp, + "erofs: root inode nid=%ju is not a directory", + (uintmax_t)sbi->root_nid); + error = EINVAL; + goto fail; + } + error = erofs_xattr_prefixes_init(sbi); + if (error != 0) + goto fail; + + mp->mnt_data = sbi; + if (sbi->dif0.source_kind == EROFS_SOURCE_FILE) + vfs_getnewfsid(mp); + else { + mp->mnt_stat.f_fsid.val[0] = dev2udev(sbi->dif0.devvp->v_rdev); + mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum; + } + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY; + mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED | + MNTK_USES_BCACHE; + MNT_IUNLOCK(mp); + erofs_set_xattr_state(sbi, xattr_user_enabled); + erofs_set_acl_state(sbi, acl_enabled); + error = erofs_register_sysfs(sbi); + if (error != 0) + goto fail; + return (0); +fail: + cleanup_error = erofs_sb_free(sbi); + if (mp->mnt_data == sbi) + mp->mnt_data = NULL; + return (cleanup_error != 0 ? cleanup_error : error); +} + +static int +erofs_mount(struct mount *mp) +{ + struct erofs_device_arg *args; + struct erofs_device_info primary; + struct erofs_sb_info *sbi; + struct erofs_sb_info_opts next_opt; + char *fspec; + uint64_t fsoffset, new_dir_ra; + unsigned int next_sync; + unsigned int arg_count; + bool directio_enabled, directio_present; + bool fsoffset_present, xattr_user_enabled, acl_enabled; + bool cache_present, sync_present, priority_present, dir_ra_present; + int error, len; + + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_RDONLY; + MNT_IUNLOCK(mp); + if (mp->mnt_flag & MNT_UPDATE) { + sbi = MTOE(mp); + if (sbi == NULL) + return (EINVAL); + if (mp->mnt_flag & MNT_RELOAD) + return (EOPNOTSUPP); + error = erofs_reject_device_update(mp); + if (error != 0) + return (error); + error = erofs_parse_fsoffset(mp, &fsoffset, &fsoffset_present); + if (error != 0) + goto update_fail; + error = erofs_parse_directio_options(mp, &directio_enabled, + &directio_present); + if (error != 0) + goto update_fail; + if (fsoffset_present && fsoffset != sbi->dif0.fsoff) { + vfs_mount_error(mp, + "erofs: fsoffset cannot be changed on update"); + error = EOPNOTSUPP; + goto update_fail; + } + error = vfs_filteropt(mp->mnt_optnew, erofs_opts); + if (error != 0) + goto update_fail; + mtx_lock(&sbi->control_lock); + bzero(&next_opt, sizeof(next_opt)); + next_opt.cache_strategy = sbi->opt.cache_strategy; + next_opt.worker_queue_priority = + sbi->opt.worker_queue_priority; + next_sync = sbi->sync_decompress; + mtx_unlock(&sbi->control_lock); + error = erofs_parse_zip_mount_options(mp, &next_opt, + &next_sync, &cache_present, &sync_present, + &priority_present); + if (error != 0) + goto update_fail; + error = erofs_parse_acl_options(mp, + test_opt(&sbi->opt, POSIX_ACL) != 0, &acl_enabled); + if (error != 0) + goto update_fail; + error = erofs_parse_xattr_options(mp, + test_opt(&sbi->opt, XATTR_USER) != 0, &xattr_user_enabled); + if (error != 0) + goto update_fail; + new_dir_ra = 0; + error = erofs_parse_dir_ra_value(sbi, mp->mnt_optnew, + &new_dir_ra, &dir_ra_present); + if (error != 0) { + vfs_mount_error(mp, + "erofs: invalid dir_ra_bytes option"); + goto update_fail; + } + vfs_deleteopt(mp->mnt_optnew, "dir_ra_bytes"); + if (dir_ra_present) { + error = erofs_set_dir_ra_bytes(sbi, new_dir_ra); + if (error != 0) + goto update_fail; + } + mtx_lock(&sbi->control_lock); + if (cache_present) + sbi->opt.cache_strategy = next_opt.cache_strategy; + if (sync_present) + sbi->sync_decompress = next_sync; + if (priority_present) { + sbi->opt.worker_queue_priority = + next_opt.worker_queue_priority; + sbi->decompress_queue_priority = + next_opt.worker_queue_priority; + } + mtx_unlock(&sbi->control_lock); + erofs_set_xattr_state(sbi, xattr_user_enabled); + erofs_set_acl_state(sbi, acl_enabled); + if (directio_present) { + if (directio_enabled) + set_opt(&sbi->opt, DIRECT_IO); + else + clear_opt(&sbi->opt, DIRECT_IO); + } + return (0); +update_fail: + return (error); + } + args = NULL; + arg_count = 0; + error = erofs_parse_device_options(mp, &args, &arg_count); + if (error != 0) + return (error); + if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) { + error = EINVAL; + goto out_args; + } + error = erofs_parse_fsoffset(mp, &fsoffset, &fsoffset_present); + if (error != 0) + goto out_args; + error = erofs_parse_directio_options(mp, &directio_enabled, + &directio_present); + if (error != 0) + goto out_args; + error = erofs_parse_xattr_options(mp, true, &xattr_user_enabled); + if (error != 0) + goto out_args; + error = erofs_parse_acl_options(mp, true, &acl_enabled); + if (error != 0) + goto out_args; + fspec = NULL; + error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len); + if (error != 0 || fspec == NULL || len == 0 || + fspec[len - 1] != '\0') { + error = EINVAL; + goto out_args; + } + mp->mnt_iosize_max = MAXPHYS; + error = erofs_open_device(mp, NULL, fspec, &primary); + if (error != 0) + goto out_args; + primary.fsoff = fsoffset; + erofs_update_iosize_max(mp, &primary); + error = erofs_mountfs(&primary, mp, args, arg_count, xattr_user_enabled, + acl_enabled, directio_enabled); + if (error != 0) + erofs_release_device_info(&primary); +out_args: + erofs_free_device_args(args, arg_count); + if (error != 0) + return (error); + vfs_mountedfrom(mp, fspec); + return (erofs_statfs(mp, &mp->mnt_stat)); +} + +static int +erofs_root(struct mount *mp, int flags, struct vnode **vpp) +{ + int error; + + error = erofs_vget_raw(mp, MTOE(mp)->root_nid, flags, vpp); + if (error != 0) + vfs_mount_error(mp, + "erofs: failed to load root nid %ju: error %d", + (uintmax_t)MTOE(mp)->root_nid, error); + return (error); +} + +static int +erofs_statfs(struct mount *mp, struct statfs *sbp) +{ + struct erofs_sb_info *sbi; + + sbi = MTOE(mp); + sbp->f_bsize = sbi->block_size; + sbp->f_iosize = sbi->block_size; + sbp->f_blocks = sbi->total_blocks; + sbp->f_bfree = 0; + sbp->f_bavail = 0; + sbp->f_files = INT64_MAX; + sbp->f_ffree = sbi->inos <= (uint64_t)INT64_MAX ? + INT64_MAX - (int64_t)sbi->inos : 0; + return (0); +} + +static int +erofs_unmount(struct mount *mp, int mntflags) +{ + struct erofs_sb_info *sbi; + int error, flags; + + sbi = MTOE(mp); + if (sbi == NULL) + return (EINVAL); + flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0; + error = vflush(mp, 0, flags, curthread); + if (error != 0) + return (error); + error = erofs_sb_free(sbi); + if (error != 0) { + return (error); + } + mp->mnt_data = NULL; + return (0); +} + +/* Persistent EROFS file handle to locked vnode. */ +static int +erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + struct vnode *vp; + erofs_nid_t nid; + int error; + + *vpp = NULL; + bzero(&efid, sizeof(efid)); + memcpy(&efid, fhp, sizeof(efid)); + if (efid.len != sizeof(efid) || efid.pad != 0) + return (EINVAL); + nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo; + if (!erofs_nid_is_valid(MTOE(mp), nid)) + return (ESTALE); + error = erofs_vget_raw(mp, nid, flags, &vp); + if (error != 0) + return (error); + vi = VTOE(vp); + if (vi->mode == 0 || vi->nlink == 0 || vi->nid != nid || + vi->generation != efid.gen) { + vput(vp); + return (ESTALE); + } + *vpp = vp; + return (0); +} + +static struct vfsops erofs_vfsops = { + .vfs_fhtovp = erofs_fhtovp, + .vfs_mount = erofs_mount, + .vfs_root = erofs_root, + .vfs_statfs = erofs_statfs, + .vfs_unmount = erofs_unmount, + .vfs_vget = erofs_vget, +}; +VFS_SET(erofs_vfsops, erofs, VFCF_READONLY); +#if WITH_EROFS_POSIX_ACL +MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1); +#endif +#if WITH_EROFS_ZIP_DEFLATE +MODULE_DEPEND(erofs, zlib, 1, 1, 1); +#endif +MODULE_VERSION(erofs, 1); +#ifdef KDTRACE_HOOKS +MODULE_DEPEND(erofs, sdt, 1, 1, 1); +#endif diff --git a/sysfs.c b/sysfs.c new file mode 100644 index 0000000..94b4e50 --- /dev/null +++ b/sysfs.c @@ -0,0 +1,571 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* FreeBSD per-mount management controls for EROFS. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" +#include "internal.h" + +#define EROFS_DIR_RA_DEFAULT_BYTES (16UL * 1024) +#define EROFS_DIR_RA_HARD_MAX_BYTES (1024UL * 1024) +#define EROFS_DIR_RA_SLOTS 256UL + +enum erofs_cache_metric { + EROFS_CACHE_HITS, + EROFS_CACHE_MISSES, + EROFS_CACHE_BYPASSES, + EROFS_CACHE_EVICTIONS, + EROFS_CACHE_RECLAIMS, + EROFS_CACHE_WAITER_MERGES, + EROFS_CACHE_ENTRIES, + EROFS_CACHE_BYTES, +}; + +static struct mtx erofs_control_id_lock; +static uint64_t erofs_control_next_id = 1; + +MTX_SYSINIT(erofs_control_id, &erofs_control_id_lock, "erofs control id", + MTX_DEF); + +SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "EROFS filesystem"); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, mounts, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Mounted EROFS filesystems"); + +void +erofs_management_init(struct erofs_sb_info *sbi) +{ + + mtx_init(&sbi->control_lock, "erofs control", NULL, MTX_DEF); + sbi->dir_ra_bytes = EROFS_DIR_RA_DEFAULT_BYTES; + sbi->dir_ra_max_bytes = EROFS_DIR_RA_HARD_MAX_BYTES; + sbi->control_initialized = true; +} + +void +erofs_management_fini(struct erofs_sb_info *sbi) +{ + + if (!sbi->control_initialized) + return; + KASSERT(!sbi->sysctl_registered, + ("erofs management sysctls remain registered")); + mtx_destroy(&sbi->control_lock); + sbi->control_initialized = false; +} + +uint64_t +erofs_dir_ra_bytes(struct erofs_sb_info *sbi) +{ + uint64_t value; + + mtx_lock(&sbi->control_lock); + value = sbi->dir_ra_bytes; + mtx_unlock(&sbi->control_lock); + return (value); +} + +/* The readahead is limited by the size of the per-directory run lists. */ +static uint64_t +erofs_dir_ra_limit(const struct erofs_sb_info *sbi) +{ + + return (MIN((uint64_t)EROFS_DIR_RA_HARD_MAX_BYTES, + (uint64_t)sbi->block_size * EROFS_DIR_RA_SLOTS)); +} + +int +erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value) +{ + + if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) || + (value != 0 && value % sbi->block_size != 0)) + return (EINVAL); + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + sbi->dir_ra_bytes = value; + mtx_unlock(&sbi->control_lock); + return (0); +} + +void +erofs_record_dir_readahead(struct erofs_sb_info *sbi, unsigned int blocks) +{ + + mtx_lock(&sbi->control_lock); + ++sbi->dir_ra_requests; + sbi->dir_ra_blocks += blocks; + mtx_unlock(&sbi->control_lock); +} + +static int +erofs_parse_uint64(const char *value, size_t len, uint64_t *result) +{ + size_t i; + uint64_t parsed; + + if (value == NULL || len <= 1 || value[len - 1] != '\0' || + strnlen(value, len) != len - 1) + return (EINVAL); + parsed = 0; + for (i = 0; i < len - 1; ++i) { + if (value[i] < '0' || value[i] > '9') + return (EINVAL); + if (parsed > (UINT64_MAX - (value[i] - '0')) / 10) + return (ERANGE); + parsed = parsed * 10 + (value[i] - '0'); + } + *result = parsed; + return (0); +} + +int +erofs_parse_dir_ra_value(struct erofs_sb_info *sbi, struct vfsoptlist *opts, + uint64_t *valuep, bool *presentp) +{ + struct vfsopt *opt; + uint64_t value; + int error; + + *presentp = false; + TAILQ_FOREACH(opt, opts, link) { + if (strcmp(opt->name, "dir_ra_bytes") != 0) + continue; + error = erofs_parse_uint64(opt->value, opt->len, &value); + if (error != 0) + return (error); + if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) || + (value != 0 && value % sbi->block_size != 0)) + return (EINVAL); + *valuep = value; + *presentp = true; + return (0); + } + return (0); +} + +int +erofs_parse_dir_ra_option(struct erofs_sb_info *sbi, struct vfsoptlist *opts) +{ + uint64_t value; + bool present; + int error; + + error = erofs_parse_dir_ra_value(sbi, opts, &value, &present); + if (error != 0 || !present) + return (error); + return (erofs_set_dir_ra_bytes(sbi, value)); +} + +static int +erofs_sysctl_dir_ra(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + int error; + + sbi = arg1; + value = erofs_dir_ra_bytes(sbi); + error = sysctl_handle_64(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + return (erofs_set_dir_ra_bytes(sbi, value)); +} + +static int +erofs_sysctl_dir_ra_metric(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = arg2 == 0 ? sbi->dir_ra_requests : sbi->dir_ra_blocks; + mtx_unlock(&sbi->control_lock); + return (sysctl_handle_64(oidp, &value, 0, req)); +} + +#if WITH_EROFS_ZIP +static int +erofs_validate_zip_value(const char *value, int len) +{ + + if (value == NULL || len <= 1 || value[len - 1] != '\0' || + strnlen(value, (size_t)len) != (size_t)len - 1) + return (EINVAL); + return (0); +} + +static int +erofs_parse_zip_uint(const char *value, int len, unsigned int *result) +{ + size_t i; + unsigned int parsed; + + if (erofs_validate_zip_value(value, len) != 0) + return (EINVAL); + parsed = 0; + for (i = 0; i < (size_t)len - 1; ++i) { + if (value[i] < '0' || value[i] > '9' || + parsed > (UINT_MAX - (value[i] - '0')) / 10) + return (EINVAL); + parsed = parsed * 10 + (value[i] - '0'); + } + *result = parsed; + return (0); +} + +int +erofs_parse_zip_options(struct mount *mp, struct erofs_sb_info_opts *opts, + unsigned int *syncp, bool *cache_presentp, bool *sync_presentp) +{ + struct vfsopt *opt; + unsigned int value; + bool queue_priority_present; + int error; + + *cache_presentp = false; + *sync_presentp = false; + queue_priority_present = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "cache_strategy") == 0) { + if (opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0' || + strnlen(opt->value, opt->len) != + (size_t)opt->len - 1) + return (EINVAL); + if (strcmp(opt->value, "disabled") == 0) + opts->cache_strategy = EROFS_ZIP_CACHE_DISABLED; + else if (strcmp(opt->value, "readahead") == 0) + opts->cache_strategy = + EROFS_ZIP_CACHE_READAHEAD; + else if (strcmp(opt->value, "readaround") == 0) + opts->cache_strategy = + EROFS_ZIP_CACHE_READAROUND; + else + return (EINVAL); + *cache_presentp = true; + continue; + } + if (strcmp(opt->name, "worker_queue_priority") == 0) { + error = erofs_parse_zip_uint(opt->value, opt->len, + &value); + if (error != 0 || value > UINT8_MAX) + return (EINVAL); + opts->worker_queue_priority = (uint8_t)value; + queue_priority_present = true; + continue; + } + if (strcmp(opt->name, "sync_decompress") != 0) + continue; + error = erofs_validate_zip_value(opt->value, opt->len); + if (error != 0) + return (error); + if (strcmp(opt->value, "auto") == 0) + value = EROFS_SYNC_DECOMPRESS_AUTO; + else if (strcmp(opt->value, "on") == 0) + value = EROFS_SYNC_DECOMPRESS_FORCE_ON; + else if (strcmp(opt->value, "off") == 0) + value = EROFS_SYNC_DECOMPRESS_FORCE_OFF; + else { + error = erofs_parse_zip_uint(opt->value, opt->len, + &value); + if (error != 0 || + value > EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (EINVAL); + } + *syncp = value; + *sync_presentp = true; + } + if (*cache_presentp) + vfs_deleteopt(mp->mnt_optnew, "cache_strategy"); + if (*sync_presentp) + vfs_deleteopt(mp->mnt_optnew, "sync_decompress"); + if (queue_priority_present) + vfs_deleteopt(mp->mnt_optnew, "worker_queue_priority"); + return (0); +} + +static int +erofs_sysctl_drop_caches(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + int error; + + if (req->newptr == NULL) + return (EPERM); + sbi = arg1; + value = 0; + error = sysctl_handle_64(oidp, &value, 0, req); + if (error != 0) + return (error); + if (value != 1) + return (EINVAL); + return (z_erofs_extent_cache_drop(sbi)); +} + +static int +erofs_sysctl_cache_metric(SYSCTL_HANDLER_ARGS) +{ + struct erofs_cache_stats stats; + struct erofs_sb_info *sbi; + uint64_t value; + + sbi = arg1; + z_erofs_extent_cache_stats(sbi, &stats); + switch (arg2) { + case EROFS_CACHE_HITS: + value = stats.hits; + break; + case EROFS_CACHE_MISSES: + value = stats.misses; + break; + case EROFS_CACHE_BYPASSES: + value = stats.bypasses; + break; + case EROFS_CACHE_EVICTIONS: + value = stats.evictions; + break; + case EROFS_CACHE_RECLAIMS: + value = stats.reclaims; + break; + case EROFS_CACHE_WAITER_MERGES: + value = stats.waiter_merges; + break; + case EROFS_CACHE_ENTRIES: + value = stats.entries; + break; + case EROFS_CACHE_BYTES: + value = stats.bytes; + break; + default: + return (EINVAL); + } + return (sysctl_handle_64(oidp, &value, 0, req)); +} + +static int +erofs_sysctl_zip_setting(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + int value; + int error; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = arg2 == 0 ? (int)sbi->sync_decompress : + (int)sbi->opt.cache_strategy; + mtx_unlock(&sbi->control_lock); + error = sysctl_handle_int(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (arg2 == 0) { + if (value < EROFS_SYNC_DECOMPRESS_AUTO || + value > EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (EINVAL); + } else { + if (value < EROFS_ZIP_CACHE_DISABLED || + value > EROFS_ZIP_CACHE_READAROUND) + return (EINVAL); + } + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + if (arg2 == 0) + sbi->sync_decompress = (unsigned int)value; + else + sbi->opt.cache_strategy = (uint8_t)value; + mtx_unlock(&sbi->control_lock); + return (0); +} + +static int +erofs_sysctl_worker_queue_priority(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + int error, value; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = sbi->decompress_queue_priority; + mtx_unlock(&sbi->control_lock); + error = sysctl_handle_int(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 0 || value > UINT8_MAX) + return (EINVAL); + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + sbi->decompress_queue_priority = (uint8_t)value; + sbi->opt.worker_queue_priority = (uint8_t)value; + mtx_unlock(&sbi->control_lock); + return (0); +} +#endif + +int +erofs_register_sysfs(struct erofs_sb_info *sbi) +{ + struct sysctl_oid *cache_node, *mount_node; + char name[24]; + int error; + + mtx_lock(&erofs_control_id_lock); + sbi->control_id = erofs_control_next_id++; + mtx_unlock(&erofs_control_id_lock); + + sbi->dir_ra_max_bytes = erofs_dir_ra_limit(sbi); + + snprintf(name, sizeof(name), "%ju", (uintmax_t)sbi->control_id); + sysctl_ctx_init(&sbi->sysctl_ctx); + sbi->sysctl_registered = true; + mount_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx, + SYSCTL_STATIC_CHILDREN(_vfs_erofs_mounts), OID_AUTO, name, + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "EROFS mount"); + if (mount_node == NULL) + goto fail; + if (SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "mount_id", CTLFLAG_RD, &sbi->control_id, 0, + "Unique mount control identifier") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_bytes", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_dir_ra, "QU", + "Directory readahead window in bytes") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_max_bytes", CTLFLAG_RD, &sbi->dir_ra_max_bytes, 0, + "Maximum directory readahead window") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_requests", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_dir_ra_metric, "QU", + "Directory readahead submissions") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_blocks", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, 1, erofs_sysctl_dir_ra_metric, "QU", + "Directory readahead blocks submitted") == NULL) + goto fail; +#if WITH_EROFS_ZIP + if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "drop_caches", CTLTYPE_U64 | CTLFLAG_WR | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_drop_caches, "QU", + "Write 1 to drop the FreeBSD decoded extent cache") == NULL) + goto fail; + if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "sync_decompress", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_zip_setting, "I", + "0=auto, 1=force synchronous, 2=taskqueue fallback") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "cache_strategy", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 1, erofs_sysctl_zip_setting, "I", + "0=disabled, 1=readahead, 2=readaround") == NULL) + goto fail; + cache_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx, + SYSCTL_CHILDREN(mount_node), OID_AUTO, "decoded_cache", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, + "FreeBSD decoded extent cache, not Linux managed pclusters"); + if (cache_node == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "hits", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_HITS, erofs_sysctl_cache_metric, "QU", + "Cache hits") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "misses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_MISSES, erofs_sysctl_cache_metric, "QU", + "Cache misses") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "bypasses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_BYPASSES, erofs_sysctl_cache_metric, "QU", + "Cache admission bypasses") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "evictions", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_EVICTIONS, erofs_sysctl_cache_metric, "QU", + "Cache evictions") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "reclaims", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_RECLAIMS, erofs_sysctl_cache_metric, "QU", + "Cache pressure reclaims") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "waiter_merges", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_WAITER_MERGES, erofs_sysctl_cache_metric, "QU", + "Inflight cache waiter merges") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "entries", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_ENTRIES, erofs_sysctl_cache_metric, "QU", + "Resident decoded extents") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "bytes", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_BYTES, erofs_sysctl_cache_metric, "QU", + "Resident decoded bytes") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_submissions", CTLFLAG_RD, + &sbi->decompress_submissions, + 0, "Taskqueue decompression submissions") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_runs", CTLFLAG_RD, &sbi->decompress_worker_runs, + 0, "Taskqueue decompression worker runs") == NULL || + SYSCTL_ADD_UINT(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_count", CTLFLAG_RD, &sbi->decompress_worker_count, + 0, "Bounded decompression worker count") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "queue_fallbacks", CTLFLAG_RD, + &sbi->decompress_queue_fallbacks, 0, + "Direct decode after taskqueue fallback") == NULL || + SYSCTL_ADD_U8(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_cpu_bound", CTLFLAG_RD, + (uint8_t *)&sbi->decompress_worker_cpu_bound, 0, + "Workers restricted to the online CPU set") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_queue_priority", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sbi, 0, + erofs_sysctl_worker_queue_priority, "I", + "Taskqueue order priority; 0 is highest and 255 is lowest") == NULL) + goto fail; +#else + (void)cache_node; +#endif + return (0); + +fail: + error = erofs_unregister_sysfs(sbi); + return (error != 0 ? error : ENOMEM); +} + +int +erofs_unregister_sysfs(struct erofs_sb_info *sbi) +{ + int error; + + if (!sbi->sysctl_registered) + return (0); + mtx_lock(&sbi->control_lock); + sbi->control_closing = true; + mtx_unlock(&sbi->control_lock); + error = sysctl_ctx_free(&sbi->sysctl_ctx); + if (error != 0) { + mtx_lock(&sbi->control_lock); + sbi->control_closing = false; + mtx_unlock(&sbi->control_lock); + return (error); + } + sbi->sysctl_registered = false; + return (0); +} diff --git a/trace.c b/trace.c new file mode 100644 index 0000000..25c8685 --- /dev/null +++ b/trace.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "internal.h" +#include "trace.h" + +SDT_PROVIDER_DEFINE(erofs); + +SDT_PROBE_DEFINE4(erofs, data, map_blocks, enter, + "uint64_t", "uint64_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE6(erofs, data, map_blocks, exit, + "uint64_t", "uint64_t", "uint64_t", "uint64_t", "uint64_t", + "uint64_t"); +SDT_PROBE_DEFINE5(erofs, data, read_folio, entry, + "uint64_t", "uint64_t", "uint64_t", "int", "bool"); +SDT_PROBE_DEFINE4(erofs, data, readahead, entry, + "uint64_t", "uint64_t", "uint64_t", "bool"); +SDT_PROBE_DEFINE3(erofs, inode, fill_inode, entry, + "uint64_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE4(erofs, namei, lookup, entry, + "uint64_t", "uint32_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE3(erofs, , , corrupt, "uint64_t", "int", "int"); diff --git a/trace.h b/trace.h new file mode 100644 index 0000000..77bc6bd --- /dev/null +++ b/trace.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _FS_EROFS_TRACE_H_ +#define _FS_EROFS_TRACE_H_ + +#include + +#include "internal.h" + +SDT_PROVIDER_DECLARE(erofs); + +SDT_PROBE_DECLARE(erofs, data, map_blocks, enter); +SDT_PROBE_DECLARE(erofs, data, map_blocks, exit); +SDT_PROBE_DECLARE(erofs, data, read_folio, entry); +SDT_PROBE_DECLARE(erofs, data, readahead, entry); +SDT_PROBE_DECLARE(erofs, inode, fill_inode, entry); +SDT_PROBE_DECLARE(erofs, namei, lookup, entry); +SDT_PROBE_DECLARE(erofs, , , corrupt); + +#if WITH_EROFS_SDT +#define EROFS_TRACE_CORRUPT(nid, line, error) \ + SDT_PROBE3(erofs, , , corrupt, (nid), (line), (error)) +#else +#define EROFS_TRACE_CORRUPT(nid, line, error) do { } while (0) +#endif + +#define EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, flags) \ + SDT_PROBE4(erofs, data, map_blocks, enter, \ + (vi)->nid, (map)->m_la, (map)->m_llen, (flags)) + +#ifdef KDTRACE_HOOKS +#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \ + uint64_t erofs_trace_status; \ + if (SDT_PROBES_ENABLED()) { \ + erofs_trace_status = ((uint64_t)(uint32_t)(error) << 32) | \ + (uint64_t)(uint32_t)(map)->m_flags; \ + SDT_PROBE6(erofs, data, map_blocks, exit, (vi)->nid, \ + (map)->m_la, (map)->m_pa, (map)->m_llen, (map)->m_plen, \ + erofs_trace_status); \ + } \ +} while (0) +#else +#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \ + (void)(vi); \ + (void)(map); \ + (void)(error); \ +} while (0) +#endif + +#define EROFS_TRACE_READ_FOLIO(vi, offset, length, raw) \ + SDT_PROBE5(erofs, data, read_folio, entry, (vi)->nid, \ + (uint64_t)(offset), (uint64_t)(length), (vi)->vtype == VDIR, (raw)) + +#define EROFS_TRACE_READAHEAD(vi, start, nrpages, raw) \ + SDT_PROBE4(erofs, data, readahead, entry, (vi)->nid, \ + (uint64_t)(start), (uint64_t)(nrpages), (raw)) + +#define EROFS_TRACE_FILL_INODE(nid, blkaddr, offset) \ + SDT_PROBE3(erofs, inode, fill_inode, entry, (nid), \ + (uint64_t)(blkaddr), (uint32_t)(offset)) + +static __inline void +erofs_trace_lookup(const struct erofs_inode *dir, const char *name, + size_t length, uint32_t flags) +{ +#ifdef KDTRACE_HOOKS + uint32_t hash; + size_t i; + + if (!SDT_PROBES_ENABLED()) + return; + hash = UINT32_C(2166136261); + for (i = 0; i < length; ++i) { + hash ^= (uint8_t)name[i]; + hash *= UINT32_C(16777619); + } + SDT_PROBE4(erofs, namei, lookup, entry, dir->nid, hash, + (uint64_t)length, flags); +#else + (void)dir; + (void)name; + (void)length; + (void)flags; +#endif +} + +#endif diff --git a/xattr.c b/xattr.c new file mode 100644 index 0000000..07671e4 --- /dev/null +++ b/xattr.c @@ -0,0 +1,1151 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021-2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include + +#if WITH_EROFS_POSIX_ACL +#include +#endif + +#include + +#include "xattr.h" + +#if WITH_EROFS_POSIX_ACL +struct posix_acl_xattr_entry { + uint16_t e_tag; + uint16_t e_perm; + uint32_t e_id; +}; + +struct posix_acl_xattr_header { + uint32_t a_version; +}; + +#define POSIX_ACL_XATTR_VERSION 0x0002 +#define EROFS_XATTR_FILTER_POSIX_ACL \ + ((1U << 21) | (1U << 30)) +#endif +#define EROFS_XATTR_CACHE_BODY_LIMIT (64UL * 1024) +#define EROFS_XATTR_CACHE_MOUNT_BUDGET (1024UL * 1024) + +struct erofs_xattr_iter { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + int attrnamespace; + const char *name; + size_t name_len; + struct uio *uio; + size_t *sizep; +}; + +static int erofs_xattr_load_body(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep); + +static uint32_t +erofs_xxh32_rotl(uint32_t value, unsigned int count) +{ + return ((value << count) | (value >> (32 - count))); +} + +static uint32_t +erofs_xxh32_round(uint32_t seed, uint32_t input) +{ + seed += input * UINT32_C(2246822519); + seed = erofs_xxh32_rotl(seed, 13); + return (seed * UINT32_C(2654435761)); +} + +static uint32_t +erofs_xxh32(const void *input, size_t length, uint32_t seed) +{ + const uint8_t *cursor, *end; + uint32_t hash; + + cursor = input; + end = cursor + length; + if (length >= 16) { + const uint8_t *limit; + uint32_t v1, v2, v3, v4; + + limit = end - 16; + v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519); + v2 = seed + UINT32_C(2246822519); + v3 = seed; + v4 = seed - UINT32_C(2654435761); + do { + v1 = erofs_xxh32_round(v1, le32dec(cursor)); + cursor += 4; + v2 = erofs_xxh32_round(v2, le32dec(cursor)); + cursor += 4; + v3 = erofs_xxh32_round(v3, le32dec(cursor)); + cursor += 4; + v4 = erofs_xxh32_round(v4, le32dec(cursor)); + cursor += 4; + } while (cursor <= limit); + hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) + + erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18); + } else { + hash = seed + UINT32_C(374761393); + } + hash += (uint32_t)length; + while (cursor + 4 <= end) { + hash += le32dec(cursor) * UINT32_C(3266489917); + hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263); + cursor += 4; + } + while (cursor < end) { + hash += *cursor++ * UINT32_C(374761393); + hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761); + } + hash ^= hash >> 15; + hash *= UINT32_C(2246822519); + hash ^= hash >> 13; + hash *= UINT32_C(3266489917); + hash ^= hash >> 16; + return (hash); +} + +static bool +erofs_xattr_filter_name(int attrnamespace, const char *name, size_t name_len, + uint8_t *indexp, const char **filter_namep, size_t *filter_name_lenp) +{ +#if WITH_EROFS_POSIX_ACL + static const char acl_access[] = "posix_acl_access"; + static const char acl_default[] = "posix_acl_default"; +#endif +#if WITH_EROFS_SECURITY + static const char security[] = "security."; +#endif + static const char trusted[] = "trusted."; + + if (attrnamespace == EXTATTR_NAMESPACE_USER) { + *indexp = EROFS_XATTR_INDEX_USER; + *filter_namep = name; + *filter_name_lenp = name_len; + return (true); + } + if (attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (false); +#if WITH_EROFS_POSIX_ACL + if (name_len == sizeof(acl_access) - 1 && + memcmp(name, acl_access, sizeof(acl_access) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len == sizeof(acl_default) - 1 && + memcmp(name, acl_default, sizeof(acl_default) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } +#endif + if (name_len >= sizeof(trusted) - 1 && + memcmp(name, trusted, sizeof(trusted) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_TRUSTED; + *filter_namep = name + sizeof(trusted) - 1; + *filter_name_lenp = name_len - (sizeof(trusted) - 1); + return (true); + } +#if WITH_EROFS_SECURITY + if (name_len >= sizeof(security) - 1 && + memcmp(name, security, sizeof(security) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_SECURITY; + *filter_namep = name + sizeof(security) - 1; + *filter_name_lenp = name_len - (sizeof(security) - 1); + return (true); + } +#endif + return (false); +} + +static int +erofs_xattr_read_backing(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, erofs_off_t off, size_t len, + struct erofs_buf *buf) +{ + void *data; + int error; + + /* Both reads check the range (and report it) themselves. */ + if (backing_en != NULL) + error = erofs_read_data(sbi, backing_en, off, len, &data); + else + error = erofs_bread(sbi, off, len, &data); + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_xattr_filter_negative(struct erofs_sb_info *sbi, struct erofs_inode *vi, + int attrnamespace, const char *name, size_t name_len, bool *negativep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + const char *filter_name; + size_t filter_name_len, header_size; + uint32_t hashbit, name_filter; + uint8_t index; + int error; + + *negativep = false; + if (!erofs_sb_has_xattr_filter_v1(sbi) || + !erofs_xattr_filter_name(attrnamespace, name, name_len, &index, + &filter_name, &filter_name_len)) + return (0); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + name_filter = le32toh(ih->h_name_filter); + hashbit = erofs_xxh32(filter_name, filter_name_len, + EROFS_XATTR_FILTER_SEED + index); + hashbit &= EROFS_XATTR_FILTER_BITS - 1; + *negativep = (name_filter & (1U << hashbit)) != 0; + error = 0; + erofs_put_metabuf(&body); + return (error); +} + +/* + * Read one prefix table metadata record. + * + * When backing_en == NULL the record is in the physical metadata area; + * otherwise it lives in the selected metadata carrier's logical data stream. + */ +static int +erofs_xattr_read_metadata(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, + erofs_off_t *offp, struct erofs_buf *buf, size_t *lenp) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + uint16_t raw_len; + erofs_off_t off; + size_t len; + int error; + + /* The offset is below 2^35: 255 records of at most 64 KiB each. */ + off = roundup2(*offp, sizeof(struct erofs_xattr_entry)); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(raw_len), + &hdrbuf); + if (error != 0) + return (error); + raw_len = le16dec(hdrbuf.data); + erofs_put_metabuf(&hdrbuf); + len = (raw_len == 0) ? (size_t)UINT16_MAX + 1 : raw_len; + if (len < sizeof(struct erofs_xattr_long_prefix) || + len > EROFS_NAME_LEN + sizeof(struct erofs_xattr_long_prefix)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "xattr prefix record of %zu bytes at %ju has an invalid " + "size", + len, (uintmax_t)off)); + error = erofs_xattr_read_backing(sbi, backing_en, + off + sizeof(raw_len), len, buf); + if (error != 0) + return (error); + *offp = off + sizeof(raw_len) + len; + *lenp = len; + return (0); +} + +static int +erofs_xattr_move(void *value, size_t value_size, struct uio *uio, size_t *sizep) +{ + if (sizep != NULL) + *sizep = value_size; + if (uio == NULL || value_size == 0) + return (0); + return (uiomove(value, value_size, uio)); +} + +static int +erofs_xattr_load_body_uncached(struct erofs_sb_info *sbi, + struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + char *body; + uint64_t body_off; + size_t header_size; + int error; + + if (vi->xattr_isize < sizeof(*ih)) + return (erofs_corrupt(sbi, vi->nid, + "xattr area of %zu bytes is smaller than its header", + (size_t)vi->xattr_isize)); + body_off = vi->inode_off + vi->inode_isize; + error = erofs_xattr_read_backing(sbi, + erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL, body_off, + vi->xattr_isize, &buf); + if (error != 0) + return (error); + body = buf.data; + ih = (struct erofs_xattr_ibody_header *)body; + if (vi->xattr_isize == sizeof(*ih)) { + error = EOPNOTSUPP; + goto fail; + } + header_size = sizeof(*ih) + sizeof(uint32_t) * ih->h_shared_count; + if (header_size > vi->xattr_isize) { + error = erofs_corrupt(sbi, vi->nid, + "%u shared xattr ids do not fit the xattr area of %zu " + "bytes", + ih->h_shared_count, (size_t)vi->xattr_isize); + goto fail; + } + *bodybuf = buf; + *ihp = ih; + *header_sizep = header_size; + return (0); +fail: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_xattr_validate_entry(struct erofs_xattr_entry *entry, size_t remaining, + size_t *entry_sizep, size_t *value_sizep) +{ + size_t entry_size, value_size; + + if (remaining < sizeof(*entry)) + return (EINTEGRITY); + value_size = le16toh(entry->e_value_size); + /* The aligned size covers the header, the name and the value. */ + entry_size = erofs_xattr_entry_size(entry); + if (entry_size > remaining) + return (EINTEGRITY); + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static bool +erofs_xattr_prefix(uint8_t base_index, int *namespacep, + const char **prefixp, size_t *prefix_lenp) +{ + switch (base_index) { + case EROFS_XATTR_INDEX_USER: + *namespacep = EXTATTR_NAMESPACE_USER; + *prefixp = NULL; + *prefix_lenp = 0; + return (true); +#if WITH_EROFS_POSIX_ACL + case EROFS_XATTR_INDEX_POSIX_ACL_ACCESS: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_access"; + *prefix_lenp = sizeof("posix_acl_access") - 1; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_default"; + *prefix_lenp = sizeof("posix_acl_default") - 1; + return (true); +#endif + case EROFS_XATTR_INDEX_TRUSTED: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "trusted."; + *prefix_lenp = sizeof("trusted.") - 1; + return (true); +#if WITH_EROFS_SECURITY + case EROFS_XATTR_INDEX_SECURITY: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "security."; + *prefix_lenp = sizeof("security.") - 1; + return (true); +#endif + case EROFS_XATTR_INDEX_LUSTRE: + default: + return (false); + } +} + +static int +erofs_xattr_namespace_prefix(int attrnamespace, uint8_t base_index, + const char **prefixp, size_t *prefix_lenp) +{ + int mapped_namespace; + + if (attrnamespace != EXTATTR_NAMESPACE_USER && + attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (EOPNOTSUPP); + if (!erofs_xattr_prefix(base_index, &mapped_namespace, prefixp, + prefix_lenp)) + return (ENOATTR); + if (mapped_namespace != attrnamespace) + return (ENOATTR); + return (0); +} + +static int +erofs_xattr_list_move(const char *namespace_prefix, size_t namespace_prefix_len, + const char *infix, size_t infix_len, const char *name, uint8_t name_len, + struct uio *uio, size_t *sizep) +{ + uint8_t total_name_len; + int error; + + if (namespace_prefix_len + infix_len + name_len > EROFS_NAME_LEN) + return (EINTEGRITY); + total_name_len = namespace_prefix_len + infix_len + name_len; + if (sizep != NULL) { + *sizep += total_name_len + 1; + return (0); + } + if (uio == NULL) + return (0); + error = uiomove(__DECONST(void *, &total_name_len), 1, uio); + if (error != 0) + return (error); + if (namespace_prefix_len != 0) { + error = uiomove(__DECONST(void *, namespace_prefix), + namespace_prefix_len, uio); + if (error != 0) + return (error); + } + if (infix_len != 0) { + error = uiomove(__DECONST(void *, infix), infix_len, uio); + if (error != 0) + return (error); + } + return (uiomove(__DECONST(void *, name), name_len, uio)); +} + +static int +erofs_xattr_resolve_name(struct erofs_sb_info *sbi, + const struct erofs_xattr_entry *entry, uint8_t *base_indexp, + const char **infixp, size_t *infix_lenp) +{ + struct erofs_xattr_prefix_item *prefix; + uint8_t prefix_id; + + if ((entry->e_name_index & EROFS_XATTR_LONG_PREFIX) == 0) { + *base_indexp = entry->e_name_index; + *infixp = NULL; + *infix_lenp = 0; + return (0); + } + if (sbi->xattr_prefixes == NULL) + return (ENOATTR); + prefix_id = entry->e_name_index & EROFS_XATTR_LONG_PREFIX_MASK; + if (prefix_id >= sbi->xattr_prefix_count) + return (ENOATTR); + prefix = &sbi->xattr_prefixes[prefix_id]; + *base_indexp = prefix->base_index; + *infixp = prefix->infix; + *infix_lenp = prefix->infix_len; + return (0); +} + +static bool +erofs_xattr_name_match(const char *namespace_prefix, + size_t namespace_prefix_len, const char *infix, size_t infix_len, + const struct erofs_xattr_entry *entry, const char *name, size_t name_len) +{ + if (name_len != namespace_prefix_len + infix_len + entry->e_name_len) + return (false); + if (namespace_prefix_len != 0 && + memcmp(name, namespace_prefix, namespace_prefix_len) != 0) + return (false); + if (infix_len != 0 && + memcmp(name + namespace_prefix_len, infix, infix_len) != 0) + return (false); + return (memcmp(name + namespace_prefix_len + infix_len, entry->e_name, + entry->e_name_len) == 0); +} + +static erofs_off_t +erofs_xattr_shared_entry_offset(const struct erofs_sb_info *sbi, + uint32_t shared_id) +{ + /* A 32-bit block address and a 16-bit shift: below 2^48. */ + return (((erofs_off_t)sbi->xattr_blkaddr << sbi->blkszbits) + + (erofs_off_t)shared_id * sizeof(uint32_t)); +} + +static int +erofs_xattr_load_shared_entry(struct erofs_sb_info *sbi, uint32_t shared_id, + struct erofs_buf *entrybuf, size_t *value_sizep) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + struct erofs_inode *backing_en; + erofs_off_t off; + size_t entry_size, value_size; + int error; + + backing_en = erofs_sb_has_shared_ea_in_metabox(sbi) ? + sbi->metabox_en : NULL; + if (erofs_sb_has_shared_ea_in_metabox(sbi) && backing_en == NULL) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "shared xattrs are in the metabox, but the image has " + "none")); + + off = erofs_xattr_shared_entry_offset(sbi, shared_id); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(*entry), + &hdrbuf); + if (error != 0) + return (error); + entry = hdrbuf.data; + value_size = le16toh(entry->e_value_size); + entry_size = erofs_xattr_entry_size(entry); + + erofs_put_metabuf(&hdrbuf); + + /* + * The entry is read whole, so the name and the value are inside it by + * construction (entry_size covers both). + */ + error = erofs_xattr_read_backing(sbi, backing_en, off, entry_size, + entrybuf); + if (error != 0) + return (error); + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static int +erofs_listxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int err; + + err = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (err == ENOATTR) + return (0); + if (err != 0) + return (err); + err = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (err == ENOATTR) + return (0); + if (err != 0) + return (err); + err = erofs_xattr_list_move(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry->e_name, entry->e_name_len, it->uio, + it->sizep); + if (err == EINTEGRITY) + return (erofs_corrupt(it->sbi, it->vi->nid, + "xattr name is longer than %d bytes", EROFS_NAME_LEN)); + return (err); +} + +static int +erofs_getxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry, size_t value_size) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error != 0) + return (error); + if (!erofs_xattr_name_match(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry, it->name, it->name_len)) + return (ENOATTR); + return (erofs_xattr_move(entry->e_name + entry->e_name_len, value_size, + it->uio, it->sizep)); +} + +static int +erofs_xattr_iter_inline(struct erofs_xattr_iter *it, char *body, + size_t header_size, bool get) +{ + struct erofs_xattr_entry *entry; + char *cursor; + size_t entry_size, remaining, value_size; + int ret; + + remaining = it->vi->xattr_isize - header_size; + cursor = body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + ret = erofs_xattr_validate_entry(entry, remaining, + &entry_size, get ? &value_size : NULL); + if (ret != 0) + return (erofs_corrupt(it->sbi, it->vi->nid, + "invalid xattr entry at offset %zu of the xattr " + "area", + (size_t)(cursor - body))); + if (get) + ret = erofs_getxattr_foreach(it, entry, value_size); + else + ret = erofs_listxattr_foreach(it, entry); + if (get) { + if (ret != ENOATTR) + return (ret); + } else if (ret != 0) { + return (ret); + } + cursor += entry_size; + remaining -= entry_size; + } + return (get ? ENOATTR : 0); +} + +static int +erofs_xattr_iter_shared(struct erofs_xattr_iter *it, + struct erofs_xattr_ibody_header *ih, bool get) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + uint32_t shared_id; + size_t value_size; + int ret; + + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + ret = erofs_xattr_load_shared_entry(it->sbi, shared_id, &buf, + get ? &value_size : NULL); + if (ret != 0) + return (ret); + entry = buf.data; + if (get) + ret = erofs_getxattr_foreach(it, entry, value_size); + else + ret = erofs_listxattr_foreach(it, entry); + erofs_put_metabuf(&buf); + if (get) { + if (ret != ENOATTR) + return (ret); + } else if (ret != 0) { + return (ret); + } + } + return (get ? ENOATTR : 0); +} + +/* + * Name exposure rules: + * - user namespace: bare name, no "user." prefix; + * - system namespace: the full "trusted.*" and "security.*" names, and + * "posix_acl_access" and "posix_acl_default" for the ACL attributes. + */ +int +erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size, name_len; + bool filter_negative; + int ret; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (name == NULL || name[0] == '\0') + return (EINVAL); + name_len = strlen(name); + if (name_len > EROFS_NAME_LEN) + return (EINVAL); + if (vi->xattr_isize == 0) + return (ENOATTR); + ret = erofs_xattr_filter_negative(sbi, vi, attrnamespace, name, + name_len, &filter_negative); + if (ret != 0) + return (ret); + if (filter_negative) + return (ENOATTR); + ret = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (ret != 0) + return (ret); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = name; + it.name_len = name_len; + it.uio = uio; + it.sizep = sizep; + ret = erofs_xattr_iter_inline(&it, body.data, header_size, true); + if (ret == ENOATTR) + ret = erofs_xattr_iter_shared(&it, ih, true); + erofs_put_metabuf(&body); + return (ret); +} + +/* + * Return format: 1-byte name length followed by non-NUL-terminated name bytes. + */ +int +erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size; + int ret; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (sizep != NULL) + *sizep = 0; + if (vi->xattr_isize == 0) + return (0); + ret = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (ret != 0) + return (ret); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = NULL; + it.name_len = 0; + it.uio = uio; + it.sizep = sizep; + ret = erofs_xattr_iter_inline(&it, body.data, header_size, false); + if (ret == 0) + ret = erofs_xattr_iter_shared(&it, ih, false); + erofs_put_metabuf(&body); + return (ret); +} + +static bool +erofs_xattr_cache_reserve(struct erofs_sb_info *sbi, size_t size) +{ + u_long resident; + + for (;;) { + resident = atomic_load_acq_long(&sbi->xattr_cache_resident); + if (resident > EROFS_XATTR_CACHE_MOUNT_BUDGET || + size > EROFS_XATTR_CACHE_MOUNT_BUDGET - resident) + return (false); + if (atomic_cmpset_acq_long(&sbi->xattr_cache_resident, resident, + resident + size)) + return (true); + } +} + +static void +erofs_xattr_cache_release(struct erofs_sb_info *sbi, size_t size) +{ + KASSERT(size <= EROFS_XATTR_CACHE_MOUNT_BUDGET, + ("erofs xattr cache release exceeds mount budget")); + atomic_subtract_rel_long(&sbi->xattr_cache_resident, size); +} + +static void * +erofs_xattr_cache_invalidate_locked(struct erofs_sb_info *sbi, + struct erofs_xattr_cache *cache) +{ + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + erofs_xattr_cache_release(sbi, cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + cache->error = 0; + cache->state = EROFS_XATTR_CACHE_EMPTY; + return (data); +} + +void +erofs_xattr_cache_init(struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + + cache = &vi->xattr_cache; + bzero(cache, sizeof(*cache)); + mtx_init(&cache->lock, "erofs xattr", NULL, MTX_DEF); + cv_init(&cache->cv, "erofs xattr"); + cache->state = EROFS_XATTR_CACHE_EMPTY; + cache->initialized = true; +} + +void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + void *data; + + cache = &vi->xattr_cache; + if (!cache->initialized) + return; + mtx_lock(&cache->lock); + cache->closing = true; + while (cache->state == EROFS_XATTR_CACHE_INFLIGHT || + cache->waiters != 0) + cv_wait(&cache->cv, &cache->lock); + data = erofs_xattr_cache_invalidate_locked(sbi, cache); + cache->initialized = false; + mtx_unlock(&cache->lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&cache->lock); +} + +static int +erofs_xattr_load_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_xattr_cache *cache; + bool reserved; + int error; + + cache = &vi->xattr_cache; + if (!cache->initialized || + vi->xattr_isize > EROFS_XATTR_CACHE_BODY_LIMIT) + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep)); + mtx_lock(&cache->lock); + if (cache->closing) { + mtx_unlock(&cache->lock); + return (ENXIO); + } + if (cache->state == EROFS_XATTR_CACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs xattr ready cache has no body")); + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + mtx_unlock(&cache->lock); + return (0); + } + if (cache->state == EROFS_XATTR_CACHE_INFLIGHT) { + cache->waiters++; + do { + cv_wait(&cache->cv, &cache->lock); + } while (cache->state == EROFS_XATTR_CACHE_INFLIGHT); + if (cache->state == EROFS_XATTR_CACHE_READY) { + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + error = 0; + } else { + KASSERT(cache->state == EROFS_XATTR_CACHE_FAILED && + cache->error > 0, + ("erofs xattr failed cache has no typed error")); + error = cache->error; + } + cache->waiters--; + if (cache->state == EROFS_XATTR_CACHE_FAILED && + cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (error); + } + if (cache->state == EROFS_XATTR_CACHE_FAILED) { + if (cache->waiters != 0) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, + ihp, header_sizep)); + } + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + } + reserved = erofs_xattr_cache_reserve(sbi, vi->xattr_isize); + if (!reserved) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep)); + } + cache->state = EROFS_XATTR_CACHE_INFLIGHT; + cache->error = 0; + mtx_unlock(&cache->lock); + + error = erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep); + mtx_lock(&cache->lock); + if (error == 0 && !cache->closing) { + cache->data = bodybuf->data; + cache->charged_bytes = vi->xattr_isize; + cache->state = EROFS_XATTR_CACHE_READY; + bodybuf->release = NULL; + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (0); + } + if (error == 0) + error = ENXIO; + erofs_xattr_cache_release(sbi, vi->xattr_isize); + cache->error = error; + cache->state = EROFS_XATTR_CACHE_FAILED; + if (cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + erofs_put_metabuf(bodybuf); + return (error); +} + +void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi) +{ + if (sbi->xattr_prefixes == NULL) + return; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) + free(sbi->xattr_prefixes[i].infix, M_EROFS); + free(sbi->xattr_prefixes, M_EROFS); + sbi->xattr_prefixes = NULL; +} + +int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_long_prefix *prefix = NULL; + struct erofs_inode *prefix_en; + erofs_off_t off; + size_t infix_len, len; + int ret; + + if (!erofs_sb_has_xattr_prefixes(sbi) || sbi->xattr_prefix_count == 0) + return (0); + prefix_en = NULL; + if (!erofs_sb_has_plain_xattr_pfx(sbi)) { + if (erofs_sb_has_metabox(sbi)) { + if (sbi->metabox_en == NULL) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "xattr prefixes are in the metabox, but " + "the image has none")); + prefix_en = sbi->metabox_en; + } else if (sbi->packed_inode != NULL) { + prefix_en = sbi->packed_inode; + } + } + sbi->xattr_prefixes = malloc(sizeof(*sbi->xattr_prefixes) * + sbi->xattr_prefix_count, + M_EROFS, M_WAITOK | M_ZERO); + off = (uint64_t)sbi->xattr_prefix_start << 2; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) { + ret = erofs_xattr_read_metadata(sbi, prefix_en, &off, + &buf, &len); + if (ret != 0) + goto fail; + prefix = buf.data; + infix_len = len - sizeof(*prefix); + sbi->xattr_prefixes[i].base_index = prefix->base_index; + sbi->xattr_prefixes[i].infix_len = infix_len; + sbi->xattr_prefixes[i].infix = malloc(infix_len + 1, M_EROFS, + M_WAITOK); + memcpy(sbi->xattr_prefixes[i].infix, prefix->infix, infix_len); + sbi->xattr_prefixes[i].infix[infix_len] = '\0'; + erofs_put_metabuf(&buf); + } + return (0); +fail: + erofs_put_metabuf(&buf); + erofs_xattr_prefixes_cleanup(sbi); + return (ret); +} + +#if WITH_EROFS_POSIX_ACL +static int +erofs_inode_has_noacl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + bool *noaclp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + struct erofs_inode *backing_en; + erofs_off_t body_off; + uint32_t name_filter; + int error; + + *noaclp = false; + if (vi->xattr_isize < sizeof(*ih)) { + *noaclp = true; + return (0); + } + if (!erofs_sb_has_xattr_filter(sbi) || + sbi->xattr_filter_reserved != 0) + return (0); + body_off = vi->inode_off + vi->inode_isize; + backing_en = erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL; + error = erofs_xattr_read_backing(sbi, backing_en, body_off, sizeof(*ih), + &buf); + if (error != 0) + return (error); + ih = buf.data; + name_filter = le32toh(ih->h_name_filter); + erofs_put_metabuf(&buf); + *noaclp = (name_filter & EROFS_XATTR_FILTER_POSIX_ACL) == + EROFS_XATTR_FILTER_POSIX_ACL; + return (0); +} + +static void +erofs_acl_from_mode(mode_t mode, acl_type_t type, struct acl *aclp) +{ + if (type == ACL_TYPE_DEFAULT) { + aclp->acl_cnt = 0; + return; + } + aclp->acl_cnt = 3; + aclp->acl_entry[0].ae_tag = ACL_USER_OBJ; + aclp->acl_entry[0].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[0].ae_perm = acl_posix1e_mode_to_perm(ACL_USER_OBJ, + mode); + aclp->acl_entry[1].ae_tag = ACL_GROUP_OBJ; + aclp->acl_entry[1].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[1].ae_perm = acl_posix1e_mode_to_perm(ACL_GROUP_OBJ, + mode); + aclp->acl_entry[2].ae_tag = ACL_OTHER; + aclp->acl_entry[2].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[2].ae_perm = acl_posix1e_mode_to_perm(ACL_OTHER, mode); +} + +static int +erofs_posix_acl_from_xattr(const void *value, size_t size, mode_t mode, + acl_type_t type, struct acl *aclp) +{ + const uint8_t *buf; + struct posix_acl_xattr_header hdr; + struct posix_acl_xattr_entry entry; + uint32_t id; + int count, i; + + buf = value; + if (size < sizeof(hdr) || (size - sizeof(hdr)) % sizeof(entry) != 0) + return (EINTEGRITY); + + memcpy(&hdr, buf, sizeof(hdr)); + if (le32toh(hdr.a_version) != POSIX_ACL_XATTR_VERSION) + return (EINTEGRITY); + + count = (size - sizeof(hdr)) / sizeof(entry); + if (count > ACL_MAX_ENTRIES) + return (EINTEGRITY); + if (count == 0) { + erofs_acl_from_mode(mode, type, aclp); + return (0); + } + + aclp->acl_cnt = count; + for (i = 0; i < count; i++) { + memcpy(&entry, buf + sizeof(hdr) + i * sizeof(entry), + sizeof(entry)); + id = le32toh(entry.e_id); + aclp->acl_entry[i].ae_tag = le16toh(entry.e_tag); + aclp->acl_entry[i].ae_perm = le16toh(entry.e_perm); + aclp->acl_entry[i].ae_id = (id == UINT32_MAX) ? + ACL_UNDEFINED_ID : id; + } + /* + * Tags, their counts, permission bits and unnamed ids are the + * kernel's ACL contract: acl_posix1e_check() enforces them. Entry + * order and duplicate ids are not checked: vaccess_acl_posix1e() + * does not depend on either, and Linux accepts both. + */ + if (acl_posix1e_check(aclp) != 0) + return (EINTEGRITY); + + return (0); +} + +int +erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + const char *xattr_name; + struct uio auio; + struct iovec aiov; + uint8_t buf[sizeof(struct posix_acl_xattr_header) + + sizeof(struct posix_acl_xattr_entry) * ACL_MAX_ENTRIES]; + size_t size; + bool noacl; + int error; + + sbi = MTOE(vp->v_mount); + if (!test_opt(&sbi->opt, POSIX_ACL)) + return (EOPNOTSUPP); + + vi = VTOE(vp); + + switch (type) { + case ACL_TYPE_ACCESS: + xattr_name = "posix_acl_access"; + break; + case ACL_TYPE_DEFAULT: + if (vp->v_type != VDIR) + return (EINVAL); + xattr_name = "posix_acl_default"; + break; + default: + return (EINVAL); + } + error = erofs_inode_has_noacl(sbi, vi, &noacl); + if (error != 0) + return (error); + if (noacl) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, NULL, + &size); + if (error == ENOATTR) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + if (error != 0) + return (error); + if (size > sizeof(buf)) + return (erofs_corrupt(sbi, vi->nid, + "ACL xattr of %zu bytes is too large", size)); + + aiov.iov_base = buf; + aiov.iov_len = size; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_offset = 0; + auio.uio_resid = size; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_rw = UIO_READ; + auio.uio_td = curthread; + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, &auio, + NULL); + if (error != 0) + return (error); + if (auio.uio_resid != 0) + return (erofs_corrupt(sbi, vi->nid, + "short read of the ACL xattr")); + + error = erofs_posix_acl_from_xattr(buf, size, vi->mode, type, aclp); + if (error == EINTEGRITY) + return (erofs_corrupt(sbi, vi->nid, + "invalid ACL xattr of %zu bytes", size)); + return (error); +} +#endif diff --git a/xattr.h b/xattr.h new file mode 100644 index 0000000..d7d17a5 --- /dev/null +++ b/xattr.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef _FS_EROFS_XATTR_H_ +#define _FS_EROFS_XATTR_H_ + +#if WITH_EROFS_XATTR && WITH_EROFS_POSIX_ACL +#include +#endif + +#include "internal.h" + +#if WITH_EROFS_XATTR +int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi); +void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi); +int erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep); +int erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep); +#if WITH_EROFS_POSIX_ACL +int erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp); +#endif +void erofs_xattr_cache_init(struct erofs_inode *vi); +void erofs_xattr_cache_fini(struct erofs_sb_info *sbi, + struct erofs_inode *vi); +#else +static inline int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi __unused) +{ + return (0); +} + +static inline void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi __unused) +{ +} + +static inline void +erofs_xattr_cache_init(struct erofs_inode *vi __unused) +{ +} + +static inline void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi __unused, + struct erofs_inode *vi __unused) +{ +} +#endif +#endif diff --git a/zdata.c b/zdata.c new file mode 100644 index 0000000..6e91482 --- /dev/null +++ b/zdata.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" + +enum z_erofs_cache_lookup { + Z_EROFS_CACHE_BYPASS, + Z_EROFS_CACHE_HIT, + Z_EROFS_CACHE_OWNER, + Z_EROFS_CACHE_ERROR, +}; + +#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024) +#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024) +#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024) +#define EROFS_ZCACHE_WORKER_HARD_LIMIT EROFS_ZCACHE_MAX_ENTRIES + +static int z_erofs_cache_enabled = 1; +static unsigned long z_erofs_cache_mount_budget = + EROFS_ZCACHE_MOUNT_HARD_BUDGET; +static unsigned long z_erofs_cache_global_budget = + EROFS_ZCACHE_GLOBAL_HARD_BUDGET; +static unsigned long z_erofs_cache_minimum_decode_work = + EROFS_ZCACHE_MIN_DECODE_WORK; +static unsigned long z_erofs_cache_worker_limit = + EROFS_ZCACHE_WORKER_HARD_LIMIT; + +TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled); +TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget", + &z_erofs_cache_mount_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget", + &z_erofs_cache_global_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work", + &z_erofs_cache_minimum_decode_work); +TUNABLE_ULONG("vfs.erofs.decoded_cache.worker_limit", + &z_erofs_cache_worker_limit); + +static struct mtx z_erofs_cache_list_lock; +static struct mtx z_erofs_cache_budget_lock; +static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts = + LIST_HEAD_INITIALIZER(z_erofs_cache_mounts); +static size_t z_erofs_cache_global_charged; +static eventhandler_tag z_erofs_cache_lowmem_tag; + +MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock, + "erofs zcache list", MTX_DEF); +MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock, + "erofs zcache budget", MTX_DEF); + +static size_t +z_erofs_extent_cache_global_limit(void) +{ + + return (MIN(z_erofs_cache_global_budget, + EROFS_ZCACHE_GLOBAL_HARD_BUDGET)); +} + +static bool +z_erofs_extent_cache_reserve(size_t bytes) +{ + size_t limit; + bool reserved; + + reserved = false; + mtx_lock(&z_erofs_cache_budget_lock); + limit = z_erofs_extent_cache_global_limit(); + if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) { + z_erofs_cache_global_charged += bytes; + reserved = true; + } + mtx_unlock(&z_erofs_cache_budget_lock); + return (reserved); +} + +static void +z_erofs_extent_cache_release(size_t bytes) +{ + + mtx_lock(&z_erofs_cache_budget_lock); + KASSERT(bytes <= z_erofs_cache_global_charged, + ("erofs decoded cache budget underflow")); + z_erofs_cache_global_charged -= bytes; + mtx_unlock(&z_erofs_cache_budget_lock); +} + +static bool +z_erofs_extent_cache_match(const struct erofs_zextent_cache_entry *entry, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size) +{ + const struct erofs_zextent_key *key = &entry->key; + + return (entry->active && entry->state != EROFS_ZCACHE_EMPTY && + key->nid == vi->nid && key->decoded_size == decoded_size && + key->map.m_pa == map->m_pa && key->map.m_la == map->m_la && + key->map.m_plen == map->m_plen && key->map.m_llen == map->m_llen && + key->map.m_deviceid == map->m_deviceid && + key->map.m_algorithmformat == map->m_algorithmformat && + key->map.m_flags == map->m_flags); +} + +static void +z_erofs_extent_cache_reset_locked(struct erofs_zextent_cache *cache, + struct erofs_zextent_cache_entry *entry) +{ + + KASSERT(entry->state != EROFS_ZCACHE_INFLIGHT && entry->waiters == 0, + ("erofs cache entry reset while busy")); + if (entry->active) { + KASSERT(cache->entry_count != 0, + ("erofs cache entry underflow")); + --cache->entry_count; + } + entry->active = false; + entry->state = EROFS_ZCACHE_EMPTY; + entry->error = 0; + entry->lru = 0; + bzero(&entry->key, sizeof(entry->key)); +} + +static void * +z_erofs_extent_cache_drop_entry_locked(struct erofs_sb_info *sbi __unused, + struct erofs_zextent_cache *cache, struct erofs_zextent_cache_entry *entry, + bool evicted, bool reclaimed) +{ + struct erofs_zextent_cache_metrics *metrics; + void *data; + + KASSERT(entry->state == EROFS_ZCACHE_READY && entry->waiters == 0, + ("erofs cache drop of busy entry")); + data = entry->data; + metrics = &cache->metrics[entry->key.map.m_algorithmformat]; + KASSERT(metrics->resident_bytes >= entry->charged_bytes, + ("erofs decoded cache codec accounting mismatch")); + metrics->resident_bytes -= entry->charged_bytes; + if (evicted) + ++metrics->evictions; + if (reclaimed) + ++metrics->reclaims; + KASSERT(cache->charged_bytes >= entry->charged_bytes, + ("erofs decoded cache mount accounting underflow")); + cache->charged_bytes -= entry->charged_bytes; + z_erofs_extent_cache_release(entry->charged_bytes); + entry->data = NULL; + entry->charged_bytes = 0; + z_erofs_extent_cache_reset_locked(cache, entry); + cv_broadcast(&entry->cv); + return (data); +} + +static enum z_erofs_cache_lookup +z_erofs_extent_cache_claim(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + struct erofs_zextent_cache_entry *entry, *victim; + void *old; + enum z_erofs_cache_lookup result; + unsigned int i; + + KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS")); + KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff, + ("erofs extent cache copy exceeds decoded extent")); + *errorp = 0; + if (!sbi->z_extent_cache_initialized) + return (Z_EROFS_CACHE_BYPASS); + cache = &sbi->z_extent_cache; + KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache claim has unknown codec")); + metrics = &cache->metrics[map->m_algorithmformat]; + entry = NULL; + victim = NULL; + old = NULL; + mtx_lock(&sbi->z_extent_cache_lock); + if (cache->closing) { + ++metrics->bypasses; + goto bypass; + } + for (i = 0; i < nitems(cache->entries); ++i) { + if (z_erofs_extent_cache_match(&cache->entries[i], vi, map, + decoded_size)) { + entry = &cache->entries[i]; + break; + } + } + if (entry != NULL) { + if (entry->state == EROFS_ZCACHE_READY) { + KASSERT(entry->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)entry->data + (size_t)mapoff, len); + entry->lru = ++cache->lru_clock; + ++metrics->hits; + mtx_unlock(&sbi->z_extent_cache_lock); + return (Z_EROFS_CACHE_HIT); + } + if (entry->state == EROFS_ZCACHE_INFLIGHT) { + ++metrics->waiter_merges; + ++entry->waiters; + do { + cv_wait(&entry->cv, &sbi->z_extent_cache_lock); + } while (entry->state == EROFS_ZCACHE_INFLIGHT); + if (entry->state == EROFS_ZCACHE_READY) { + memcpy(dst, + (char *)entry->data + (size_t)mapoff, len); + entry->lru = ++cache->lru_clock; + ++metrics->hits; + result = Z_EROFS_CACHE_HIT; + } else { + KASSERT(entry->state == EROFS_ZCACHE_FAILED && + entry->error > 0, + ("erofs inflight cache has no typed " + "result")); + *errorp = entry->error; + result = Z_EROFS_CACHE_ERROR; + } + --entry->waiters; + if (entry->state == EROFS_ZCACHE_FAILED && + entry->waiters == 0) + z_erofs_extent_cache_reset_locked(cache, entry); + if (entry->waiters == 0) + cv_broadcast(&entry->cv); + mtx_unlock(&sbi->z_extent_cache_lock); + return (result); + } + if (entry->waiters != 0) { + ++metrics->bypasses; + goto bypass; + } + z_erofs_extent_cache_reset_locked(cache, entry); + } + for (i = 0; i < nitems(cache->entries); ++i) { + if (!cache->entries[i].active) { + entry = &cache->entries[i]; + break; + } + if (cache->entries[i].state == EROFS_ZCACHE_READY && + cache->entries[i].waiters == 0 && + (victim == NULL || cache->entries[i].lru < victim->lru)) + victim = &cache->entries[i]; + } + if (entry == NULL && victim != NULL) { + entry = victim; + old = z_erofs_extent_cache_drop_entry_locked(sbi, cache, entry, + true, false); + } + if (entry == NULL || decoded_size > cache->budget_bytes || + cache->charged_bytes > cache->budget_bytes - decoded_size || + !z_erofs_extent_cache_reserve(decoded_size)) { + ++metrics->bypasses; + goto bypass; + } + entry->key.map = *map; + entry->key.nid = vi->nid; + entry->key.decoded_size = decoded_size; + entry->charged_bytes = decoded_size; + entry->error = 0; + entry->state = EROFS_ZCACHE_INFLIGHT; + entry->active = true; + entry->lru = ++cache->lru_clock; + ++cache->entry_count; + cache->charged_bytes += decoded_size; + ++metrics->misses; + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_OWNER); + +bypass: + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_BYPASS); +} + +static void +z_erofs_extent_cache_complete(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, void *decoded, int error) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_entry *entry; + struct erofs_zextent_cache_metrics *metrics; + unsigned int i; + + KASSERT((error == 0) == (decoded != NULL), + ("erofs extent cache completion is untyped")); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + entry = NULL; + for (i = 0; i < nitems(cache->entries); ++i) { + if (z_erofs_extent_cache_match(&cache->entries[i], vi, map, + decoded_size)) { + entry = &cache->entries[i]; + break; + } + } + KASSERT(entry != NULL && entry->state == EROFS_ZCACHE_INFLIGHT, + ("erofs extent cache owner lost its key")); + metrics = &cache->metrics[map->m_algorithmformat]; + if (error == 0) { + KASSERT(entry->charged_bytes == decoded_size, + ("erofs decoded cache success accounting mismatch")); + entry->data = decoded; + metrics->resident_bytes += decoded_size; + entry->error = 0; + entry->state = EROFS_ZCACHE_READY; + entry->lru = ++cache->lru_clock; + } else { + KASSERT(error > 0, + ("erofs extent cache published negative errno")); + KASSERT(cache->charged_bytes >= entry->charged_bytes, + ("erofs decoded cache accounting underflow")); + cache->charged_bytes -= entry->charged_bytes; + z_erofs_extent_cache_release(entry->charged_bytes); + entry->charged_bytes = 0; + entry->error = error; + entry->state = EROFS_ZCACHE_FAILED; + if (entry->waiters == 0) + z_erofs_extent_cache_reset_locked(cache, entry); + } + cv_broadcast(&entry->cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_extent_cache_lowmem(void *arg __unused, int howto __unused) +{ + struct erofs_zextent_cache *cache; + struct erofs_sb_info *sbi; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + mtx_lock(&z_erofs_cache_list_lock); + LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) { + count = 0; + if (!mtx_trylock(&sbi->z_extent_cache_lock)) + continue; + cache = &sbi->z_extent_cache; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY && + cache->entries[i].waiters == 0) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], true, true); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + } + mtx_unlock(&z_erofs_cache_list_lock); +} + +static void +z_erofs_extent_cache_global_init(void *arg __unused) +{ + z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +z_erofs_extent_cache_global_fini(void *arg __unused) +{ + if (z_erofs_cache_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag); + KASSERT(LIST_EMPTY(&z_erofs_cache_mounts), + ("erofs decoded cache mounts remain at unload")); + KASSERT(z_erofs_cache_global_charged == 0, + ("erofs decoded cache bytes remain at unload")); +} + +SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST, + z_erofs_extent_cache_global_init, NULL); +SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST, + z_erofs_extent_cache_global_fini, NULL); + +#if WITH_EROFS_PCPU_KTHREAD +static unsigned int +z_erofs_decode_worker_count(void) +{ + unsigned long count; + + count = z_erofs_cache_worker_limit; + if (count == 0) + count = 1; + count = MIN(count, (unsigned long)EROFS_ZCACHE_WORKER_HARD_LIMIT); + count = MIN(count, (unsigned long)MAX(1, mp_ncpus)); + return ((unsigned int)count); +} +#endif + +static void +z_erofs_decode_taskqueue_init(struct erofs_sb_info *sbi __unused) +{ +#if WITH_EROFS_PCPU_KTHREAD + unsigned int worker_count; + + worker_count = z_erofs_decode_worker_count(); + sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK, + taskqueue_thread_enqueue, &sbi->decompress_taskqueue); + if (sbi->decompress_taskqueue == NULL) + return; + if (taskqueue_start_threads_cpuset(&sbi->decompress_taskqueue, + (int)worker_count, PWAIT, &all_cpus, "erofs decode") == 0) { + sbi->decompress_worker_count = worker_count; + sbi->decompress_worker_cpu_bound = true; + return; + } + taskqueue_free(sbi->decompress_taskqueue); + sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK, + taskqueue_thread_enqueue, &sbi->decompress_taskqueue); + if (sbi->decompress_taskqueue == NULL) + return; + if (taskqueue_start_threads(&sbi->decompress_taskqueue, + (int)worker_count, PWAIT, "erofs decode") != 0) { + taskqueue_free(sbi->decompress_taskqueue); + sbi->decompress_taskqueue = NULL; + return; + } + sbi->decompress_worker_count = worker_count; +#endif +} + +static struct taskqueue * +z_erofs_decode_taskqueue_acquire(struct erofs_sb_info *sbi) +{ + struct taskqueue *taskqueue; + + mtx_lock(&sbi->z_extent_cache_lock); + if (sbi->z_extent_cache.closing || + sbi->decompress_taskqueue == NULL) { + taskqueue = NULL; + } else { + taskqueue = sbi->decompress_taskqueue; + ++sbi->decompress_taskqueue_users; + } + mtx_unlock(&sbi->z_extent_cache_lock); + return (taskqueue); +} + +static void +z_erofs_decode_taskqueue_release(struct erofs_sb_info *sbi) +{ + mtx_lock(&sbi->z_extent_cache_lock); + KASSERT(sbi->decompress_taskqueue_users != 0, + ("erofs decode taskqueue user underflow")); + if (--sbi->decompress_taskqueue_users == 0) + cv_broadcast(&sbi->decompress_taskqueue_cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +void +z_erofs_extent_cache_init(struct erofs_sb_info *sbi) +{ + + bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache)); + sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget, + EROFS_ZCACHE_MOUNT_HARD_BUDGET); + sbi->z_extent_cache.minimum_decode_work = + z_erofs_cache_minimum_decode_work; + mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF); + cv_init(&sbi->decompress_taskqueue_cv, "erofs decode tq"); + for (unsigned int i = 0; i < nitems(sbi->z_extent_cache.entries); ++i) + cv_init(&sbi->z_extent_cache.entries[i].cv, "erofs zextent"); + z_erofs_decode_taskqueue_init(sbi); + sbi->z_extent_cache_initialized = true; + mtx_lock(&z_erofs_cache_list_lock); + LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); +} + +void +z_erofs_extent_cache_fini(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + struct taskqueue *taskqueue; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&z_erofs_cache_list_lock); + LIST_REMOVE(sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); + mtx_lock(&sbi->z_extent_cache_lock); + cache->closing = true; + sbi->z_extent_cache_initialized = false; + for (;;) { + bool busy = false; + unsigned int wait_index = 0; + for (i = 0; i < nitems(cache->entries); ++i) + if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT || + cache->entries[i].waiters != 0) { + busy = true; + wait_index = i; + break; + } + if (!busy) + break; + cv_wait(&cache->entries[wait_index].cv, + &sbi->z_extent_cache_lock); + } + /* + * The workers were created with &sbi->decompress_taskqueue as their + * context and read it when they first run, which can be after this + * point if the mount failed early. Keep it valid until + * taskqueue_free() has waited for every worker to exit; closing + * already keeps new dispatches away from the queue. + */ + taskqueue = sbi->decompress_taskqueue; + sbi->decompress_worker_count = 0; + sbi->decompress_worker_cpu_bound = false; + while (sbi->decompress_taskqueue_users != 0) + cv_wait(&sbi->decompress_taskqueue_cv, + &sbi->z_extent_cache_lock); + mtx_unlock(&sbi->z_extent_cache_lock); + if (taskqueue != NULL) { + taskqueue_drain_all(taskqueue); + taskqueue_free(taskqueue); + } + mtx_lock(&sbi->z_extent_cache_lock); + sbi->decompress_taskqueue = NULL; + count = 0; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], false, false); + else if (cache->entries[i].active) + z_erofs_extent_cache_reset_locked(cache, + &cache->entries[i]); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + for (i = 0; i < nitems(cache->entries); ++i) + cv_destroy(&cache->entries[i].cv); + cv_destroy(&sbi->decompress_taskqueue_cv); + mtx_destroy(&sbi->z_extent_cache_lock); +} + +int +z_erofs_extent_cache_drop(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + if (!sbi->z_extent_cache_initialized) + return (ENXIO); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + if (cache->closing) { + mtx_unlock(&sbi->z_extent_cache_lock); + return (ENXIO); + } + for (;;) { + bool busy = false; + unsigned int wait_index = 0; + for (i = 0; i < nitems(cache->entries); ++i) + if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT || + cache->entries[i].waiters != 0) { + busy = true; + wait_index = i; + break; + } + if (!busy) + break; + cv_wait(&cache->entries[wait_index].cv, + &sbi->z_extent_cache_lock); + } + count = 0; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], true, false); + else if (cache->entries[i].active) + z_erofs_extent_cache_reset_locked(cache, + &cache->entries[i]); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + return (0); +} + +void +z_erofs_extent_cache_stats(struct erofs_sb_info *sbi, + struct erofs_cache_stats *stats) +{ + struct erofs_zextent_cache *cache; + unsigned int algorithm; + + bzero(stats, sizeof(*stats)); + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + stats->hits += cache->metrics[algorithm].hits; + stats->misses += cache->metrics[algorithm].misses; + stats->bypasses += cache->metrics[algorithm].bypasses; + stats->evictions += cache->metrics[algorithm].evictions; + stats->reclaims += cache->metrics[algorithm].reclaims; + stats->waiter_merges += cache->metrics[algorithm].waiter_merges; + stats->bytes += cache->metrics[algorithm].resident_bytes; + } + stats->entries = cache->entry_count; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static bool +z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t len) +{ + const struct erofs_zextent_cache *cache; + uint64_t decode_work; + + cache = &sbi->z_extent_cache; + decode_work = map->m_plen + len; /* at most 1 MiB + 12 MiB */ + return (z_erofs_cache_enabled != 0 && + sbi->opt.cache_strategy != EROFS_ZIP_CACHE_DISABLED && + len <= MAXPHYS && + len <= cache->budget_bytes && + decode_work >= cache->minimum_decode_work && + (map->m_flags & (EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT)) == 0 && + map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX && + sbi->z_extent_cache_initialized && vi != sbi->packed_inode && + vi != sbi->metabox_en); +} + +static void +z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map) +{ + struct erofs_zextent_cache *cache; + + if (!sbi->z_extent_cache_initialized || + map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + ++cache->metrics[map->m_algorithmformat].bypasses; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_decode_length(const struct erofs_map_blocks *map, + erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial) +{ + size_t end; + + /* z_erofs_do_read() has checked that [mapoff, mapoff + want) fits. */ + EROFS_DEBUG_ASSERT(mapoff <= map->m_llen && + want <= map->m_llen - mapoff, ("erofs: bad decode range")); + end = (size_t)mapoff + want; + *partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0; + *decoded_len = *partial ? end : (size_t)map->m_llen; +} + +int +z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + const void *input; + void *compressed, *decoded; + int error; + + *bufp = NULL; + /* + * z_erofs_do_read() handles fragment and unmapped extents itself and + * gets decoded_len from z_erofs_decode_length(). + */ + EROFS_DEBUG_ASSERT((map->m_flags & + (EROFS_MAP_FRAGMENT | EROFS_MAP_MAPPED)) == EROFS_MAP_MAPPED, + ("erofs: decoding a map with flags %#x", + (unsigned int)map->m_flags)); + EROFS_DEBUG_ASSERT(decoded_len != 0 && decoded_len <= map->m_llen && + (partial || decoded_len == map->m_llen), + ("erofs: bad decoded length %zu", decoded_len)); + + if ((map->m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata_range(sbi, vi->nid, map->m_pa, + (size_t)map->m_plen, &buf); + } else { + error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa, + (size_t)map->m_plen, &compressed); + } + if (error != 0) + return (error); + input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed; + + decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO); + error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen, + decoded, decoded_len, partial); + if ((map->m_flags & EROFS_MAP_META) != 0) + erofs_put_metabuf(&buf); + else + erofs_brelse(compressed); + if (error != 0) { + free(decoded, M_EROFS); + return (error); + } + *bufp = decoded; + return (0); +} + +static void +z_erofs_decode_task_run(void *context, int pending __unused) +{ + struct erofs_decompress_task *task = context; + + task->error = z_erofs_decode_extent(task->sbi, task->vi, &task->map, + task->decoded_len, task->partial, &task->decoded); + mtx_lock(&task->lock); + atomic_add_64(&task->sbi->decompress_worker_runs, 1); + task->done = true; + cv_signal(&task->cv); + mtx_unlock(&task->lock); +} + +static uint8_t +z_erofs_decode_queue_priority(struct erofs_sb_info *sbi) +{ + uint8_t priority; + + mtx_lock(&sbi->control_lock); + priority = sbi->decompress_queue_priority; + mtx_unlock(&sbi->control_lock); + return (priority); +} + +int +z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len, + bool partial, void **bufp) +{ + struct erofs_decompress_task *task; + struct taskqueue *taskqueue; + int error; + uint8_t queue_priority; + bool queued; + + if (sbi->sync_decompress != EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + taskqueue = z_erofs_decode_taskqueue_acquire(sbi); + if (taskqueue == NULL) { + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + } + /* + * A decode worker can get here again: reading the inline input of a + * metabox inode reads the (compressed) metabox. Waiting for another + * task from inside a worker could use up every worker, so decode in + * place. + */ + if (taskqueue_member(taskqueue, curthread)) { + z_erofs_decode_taskqueue_release(sbi); + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + } + task = malloc(sizeof(*task), M_EROFS, M_WAITOK | M_ZERO); + task->sbi = sbi; + task->vi = vi; + task->map = *map; + task->decoded_len = decoded_len; + task->partial = partial; + mtx_init(&task->lock, "erofs decomp task", NULL, MTX_DEF); + cv_init(&task->cv, "erofs decomp task"); + queue_priority = z_erofs_decode_queue_priority(sbi); + TASK_INIT(&task->task, queue_priority, z_erofs_decode_task_run, task); + error = taskqueue_enqueue(taskqueue, &task->task); + queued = error == 0; + if (queued) { + atomic_add_64(&sbi->decompress_submissions, 1); + mtx_lock(&task->lock); + while (!task->done) + cv_wait(&task->cv, &task->lock); + mtx_unlock(&task->lock); + taskqueue_drain(taskqueue, &task->task); + error = task->error; + *bufp = task->decoded; + } else { + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + error = z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp); + } + cv_destroy(&task->cv); + mtx_destroy(&task->lock); + free(task, M_EROFS); + z_erofs_decode_taskqueue_release(sbi); + return (error); +} + +static int +z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, char *out) +{ + struct erofs_map_blocks map; + void *decoded, *fragment; + erofs_off_t mapoff; + size_t decoded_len, done, want; + bool cache_eligible, partial; + enum z_erofs_cache_lookup cache_lookup; + int error; + + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) + return (EINTEGRITY); + mapoff = loff + done - map.m_la; + want = MIN((size_t)(map.m_llen - mapoff), len - done); + + if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > UINT64_MAX - mapoff) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->packed_inode, + vi->z_fragmentoff + mapoff, want, &fragment); + if (error != 0) + return (error); + memcpy(out + done, fragment, want); + erofs_brelse(fragment); + } else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + z_erofs_decode_length(&map, mapoff, want, &decoded_len, + &partial); + cache_eligible = !partial && + z_erofs_extent_cache_eligible(sbi, vi, &map, + decoded_len); + cache_lookup = Z_EROFS_CACHE_BYPASS; + if (cache_eligible) { + cache_lookup = z_erofs_extent_cache_claim(sbi, + vi, &map, decoded_len, mapoff, want, + out + done, &error); + if (cache_lookup == Z_EROFS_CACHE_HIT) { + done += want; + continue; + } + if (cache_lookup == Z_EROFS_CACHE_ERROR) + return (error); + } else + z_erofs_extent_cache_record_bypass(sbi, &map); + error = z_erofs_decode_extent_dispatch(sbi, vi, &map, + decoded_len, partial, &decoded); + if (error != 0) { + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, + &map, decoded_len, NULL, error); + return (error); + } + memcpy(out + done, (char *)decoded + (size_t)mapoff, + want); + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, + decoded_len, decoded, 0); + else + free(decoded, M_EROFS); + } + done += want; + } + return (0); +} + +int +z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + char *out; + int error; + + /* erofs_read_data_impl() has checked the arguments and the range. */ + EROFS_DEBUG_ASSERT(len != 0 && loff <= vi->size && + len <= vi->size - loff, ("erofs: bad compressed read")); + *bufp = NULL; + out = malloc(len, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, loff, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); +} + +int +z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + char *buf; + size_t want; + int error; + + /* erofs_read_uio() has rejected a negative offset. */ + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + want = MIN((size_t)uio->uio_resid, + (size_t)MIN((uint64_t)MAXPHYS, + vi->size - (uint64_t)uio->uio_offset)); + buf = malloc(want, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset, + want, buf); + if (error == 0) + error = uiomove(buf, want, uio); + free(buf, M_EROFS); + if (error != 0) + return (error); + } + return (0); +} diff --git a/zmap.c b/zmap.c new file mode 100644 index 0000000..3315261 --- /dev/null +++ b/zmap.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include + +#include "internal.h" + +struct z_erofs_maprecorder { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn; + uint8_t type, headtype; + unsigned int clusterofs; + uint16_t delta[2]; + erofs_blk_t pblk, compressedblks; + erofs_off_t nextpackoff; + bool partialref; +}; + +_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag"); + +/* + * The arithmetic in this file is plain on purpose. Every value is bounded + * before it gets here, and a position that did wrap would still be range + * checked where it is read (erofs_bread_device(), erofs_read_data_impl()): + * + * - blkszbits is in [9, PAGE_SHIFT] (super.c) and z_lclusterbits is + * blkszbits + 0..15, rejected from 31 up (z_erofs_fill_inode()), so a + * cluster is at most 1 GiB and a shift by z_lclusterbits is safe. + * - vi->size is at most OFF_MAX (erofs_read_inode()) and every request has + * m_la < vi->size (z_erofs_map_blocks()), so lcn << z_lclusterbits stays + * below 2^63 + 2^46 even after adding a 16-bit delta. + * - vi->inode_off lies inside the image (erofs_nid_is_valid()), inode_isize + * is 32 or 64 and xattr_isize is below 2^19. + * - An extent table has at most 2^48 records of at most 32 bytes each. + * - erofs_blk_t is 64 bits wide (internal.h). + * + * The list is repeated in docs/validation-policy.md. + */ + +/* The map header follows the inode and its xattrs, 8-byte aligned. */ +static inline erofs_off_t +z_erofs_header_pos(const struct erofs_inode *vi) +{ + return (roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize, 8)); +} + +static int +z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len, + struct erofs_buf *buf) +{ + return (erofs_read_metadata_range(m->sbi, m->vi->nid, pos, len, buf)); +} + +static int +z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + struct z_erofs_lcluster_index *di; + erofs_off_t base, pos; + unsigned int advise; + int error; + + vi = m->vi; + base = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header) + 8; + pos = base + lcn * sizeof(*di); + m->nextpackoff = pos + sizeof(*di); + error = z_erofs_read_index(m, pos, sizeof(*di), &buf); + if (error != 0) + return (error); + + di = buf.data; + m->lcn = lcn; + advise = le16toh(di->di_advise); + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + m->delta[0] = le16toh(di->di_u.delta[0]); + if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju has a compressed block count but " + "the inode has no big pclusters", + (uintmax_t)lcn)); + } + m->compressedblks = + m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16toh(di->di_u.delta[1]); + } else { + m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0; + m->clusterofs = le16toh(di->di_clusterofs); + m->pblk = le32toh(di->di_u.blkaddr); + } + erofs_put_metabuf(&buf); + return (0); +} + +static unsigned int +decode_compactedbits(unsigned int lobits, const uint8_t *in, + unsigned int pos, uint8_t *type) +{ + uint32_t v; + unsigned int lo; + + v = le32dec(in + pos / 8) >> (pos & 7); + lo = v & ((1U << lobits) - 1); + *type = (v >> lobits) & 3; + return (lo); +} + +static unsigned int +get_compacted_la_distance(unsigned int lobits, unsigned int encodebits, + unsigned int vcnt, const uint8_t *in, int i) +{ + unsigned int lo, d1; + uint8_t type; + + d1 = 0; + do { + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + return (d1); + ++d1; + } while (++i < (int)vcnt); + + /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ + if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0) + d1 += lo - 1; + return (d1); +} + +static int +z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + erofs_off_t ebase, pos; + uint64_t compacted_2b, totalidx, original_lcn; + unsigned int compacted_4b_initial, amortizedshift; + unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize; + bool big_pcluster; + uint8_t *in, type; + int error, i; + + vi = m->vi; + /* + * A compact index entry is 16 bits (4-byte units) or 14 bits (2-byte + * units): a 2-bit type and the offset in the lcluster. Linux has the + * same limit. + */ + if (vi->z_lclusterbits > 14) + return (erofs_corrupt(m->sbi, vi->nid, + "invalid lcluster bits %u", vi->z_lclusterbits)); + ebase = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header); + totalidx = (vi->size + (1ULL << vi->z_lclusterbits) - 1) >> + vi->z_lclusterbits; + if (lcn >= totalidx) + return (EINVAL); + + original_lcn = lcn; + m->lcn = lcn; + compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; + compacted_2b = 0; + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 && + compacted_4b_initial < totalidx) + compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16); + + pos = ebase; + amortizedshift = 2; + if (lcn >= compacted_4b_initial) { + pos += compacted_4b_initial * 4; + lcn -= compacted_4b_initial; + if (lcn < compacted_2b) { + amortizedshift = 1; + } else { + pos += compacted_2b * 2; + lcn -= compacted_2b; + } + } + pos += lcn << amortizedshift; + + if (amortizedshift == 2 && vi->z_lclusterbits <= 14) + vcnt = 2; + else if (amortizedshift == 1 && vi->z_lclusterbits <= 12) + vcnt = 16; + else { + erofs_err(m->sbi, "unsupported lcluster bits %u @ nid %ju", + vi->z_lclusterbits, (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + + packsize = vcnt << amortizedshift; + bytes = pos & (packsize - 1); + pos -= bytes; + m->nextpackoff = pos + packsize; + error = z_erofs_read_index(m, pos, packsize, &buf); + if (error != 0) + return (error); + in = buf.data; + lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT)); + encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt; + i = bytes >> amortizedshift; + + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + if (lookahead) { + m->delta[1] = get_compacted_la_distance(lobits, + encodebits, vcnt, in, i); + } + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if (!big_pcluster) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju has a compressed block count but " + "the inode has no big pclusters", + (uintmax_t)original_lcn)); + } + m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + } else { + EROFS_DEBUG_ASSERT(i > 0, + ("erofs: pack of %u slots", vcnt)); + lo = decode_compactedbits(lobits, in, + encodebits * (i - 1), &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + lo = 0; + else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) + lo = 1; + m->delta[0] = lo + 1; + } + erofs_put_metabuf(&buf); + return (0); + } + + m->clusterofs = lo; + m->delta[0] = 0; + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) + i -= lo; + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + --i; + nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; + continue; + } + if (lo <= 1) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "invalid delta %u before lcn %ju", + lo, (uintmax_t)original_lcn)); + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + m->pblk = (erofs_blk_t)le32dec(in + packsize - sizeof(uint32_t)) + nblk; + erofs_put_metabuf(&buf); + m->lcn = original_lcn; + return (0); +} + +static int +z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + int err; + + EROFS_DEBUG_ASSERT(m->vi->datalayout == + EROFS_INODE_COMPRESSED_COMPACT || + m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL, + ("erofs: data layout %u is not compressed", m->vi->datalayout)); + if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) + err = z_erofs_load_compact_lcluster(m, lcn, lookahead); + else + err = z_erofs_load_full_lcluster(m, lcn); + if (err != 0) + return (err); + /* The type is a 2-bit field, so it is always below TYPE_MAX. */ + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->clusterofs >= (1U << m->vi->z_lclusterbits)) + return (erofs_corrupt(m->sbi, m->vi->nid, + "clusterofs %u of lcn %ju is out of range", m->clusterofs, + (uintmax_t)lcn)); + return (0); +} + +static int +z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + uint64_t lcn; + int err; + + while (lookback_distance != 0 && m->lcn >= lookback_distance) { + lcn = m->lcn - lookback_distance; + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + lookback_distance = m->delta[0]; + continue; + } + m->headtype = m->type; + m->map->m_la = (lcn << m->vi->z_lclusterbits) + m->clusterofs; + return (0); + } + return (erofs_corrupt(m->sbi, m->vi->nid, + "no head lcluster found looking back from lcn %ju", + (uintmax_t)m->lcn)); +} + +static int +z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + bool bigpcl1, bigpcl2; + uint64_t lcn, lcnpos; + int err; + + vi = m->vi; + bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0; + lcn = m->lcn + 1; + lcnpos = lcn << vi->z_lclusterbits; + if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || + ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || + m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || + lcnpos >= vi->size) + m->compressedblks = 1; + if (m->compressedblks == 0) { + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->delta[0] != 1) + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju: delta %u should be 1 for the compressed " + "block count", (uintmax_t)lcn, m->delta[0])); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD || + m->compressedblks == 0) + m->compressedblks = 1; + } + m->map->m_plen = m->compressedblks * m->sbi->block_size; + return (0); +} + +static int +z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn, headlcn, lend, lcnpos; + int err; + + vi = m->vi; + map = m->map; + lcn = m->lcn; + headlcn = map->m_la >> vi->z_lclusterbits; + for (;;) { + lcnpos = lcn << vi->z_lclusterbits; + if (lcnpos >= vi->size) { + EROFS_DEBUG_ASSERT(map->m_la <= vi->size, + ("erofs: extent at %ju beyond the size %ju", + (uintmax_t)map->m_la, (uintmax_t)vi->size)); + map->m_llen = vi->size - map->m_la; + return (0); + } + err = z_erofs_load_lcluster_from_disk(m, lcn, true); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (m->delta[1] == 0) + m->delta[1] = 1; + } else { + if (lcn != headlcn) + break; + m->delta[1] = 1; + } + lcn += m->delta[1]; + } + lend = (lcn << vi->z_lclusterbits) + m->clusterofs; + EROFS_DEBUG_ASSERT(lend >= map->m_la, + ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)lend, + (uintmax_t)map->m_la)); + map->m_llen = lend - map->m_la; + return (0); +} + +static inline erofs_off_t +z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz) +{ + return (roundup2(z_erofs_header_pos(vi) + + sizeof(struct z_erofs_map_header), recsz)); +} + +static inline erofs_off_t +z_erofs_extent_record_pos(erofs_off_t table_pos, unsigned int recsz, + uint64_t index) +{ + return (table_pos + index * recsz); +} + +static int +z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + int error; + + bzero(ext, sizeof(*ext)); + error = erofs_read_metadata_range(sbi, vi->nid, pos, recsz, &buf); + if (error != 0) + return (error); + memcpy(ext, buf.data, recsz); + erofs_put_metabuf(&buf); + return (0); +} + +static uint64_t +z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz) +{ + uint64_t lstart; + + lstart = le32toh(ext->lstart_lo); + if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) + lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32; + return (lstart); +} + +static int +z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + bool fragment, ztailpacking; + struct z_erofs_maprecorder m; + uint64_t initial_lcn, ofs, end; + unsigned int endoff; + int err; + + fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + ztailpacking = vi->z_idata_size != 0; + bzero(&m, sizeof(m)); + m.sbi = sbi; + m.vi = vi; + m.map = map; + + if (vi->size == 0) { + map->m_la = 0; + map->m_llen = 0; + map->m_flags = 0; + return (0); + } + ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ? + vi->size - 1 : map->m_la; + if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 && + vi->z_tailextent_headlcn == 0) { + map->m_la = 0; + map->m_llen = vi->size; + map->m_flags = EROFS_MAP_FRAGMENT; + return (0); + } + initial_lcn = ofs >> vi->z_lclusterbits; + endoff = ofs & ((1U << vi->z_lclusterbits) - 1); + err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); + if (err != 0) + return (err); + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking) + vi->z_fragmentoff = m.nextpackoff; + + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; + end = (m.lcn + 1) << vi->z_lclusterbits; + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + endoff >= m.clusterofs) { + m.headtype = m.type; + map->m_la = (m.lcn << vi->z_lclusterbits) + m.clusterofs; + if (ztailpacking && end > vi->size) + end = vi->size; + } else { + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + end = (m.lcn << vi->z_lclusterbits) + m.clusterofs; + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + m.delta[0] = 1; + } + err = z_erofs_extent_lookback(&m, m.delta[0]); + if (err != 0) + return (err); + } + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + EROFS_DEBUG_ASSERT(end >= map->m_la, + ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)end, + (uintmax_t)map->m_la)); + map->m_llen = end - map->m_la; + + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) { + vi->z_tailextent_headlcn = m.lcn; + if (fragment && + vi->datalayout == EROFS_INODE_COMPRESSED_FULL) { + /* Both are 32-bit fields of the on-disk index. */ + vi->z_fragmentoff |= m.pblk << 32; + } + } + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_fragmentoff; + map->m_plen = vi->z_idata_size; + /* Same rule as Linux: inline data does not cross a block. */ + if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen > + sbi->block_size) + return (erofs_corrupt(sbi, vi->nid, + "inline pcluster of %ju bytes crosses its block", + (uintmax_t)map->m_plen)); + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags = EROFS_MAP_FRAGMENT; + } else { + map->m_pa = m.pblk * sbi->block_size; + err = z_erofs_get_extent_compressedlen(&m); + if (err != 0) + return (err); + } + + if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { + /* + * Interlaced pclusters are block-aligned on disk. An inline + * (ztailpacking) pcluster is not, so it can only be shifted, + * whatever the inode advises. + */ + map->m_algorithmformat = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 && + (map->m_flags & EROFS_MAP_META) == 0 ? + Z_EROFS_COMPRESSION_INTERLACED : + Z_EROFS_COMPRESSION_SHIFTED; + } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { + map->m_algorithmformat = vi->z_algorithmtype[1]; + } else { + map->m_algorithmformat = vi->z_algorithmtype[0]; + } + + if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0) { + err = z_erofs_get_extent_decompressedlen(&m); + if (err == 0) + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + return (err); + } + return (0); +} + +/* + * The extent table is not checked when the inode is loaded: that its + * records are sorted and inside the file is fsck.erofs's job, and scanning + * it would read a table that grows with the file on every load. Each + * record the search visits is range checked by the read, and a table that + * is out of order only makes it return a hole or the wrong extent. + */ +static int +z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + unsigned int recsz, bmask; + uint8_t fmt; + uint64_t cluster_size, extent_idx; + erofs_off_t extent_pos, pos, table_pos; + uint64_t lend, l, r, mid, pa, la, lstart; + bool interlaced, last; + int error; + + interlaced = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0; + recsz = z_erofs_extent_recsize(vi->z_advise); + table_pos = z_erofs_extent_table_pos(vi, recsz); + pos = table_pos; + bmask = sbi->block_size - 1; + lend = vi->size; + cluster_size = 1ULL << vi->z_lclusterbits; + map->m_flags = 0; + + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { + error = erofs_read_metadata_range(sbi, vi->nid, pos, + sizeof(uint64_t), &buf); + if (error != 0) + return (error); + pa = le64dec(buf.data); + erofs_put_metabuf(&buf); + pos += sizeof(uint64_t); + lstart = 0; + extent_idx = 0; + } else { + lstart = rounddown2(map->m_la, cluster_size); + extent_idx = lstart >> vi->z_lclusterbits; + pa = EROFS_NULL_ADDR; + } + for (;;) { + EROFS_DEBUG_ASSERT(extent_idx < vi->z_extents, + ("erofs: extent %ju of %ju", (uintmax_t)extent_idx, + (uintmax_t)vi->z_extents)); + extent_pos = z_erofs_extent_record_pos(pos, recsz, + extent_idx); + error = z_erofs_read_extent(sbi, vi, extent_pos, recsz, + &ext); + if (error != 0) + return (error); + map->m_plen = le32toh(ext.plen); + if (pa != EROFS_NULL_ADDR) { + map->m_pa = pa; + pa += map->m_plen & Z_EROFS_EXTENT_PLEN_MASK; + } else { + map->m_pa = le32toh(ext.pstart_lo); + } + extent_idx++; + lstart += cluster_size; + if (lstart > map->m_la) + break; + } + last = lstart >= roundup2(lend, cluster_size); + lend = MIN(lstart, lend); + lstart -= cluster_size; + } else { + lstart = lend; + for (l = 0, r = vi->z_extents; l < r;) { + mid = l + (r - l) / 2; + extent_pos = z_erofs_extent_record_pos(table_pos, recsz, + mid); + error = z_erofs_read_extent(sbi, vi, extent_pos, + recsz, &ext); + if (error != 0) + return (error); + la = z_erofs_extent_lstart(&ext, recsz); + pa = le32toh(ext.pstart_lo) | + ((uint64_t)le32toh(ext.pstart_hi) << 32); + if (la > map->m_la) { + r = mid; + /* + * The records are not compared with each + * other, but this one must not start past the + * next one or the size: m_llen stays within the + * file. + */ + if (la > lend) + return (erofs_corrupt(sbi, vi->nid, + "extent %ju starts at %ju, past " + "the following extent or the size " + "%ju", + (uintmax_t)mid, (uintmax_t)la, + (uintmax_t)lend)); + lend = la; + } else { + l = mid + 1; + if (map->m_la == la) + r = MIN(l + 1, r); + lstart = la; + map->m_plen = le32toh(ext.plen); + map->m_pa = pa; + } + } + last = l >= vi->z_extents; + } + + if (lstart < lend) { + map->m_la = lstart; + if (last && + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + map->m_flags = EROFS_MAP_FRAGMENT; + if (recsz > offsetof(struct z_erofs_extent, + pstart_lo)) { + vi->z_fragmentoff = map->m_plen | + (map->m_pa << 32); + } else { + vi->z_fragmentoff = map->m_plen; + } + } else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) { + map->m_flags = EROFS_MAP_MAPPED; + fmt = (uint8_t)(map->m_plen >> + Z_EROFS_EXTENT_PLEN_FMT_BIT); + if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; + if (fmt != 0) { + if (fmt - 1 >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sbi, + "unsupported compression " + "algorithm %u @ nid %ju", fmt - 1, + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + map->m_algorithmformat = (uint8_t)(fmt - 1); + } else if (interlaced && + ((map->m_pa | map->m_plen) & bmask) == 0) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } + } + map->m_llen = lend - map->m_la; + return (0); +} + +int +z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_map_header *h; + struct erofs_map_blocks map; + uint64_t cluster_size, raw; + erofs_off_t pos; + unsigned int recsz; + int err; + + if (vi->z_initialized) + return (0); + pos = z_erofs_header_pos(vi); + err = erofs_read_metadata_range(sbi, vi->nid, pos, sizeof(*h), &buf); + if (err != 0) + return (err); + h = buf.data; + if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) { + if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + } + raw = le64dec(h); + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = raw ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + erofs_put_metabuf(&buf); + vi->z_initialized = true; + return (0); + } + + vi->z_advise = le16toh(h->h_advise); + vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15); + /* The lcluster size is used as a 32-bit shift count and size below. */ + if (vi->z_lclusterbits >= 31) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, vi->nid, + "lcluster bits %u are too large", vi->z_lclusterbits)); + } + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) { + recsz = z_erofs_extent_recsize(vi->z_advise); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + cluster_size = 1ULL << vi->z_lclusterbits; + vi->z_extents = roundup2(vi->size, cluster_size) >> + vi->z_lclusterbits; + } else { + vi->z_extents = le32toh(h->h_extents_lo) | + ((uint64_t)le16toh(h->h_extents_hi) << 32); + } + erofs_put_metabuf(&buf); + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + vi->z_initialized = true; + return (0); + } + vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15); + vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4); + /* + * Both HEAD formats are on-disk algorithm IDs. IDs at or above + * Z_EROFS_COMPRESSION_MAX would alias the runtime-only SHIFTED and + * INTERLACED formats, so refuse the inode instead of decoding it as + * uncompressed data. + */ + if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || + vi->z_algorithmtype[1] >= Z_EROFS_COMPRESSION_MAX) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported compression algorithm %u/%u " + "@ nid %ju", vi->z_algorithmtype[0], vi->z_algorithmtype[1], + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) + vi->z_fragmentoff = le32toh(h->h_fragmentoff); + else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0) + vi->z_idata_size = le16toh(h->h_idata_size); + erofs_put_metabuf(&buf); + + /* Same rule as Linux: big pcluster advise needs the feature flag. */ + if (!erofs_sb_has_big_pcluster(sbi) && + (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0) + return (erofs_corrupt(sbi, vi->nid, + "big pcluster advise %#x, but the image has no big " + "pclusters", + vi->z_advise)); + /* + * Same rule as Linux: compact indexes set both big-pcluster bits or + * neither. + */ + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && + (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) != + ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0))) + return (erofs_corrupt(sbi, vi->nid, + "compact inode with inconsistent big pcluster advise %#x", + vi->z_advise)); + if (vi->z_idata_size != 0 || + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + bzero(&map, sizeof(map)); + err = z_erofs_map_blocks_fo(sbi, vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (err != 0) + return (err); + } + vi->z_initialized = true; + return (0); +} + +static int +z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) { + /* + * The fragment is read from the packed inode, and that read + * checks the packed inode and the range (z_erofs_do_read(), + * erofs_read_data()). + */ + EROFS_DEBUG_ASSERT((map->m_flags & + (EROFS_MAP_MAPPED | EROFS_MAP_META)) == 0, + ("erofs: fragment map with flags %#x", + (unsigned int)map->m_flags)); + return (0); + } + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (0); + EROFS_DEBUG_ASSERT(map->m_la < vi->size, + ("erofs: extent at %ju, beyond the size %ju", + (uintmax_t)map->m_la, (uintmax_t)vi->size)); + /* + * On-disk algorithm IDs are validated where they are read, so only + * the runtime formats can reach this point. + */ + EROFS_DEBUG_ASSERT(map->m_algorithmformat < + Z_EROFS_COMPRESSION_RUNTIME_MAX, + ("erofs: unvalidated algorithm format %u", map->m_algorithmformat)); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if ((sbi->available_compr_algs & + (1U << map->m_algorithmformat)) == 0) + return (erofs_corrupt(sbi, vi->nid, + "algorithm %u is not in the image's algorithm list", + map->m_algorithmformat)); + /* + * A fully mapped compressed extent does not decode to fewer + * bytes than it occupies, unless the end of the file cuts it + * short. + */ + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && + map->m_llen < vi->size - map->m_la) + return (erofs_corrupt(sbi, vi->nid, + "extent at %ju decompresses to %ju bytes from %ju " + "compressed", (uintmax_t)map->m_la, + (uintmax_t)map->m_llen, (uintmax_t)map->m_plen)); + } else if (map->m_llen > map->m_plen) { + /* Stored data is not longer than the pcluster holding it. */ + return (erofs_corrupt(sbi, vi->nid, + "uncompressed extent at %ju has length %ju > stored %ju", + (uintmax_t)map->m_la, (uintmax_t)map->m_llen, + (uintmax_t)map->m_plen)); + } + if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || + map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE) { + erofs_err(sbi, "extent at %ju is too large (%ju bytes encoded, " + "%ju decoded) @ nid %ju", (uintmax_t)map->m_la, + (uintmax_t)map->m_plen, (uintmax_t)map->m_llen, + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + return (0); +} + +int +z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + int err; + + if (map->m_la >= vi->size) { + map->m_llen = map->m_la - vi->size + 1; + map->m_la = vi->size; + map->m_flags = 0; + return (0); + } + err = z_erofs_fill_inode(sbi, vi); + if (err == 0) { + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) + err = z_erofs_map_blocks_ext(sbi, vi, map); + else + err = z_erofs_map_blocks_fo(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + } + if (err == 0) + err = z_erofs_map_sanity_check(sbi, vi, map); + if (err != 0) + map->m_llen = 0; + return (err); +} diff --git a/zutil.c b/zutil.c new file mode 100644 index 0000000..f3e7510 --- /dev/null +++ b/zutil.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* FreeBSD stream context pool and compression runtime accounting. */ + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024) +#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024) +#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024) +#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U) +#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U) +#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U) +#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U) + +static unsigned long erofs_stream_mount_budget = + EROFS_STREAM_MOUNT_HARD_BUDGET; +static unsigned long erofs_stream_global_budget = + EROFS_STREAM_GLOBAL_HARD_BUDGET; +static int erofs_stream_mount_contexts = + EROFS_STREAM_MOUNT_HARD_CONTEXTS; +static int erofs_stream_global_contexts_limit = + EROFS_STREAM_GLOBAL_HARD_CONTEXTS; +static int erofs_stream_mount_cached = + EROFS_STREAM_MOUNT_CACHED_PER_CODEC; +static int erofs_stream_global_cached = + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC; + +TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget", + &erofs_stream_mount_budget); +TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget", + &erofs_stream_global_budget); +TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts", + &erofs_stream_mount_contexts); +TUNABLE_INT("vfs.erofs.stream_pool.global_contexts", + &erofs_stream_global_contexts_limit); +TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec", + &erofs_stream_mount_cached); +TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec", + &erofs_stream_global_cached); + +static struct mtx erofs_stream_lock; +static struct cv erofs_stream_global_cv; +static unsigned int erofs_stream_global_waiters; +static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts = + LIST_HEAD_INITIALIZER(erofs_stream_mounts); +static uma_zone_t erofs_stream_zone; +static eventhandler_tag erofs_stream_lowmem_tag; +static size_t erofs_stream_global_resident; +static unsigned int erofs_stream_global_contexts; +static unsigned int erofs_stream_global_cached_by_codec[ + Z_EROFS_COMPRESSION_MAX]; +static unsigned long erofs_stream_cached_contexts; +static unsigned long erofs_stream_borrowed_contexts; +static unsigned long erofs_stream_idle_contexts; +static unsigned long erofs_stream_creations; +static unsigned long erofs_stream_reuses; +static unsigned long erofs_stream_fallbacks; +static unsigned long erofs_stream_exhaustions; +static unsigned long erofs_stream_waits; +static unsigned long erofs_stream_destroys; +static unsigned long erofs_stream_reclaims; + +MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool", + MTX_DEF); + +SYSCTL_DECL(_vfs_erofs); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool, + CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD, + &erofs_stream_mount_budget, 0, "Effective per-mount byte budget"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD, + &erofs_stream_global_budget, 0, "Effective global byte budget"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD, + &erofs_stream_mount_contexts, 0, + "Effective per-mount, per-algorithm context limit"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit, + CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0, + "Effective global context limit"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD, + &erofs_stream_global_resident, 0, "Currently charged global bytes"); +SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD, + &erofs_stream_global_contexts, 0, "Currently allocated contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD, + &erofs_stream_cached_contexts, 0, "Currently cacheable contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD, + &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD, + &erofs_stream_idle_contexts, 0, "Currently idle contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD, + &erofs_stream_creations, 0, "Successful context creations"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD, + &erofs_stream_reuses, 0, "Idle context acquisitions"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD, + &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD, + &erofs_stream_exhaustions, 0, "Unsatisfiable limit or budget rejections"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, waits, CTLFLAG_RD, + &erofs_stream_waits, 0, "Sleeps waiting for a context or budget release"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD, + &erofs_stream_destroys, 0, "Destroyed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD, + &erofs_stream_reclaims, 0, "Low-memory idle context reclaims"); + +struct erofs_stream_alloc_header { + struct erofs_stream_ctx *ctx; + size_t bytes; +}; + +static bool +erofs_stream_mount_budget_available_locked(const struct erofs_sb_info *sbi, + size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + return (bytes <= erofs_stream_mount_budget && + sbi->stream_pool.resident_bytes <= erofs_stream_mount_budget - + bytes); +} + +static bool +erofs_stream_global_budget_available_locked(size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + return (bytes <= erofs_stream_global_budget && + erofs_stream_global_resident <= erofs_stream_global_budget - bytes); +} + +static bool +erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + if (!erofs_stream_mount_budget_available_locked(sbi, bytes) || + !erofs_stream_global_budget_available_locked(bytes)) + return (false); + pool->resident_bytes += bytes; + erofs_stream_global_resident += bytes; + return (true); +} + +/* + * Wake threads sleeping in z_erofs_stream_ctx_get() after a context or byte + * budget was released. Mount-level sleepers and the unmount drain share the + * per-mount cv; sleepers blocked on a global limit use the global cv. + */ +static void +erofs_stream_wake_locked(struct erofs_sb_info *sbi) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + if (sbi->stream_pool.waiters != 0 || sbi->stream_pool.closing) + cv_broadcast(&sbi->stream_pool.cv); + if (erofs_stream_global_waiters != 0) + cv_broadcast(&erofs_stream_global_cv); +} + +static void +erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + KASSERT(bytes <= sbi->stream_pool.resident_bytes && + bytes <= erofs_stream_global_resident, + ("erofs stream pool byte accounting underflow")); + sbi->stream_pool.resident_bytes -= bytes; + erofs_stream_global_resident -= bytes; +} + +static struct erofs_stream_ctx * +erofs_stream_take_idle_locked(struct erofs_sb_info *sbi) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx == NULL) + continue; + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && + erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + return (ctx); + } + return (NULL); +} + +/* + * Detach one idle context so that the caller can destroy it after dropping + * the lock. Idle contexts of this mount are preferred; other mounts are only + * considered when the limit being hit is a global one. + */ +static struct erofs_stream_ctx * +erofs_stream_take_reclaimable_locked(struct erofs_sb_info *sbi, bool global) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *other; + + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL || !global) + return (ctx); + LIST_FOREACH(other, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(other); + if (ctx != NULL) + break; + } + return (ctx); +} + +static void erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx); + +/* + * Charging never sleeps for budget: the caller already owns a context, and + * waiting here would hold part of the budget while waiting for the rest. Idle + * contexts are the only budget that can be returned immediately, so reclaim + * them first and fail with ENOMEM only when the budget is truly exhausted. + */ +int +z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_ctx *idle; + bool reserved; + + if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) { + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mtx_lock(&erofs_stream_lock); + for (;;) { + if (ctx->charged_bytes > SIZE_MAX - bytes || + bytes > erofs_stream_mount_budget || + bytes > erofs_stream_global_budget) { + reserved = false; + break; + } + reserved = erofs_stream_reserve_locked(ctx->sbi, bytes); + if (reserved) + break; + idle = erofs_stream_take_reclaimable_locked(ctx->sbi, + erofs_stream_mount_budget_available_locked(ctx->sbi, + bytes)); + if (idle == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(idle); + mtx_lock(&erofs_stream_lock); + } + if (reserved) + ctx->charged_bytes += bytes; + else + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + if (!reserved) { + ctx->allocation_failed = true; + return (ENOMEM); + } + return (0); +} + +void +z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + + mtx_lock(&erofs_stream_lock); + KASSERT(bytes <= ctx->charged_bytes, + ("erofs stream context byte accounting underflow")); + ctx->charged_bytes -= bytes; + erofs_stream_release_locked(ctx->sbi, bytes); + erofs_stream_wake_locked(ctx->sbi); + mtx_unlock(&erofs_stream_lock); +} + +void * +z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_alloc_header *header; + size_t total; + + if (__builtin_add_overflow(bytes, sizeof(*header), &total) || + z_erofs_stream_ctx_charge(ctx, total) != 0) + return (NULL); + header = malloc(total, M_EROFS, M_WAITOK | M_ZERO); + header->ctx = ctx; + header->bytes = total; + return (header + 1); +} + +void +z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address) +{ + struct erofs_stream_alloc_header *header; + size_t bytes; + + if (address == NULL) + return; + header = (struct erofs_stream_alloc_header *)address - 1; + KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch")); + bytes = header->bytes; + free(header, M_EROFS); + z_erofs_stream_ctx_uncharge(ctx, bytes); +} + +static void +erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_sb_info *sbi; + bool cached; + uint8_t algorithm; + + sbi = ctx->sbi; + algorithm = ctx->algorithm; + cached = ctx->cached; + ctx->fini(ctx); + KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE, + ("erofs stream backend allocation leaked")); + uma_zfree(erofs_stream_zone, ctx); + + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + KASSERT(codec->contexts != 0 && codec->borrowed != 0 && + erofs_stream_global_contexts != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream context accounting underflow")); + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + KASSERT(codec->cached != 0 && + erofs_stream_global_cached_by_codec[algorithm] != 0 && + erofs_stream_cached_contexts != 0, + ("erofs stream cache accounting underflow")); + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_destroys; + erofs_stream_wake_locked(sbi); + mtx_unlock(&erofs_stream_lock); +} + +/* + * Sleep until a context or budget is released. Limits that only this mount + * can release are waited for on the mount cv; anything involving a global + * limit is waited for on the global cv, which every release also signals. + */ +static void +erofs_stream_wait_locked(struct erofs_sb_info *sbi, bool global) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + ++pool->waiters; + ++erofs_stream_waits; + if (global) { + ++erofs_stream_global_waiters; + cv_wait(&erofs_stream_global_cv, &erofs_stream_lock); + --erofs_stream_global_waiters; + } else { + cv_wait(&pool->cv, &erofs_stream_lock); + } + KASSERT(pool->waiters != 0, ("erofs stream waiter underflow")); + --pool->waiters; + if (pool->closing) + cv_broadcast(&pool->cv); +} + +int +z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + bool cached, global_full, mount_full, reserved __diagused; + int error; + + *ctxp = NULL; + if (algorithm == Z_EROFS_COMPRESSION_LZ4 || + algorithm >= Z_EROFS_COMPRESSION_MAX || + context_size < sizeof(*ctx) || + context_size > EROFS_STREAM_CTX_WRAPPER_SIZE) + return (EINVAL); + mtx_lock(&erofs_stream_lock); + for (;;) { + if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) { + mtx_unlock(&erofs_stream_lock); + return (ENXIO); + } + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx != NULL) { + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && + erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + ++erofs_stream_reuses; + ctx->allocation_failed = false; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); + } + /* + * Limits that no release can ever satisfy are configuration + * errors; waiting for them would never end. + */ + if (erofs_stream_mount_contexts == 0 || + erofs_stream_global_contexts_limit == 0 || + EROFS_STREAM_CTX_WRAPPER_SIZE > erofs_stream_mount_budget || + EROFS_STREAM_CTX_WRAPPER_SIZE > + erofs_stream_global_budget) { + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mount_full = codec->contexts >= + (unsigned int)erofs_stream_mount_contexts; + global_full = erofs_stream_global_contexts >= + (unsigned int)erofs_stream_global_contexts_limit || + !erofs_stream_global_budget_available_locked( + EROFS_STREAM_CTX_WRAPPER_SIZE); + if (!mount_full && !global_full && + erofs_stream_mount_budget_available_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE)) + break; + /* + * Contexts of this codec are all borrowed when mount_full, so + * only a release can help. Otherwise idle contexts of other + * codecs (or mounts, for a global limit) hold the resource and + * are never released unless reclaimed here. + */ + if (!mount_full) { + ctx = erofs_stream_take_reclaimable_locked(sbi, + global_full); + if (ctx != NULL) { + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + continue; + } + } + erofs_stream_wait_locked(sbi, !mount_full && global_full); + } + reserved = erofs_stream_reserve_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE); + KASSERT(reserved, ("erofs stream wrapper reservation failed")); + cached = codec->cached < (unsigned int)erofs_stream_mount_cached && + erofs_stream_global_cached_by_codec[algorithm] < + (unsigned int)erofs_stream_global_cached; + ++codec->contexts; + ++codec->borrowed; + ++erofs_stream_global_contexts; + ++erofs_stream_borrowed_contexts; + if (cached) { + ++codec->cached; + ++erofs_stream_global_cached_by_codec[algorithm]; + ++erofs_stream_cached_contexts; + } else { + ++erofs_stream_fallbacks; + } + mtx_unlock(&erofs_stream_lock); + + ctx = uma_zalloc(erofs_stream_zone, M_WAITOK | M_ZERO); + ctx->sbi = sbi; + ctx->fini = fini; + ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE; + ctx->algorithm = algorithm; + ctx->cached = cached; + error = init(ctx); + if (error != 0) { + erofs_stream_ctx_destroy(ctx); + return (error > 0 ? error : EIO); + } + mtx_lock(&erofs_stream_lock); + ++erofs_stream_creations; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); +} + +void +z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable) +{ + struct erofs_stream_pool_codec *codec; + + mtx_lock(&erofs_stream_lock); + codec = &ctx->sbi->stream_pool.codec[ctx->algorithm]; + if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) { + KASSERT(codec->borrowed != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream borrowed accounting underflow")); + --codec->borrowed; + --erofs_stream_borrowed_contexts; + STAILQ_INSERT_HEAD(&codec->idle, ctx, link); + ++codec->idle_count; + ++erofs_stream_idle_contexts; + erofs_stream_wake_locked(ctx->sbi); + mtx_unlock(&erofs_stream_lock); + return; + } + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); +} + +static bool +erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool) +{ + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + if (pool->codec[algorithm].contexts != 0) + return (false); + } + return (true); +} + +void +z_erofs_stream_pool_init(struct erofs_sb_info *sbi) +{ + unsigned int algorithm; + + bzero(&sbi->stream_pool, sizeof(sbi->stream_pool)); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) + STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle); + cv_init(&sbi->stream_pool.cv, "erofs stream drain"); + mtx_lock(&erofs_stream_lock); + sbi->stream_pool_initialized = true; + LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link); + mtx_unlock(&erofs_stream_lock); +} + +void +z_erofs_stream_pool_fini(struct erofs_sb_info *sbi) +{ + struct erofs_stream_ctx *ctx; + + if (!sbi->stream_pool_initialized) + return; + mtx_lock(&erofs_stream_lock); + sbi->stream_pool.closing = true; + erofs_stream_wake_locked(sbi); + for (;;) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + } + while (!erofs_stream_pool_empty_locked(&sbi->stream_pool) || + sbi->stream_pool.waiters != 0) + cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock); + KASSERT(sbi->stream_pool.resident_bytes == 0, + ("erofs stream mount bytes remain at unmount")); + LIST_REMOVE(sbi, stream_pool_link); + sbi->stream_pool_initialized = false; + mtx_unlock(&erofs_stream_lock); + cv_destroy(&sbi->stream_pool.cv); +} + +static void +erofs_stream_pool_lowmem(void *arg __unused, int howto __unused) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *sbi; + + for (;;) { + ctx = NULL; + mtx_lock(&erofs_stream_lock); + LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL) { + ++erofs_stream_reclaims; + break; + } + } + mtx_unlock(&erofs_stream_lock); + if (ctx == NULL) + break; + erofs_stream_ctx_destroy(ctx); + } +} + +static void +erofs_stream_pool_global_init(void *arg __unused) +{ + erofs_stream_mount_budget = MIN(erofs_stream_mount_budget, + EROFS_STREAM_MOUNT_HARD_BUDGET); + erofs_stream_global_budget = MIN(erofs_stream_global_budget, + EROFS_STREAM_GLOBAL_HARD_BUDGET); + erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts, + (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS)); + erofs_stream_global_contexts_limit = MAX(0, MIN( + erofs_stream_global_contexts_limit, + (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS)); + erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached, + (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC)); + erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached, + (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC)); + cv_init(&erofs_stream_global_cv, "erofs stream wait"); + /* + * The context count is bounded by erofs_stream_global_contexts_limit; + * a UMA item limit would only add a second, unrelated place to sleep. + */ + erofs_stream_zone = uma_zcreate("erofs stream ctx", + EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + KASSERT(erofs_stream_zone != NULL, + ("cannot create erofs stream UMA zone")); + uma_zone_set_maxcache(erofs_stream_zone, + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3); + erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +erofs_stream_pool_global_fini(void *arg __unused) +{ + if (erofs_stream_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag); + KASSERT(LIST_EMPTY(&erofs_stream_mounts), + ("erofs stream mounts remain at unload")); + KASSERT(erofs_stream_global_contexts == 0 && + erofs_stream_global_waiters == 0 && + erofs_stream_global_resident == 0 && + erofs_stream_borrowed_contexts == 0 && + erofs_stream_idle_contexts == 0 && + erofs_stream_cached_contexts == 0, + ("erofs stream resources remain at unload")); + uma_zdestroy(erofs_stream_zone); + cv_destroy(&erofs_stream_global_cv); +} + +SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST, + erofs_stream_pool_global_init, NULL); +SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST, + erofs_stream_pool_global_fini, NULL);