From 144eec51826d4c987de0d0cd54d4bc6130e1d4f5 Mon Sep 17 00:00:00 2001 From: Ruicheng Pan Date: Mon, 19 Oct 2026 08:35:59 +0800 Subject: [PATCH] update --- .clang-format | 199 +++++ .gitignore | 18 + Makefile | 146 ++++ compress.h | 31 + data.c | 878 +++++++++++++++++++++ decompressor.c | 212 ++++++ decompressor_deflate.c | 174 +++++ decompressor_lz4.c | 111 +++ decompressor_lzma.c | 182 +++++ decompressor_zstd.c | 172 +++++ dir.c | 387 ++++++++++ erofs_compat.h | 46 ++ erofs_fs.h | 467 ++++++++++++ erofs_vnops.c | 681 +++++++++++++++++ fileio.c | 55 ++ inode.c | 401 ++++++++++ internal.h | 721 ++++++++++++++++++ namei.c | 383 ++++++++++ super.c | 1643 ++++++++++++++++++++++++++++++++++++++++ sysfs.c | 571 ++++++++++++++ trace.c | 21 + trace.h | 87 +++ xattr.c | 1151 ++++++++++++++++++++++++++++ xattr.h | 51 ++ zdata.c | 937 +++++++++++++++++++++++ zmap.c | 949 +++++++++++++++++++++++ zutil.c | 665 ++++++++++++++++ 27 files changed, 11339 insertions(+) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 compress.h create mode 100644 data.c create mode 100644 decompressor.c create mode 100644 decompressor_deflate.c create mode 100644 decompressor_lz4.c create mode 100644 decompressor_lzma.c create mode 100644 decompressor_zstd.c create mode 100644 dir.c create mode 100644 erofs_compat.h create mode 100644 erofs_fs.h create mode 100644 erofs_vnops.c create mode 100644 fileio.c create mode 100644 inode.c create mode 100644 internal.h create mode 100644 namei.c create mode 100644 super.c create mode 100644 sysfs.c create mode 100644 trace.c create mode 100644 trace.h create mode 100644 xattr.c create mode 100644 xattr.h create mode 100644 zdata.c create mode 100644 zmap.c create mode 100644 zutil.c diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..3d436fa --- /dev/null +++ b/.clang-format @@ -0,0 +1,199 @@ +# Basic .clang-format +--- +BasedOnStyle: WebKit +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: AcrossEmptyLines +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: InlineOnly +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: TopLevelDefinitions +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeTernaryOperators: false +# TODO: BreakStringLiterals can cause very strange formatting so turn it off? +BreakStringLiterals: false +# Prefer: +# some_var = function(arg1, +# arg2) +# over: +# some_var = +# function(arg1, arg2) +PenaltyBreakAssignment: 100 +# Prefer: +# some_long_function(arg1, arg2 +# arg3) +# over: +# some_long_function( +# arg1, arg2, arg3) +PenaltyBreakBeforeFirstCallParameter: 100 +CompactNamespaces: true +DerivePointerAlignment: false +DisableFormat: false +ForEachMacros: + - ARB_ARRFOREACH + - ARB_ARRFOREACH_REVWCOND + - ARB_ARRFOREACH_REVERSE + - ARB_FOREACH + - ARB_FOREACH_FROM + - ARB_FOREACH_SAFE + - ARB_FOREACH_REVERSE + - ARB_FOREACH_REVERSE_FROM + - ARB_FOREACH_REVERSE_SAFE + - BIT_FOREACH_ISCLR + - BIT_FOREACH_ISSET + - CPU_FOREACH + - CPU_FOREACH_ISCLR + - CPU_FOREACH_ISSET + - FOREACH_THREAD_IN_PROC + - FOREACH_PROC_IN_SYSTEM + - FOREACH_PRISON_CHILD + - FOREACH_PRISON_DESCENDANT + - FOREACH_PRISON_DESCENDANT_LOCKED + - FOREACH_PRISON_DESCENDANT_LOCKED_LEVEL + - MNT_VNODE_FOREACH_ALL + - MNT_VNODE_FOREACH_ACTIVE + - RB_FOREACH + - RB_FOREACH_FROM + - RB_FOREACH_SAFE + - RB_FOREACH_REVERSE + - RB_FOREACH_REVERSE_FROM + - RB_FOREACH_REVERSE_SAFE + - SLIST_FOREACH + - SLIST_FOREACH_FROM + - SLIST_FOREACH_FROM_SAFE + - SLIST_FOREACH_SAFE + - SLIST_FOREACH_PREVPTR + - SPLAY_FOREACH + - LIST_FOREACH + - LIST_FOREACH_FROM + - LIST_FOREACH_FROM_SAFE + - LIST_FOREACH_SAFE + - STAILQ_FOREACH + - STAILQ_FOREACH_FROM + - STAILQ_FOREACH_FROM_SAFE + - STAILQ_FOREACH_SAFE + - TAILQ_FOREACH + - TAILQ_FOREACH_FROM + - TAILQ_FOREACH_FROM_SAFE + - TAILQ_FOREACH_REVERSE + - TAILQ_FOREACH_REVERSE_FROM + - TAILQ_FOREACH_REVERSE_FROM_SAFE + - TAILQ_FOREACH_REVERSE_SAFE + - TAILQ_FOREACH_SAFE + - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH +SpaceBeforeParens: ControlStatementsExceptForEachMacros +IndentCaseLabels: false +IndentPPDirectives: None +Language: Cpp +NamespaceIndentation: None +PointerAlignment: Right +ContinuationIndentWidth: 4 +IndentWidth: 8 +TabWidth: 8 +ColumnLimit: 80 +UseTab: Always +SpaceAfterCStyleCast: false +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^\"opt_.*\.h\"' + Priority: 1 + SortPriority: 10 + - Regex: '^' + Priority: 2 + SortPriority: 20 + - Regex: '^' + Priority: 2 + SortPriority: 21 + - Regex: '^' + Priority: 2 + SortPriority: 22 + - Regex: '^' + Priority: 2 + SortPriority: 23 + - Regex: '^' + Priority: 3 + SortPriority: 30 + - Regex: '^ diff --git a/compress.h b/compress.h new file mode 100644 index 0000000..6e9b117 --- /dev/null +++ b/compress.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _FS_EROFS_COMPRESS_H_ +#define _FS_EROFS_COMPRESS_H_ + +#include "internal.h" + +struct z_erofs_decompress_req { + struct erofs_sb_info *sbi; + const struct erofs_map_blocks *map; + const void *in; + size_t inputsize; + void *out; + size_t outputsize; + bool partial_decoding; +}; + +struct z_erofs_decompressor { + /* Callbacks return zero or a positive FreeBSD errno. */ + int (*config)(struct erofs_sb_info *, const struct erofs_super_block *, + const void *, size_t); + int (*decompress)(const struct z_erofs_decompress_req *); +}; + +int z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq); +extern const struct z_erofs_decompressor z_erofs_lz4_decomp; +extern const struct z_erofs_decompressor z_erofs_lzma_decomp; +extern const struct z_erofs_decompressor z_erofs_deflate_decomp; +extern const struct z_erofs_decompressor z_erofs_zstd_decomp; + +#endif diff --git a/data.c b/data.c new file mode 100644 index 0000000..5e2e5fb --- /dev/null +++ b/data.c @@ -0,0 +1,878 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "internal.h" +#include "trace.h" + +#define EROFS_DIR_READAHEAD_SLOTS 256 + +void +erofs_put_metabuf(struct erofs_buf *buf) +{ + void (*release)(void *); + void *data; + + if (buf == NULL) + return; + data = buf->data; + release = buf->release; + buf->data = NULL; + buf->release = NULL; + if (data != NULL && release != NULL) + release(data); +} + +/* + * bread() on a device vnode needs no vnode lock, but breadn() reads ahead + * through breada(), whose inmem() asserts that the vnode is locked. Take the + * lock shared (the mount's device vnode allows that) only around the request. + * + * A doomed vnode has become VBAD, which the buffer cache only accepts when it + * is locked, and it no longer has a device behind it, so fail the read. + */ +static int +erofs_breadn_devvp(struct vnode *devvp, daddr_t blkno, int size, + daddr_t *rablkno, int *rabsize, int racnt, struct buf **bpp) +{ + int error; + + vn_lock(devvp, LK_SHARED | LK_RETRY); + if (VN_IS_DOOMED(devvp)) { + VOP_UNLOCK(devvp); + *bpp = NULL; + return (ENXIO); + } + error = breadn(devvp, blkno, size, rablkno, rabsize, racnt, NOCRED, + bpp); + VOP_UNLOCK(devvp); + return (error); +} + +static int +erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno, + int *rabsize, int racnt, void **bufp) +{ + struct buf *bp; + erofs_off_t device_end, device_off, end, limit; + off_t blkoff, current; + size_t blklen, done, iosize; + char *out; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + if (!erofs_source_is_open(dif)) + return (ENODEV); + if (__builtin_add_overflow(off, (uint64_t)len, &end)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "read of %zu bytes at %ju overflows", len, (uintmax_t)off)); + if (blocks != 0) { + if (blocks > (UINT64_MAX >> sbi->blkszbits)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "device size of %ju blocks overflows", + (uintmax_t)blocks)); + limit = blocks << sbi->blkszbits; + if (end > limit) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "read of %zu bytes at %ju is beyond the device " + "(%ju bytes)", len, (uintmax_t)off, + (uintmax_t)limit)); + } + if (__builtin_add_overflow(dif->fsoff, off, &device_off) || + __builtin_add_overflow(device_off, (uint64_t)len, &device_end)) + return (EOVERFLOW); + if (device_end > dif->mediasize) + return (ENXIO); + if (device_off > INT64_MAX || device_end > (uint64_t)INT64_MAX + 1) + return (EOVERFLOW); + + iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize; + if (iosize == 0 || (iosize & (iosize - 1)) != 0) + return (EINVAL); + out = malloc(len, M_EROFS, M_WAITOK); + if (erofs_source_is_file(dif)) { + error = erofs_read_file_source(sbi, dif, device_off, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); + } + done = 0; + while (done < len) { + current = (off_t)(device_off + done); + blkoff = rounddown2(current, (off_t)iosize); + blklen = MIN(iosize - (size_t)(current - blkoff), len - done); + if (done == 0 && racnt != 0) + error = erofs_breadn_devvp(dif->devvp, btodb(blkoff), + iosize, rablkno, rabsize, racnt, &bp); + else + error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, + &bp); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (bp->b_data == NULL) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + if (bp->b_resid < 0 || bp->b_bcount < 0 || + (uintmax_t)bp->b_resid > (uintmax_t)bp->b_bcount || + (uintmax_t)(current - blkoff) + (uintmax_t)blklen > + (uintmax_t)bp->b_bcount - (uintmax_t)bp->b_resid) { + brelse(bp); + free(out, M_EROFS); + return (EIO); + } + memcpy(out + done, (char *)bp->b_data + (current - blkoff), + blklen); + brelse(bp); + done += blklen; + } + *bufp = out; + return (0); +} + +int +erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp) +{ + return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len, + NULL, NULL, 0, bufp)); +} + +void +erofs_brelse(void *buf) +{ + free(buf, M_EROFS); +} + +/* Read inode metadata from either the primary image or the metabox file. */ +int +erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf) +{ + void *data; + int error; + + if (!erofs_nid_in_metabox(nid)) { + if (off > INT64_MAX) + return (EOVERFLOW); + error = erofs_bread(sbi, (off_t)off, len, &data); + } else { + if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL) + return (erofs_corrupt(sbi, nid, + "metabox nid, but the image has no metabox")); + error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data); + } + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_inode_chunk_index *idx; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t chunk_idx, chunk_size, entry_size; + erofs_off_t loff, chunk_start, idx_off, idx_base; + uint64_t image_size, addrmask; + uint64_t blkaddr; + uint16_t raw_device_id; + int error; + + loff = map->m_la; + chunk_size = 1ULL << vi->chunkbits; + chunk_idx = loff >> vi->chunkbits; + chunk_start = chunk_idx << vi->chunkbits; + + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) + entry_size = sizeof(struct erofs_inode_chunk_index); + else + entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE; + /* + * inode_off is below 2^63 (the image fits its provider, whose size is + * an off_t; a metabox is at most OFF_MAX), isize is 32 or 64 and + * xattr_isize is below 2^19; chunk_idx is below 2^54 because + * loff < size <= OFF_MAX and a chunk has at least 512 bytes. + */ + idx_base = roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize, + entry_size); + idx_off = idx_base + chunk_idx * entry_size; + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || + idx_off > sbi->metabox_en->size || + entry_size > sbi->metabox_en->size - idx_off) + return (erofs_corrupt(sbi, vi->nid, + "chunk %ju: index entry at %ju is beyond the " + "metabox", + (uintmax_t)chunk_idx, (uintmax_t)idx_off)); + } else { + image_size = sbi->blocks << sbi->blkszbits; + if (idx_off > image_size || entry_size > image_size - idx_off) + return (erofs_corrupt(sbi, vi->nid, + "chunk %ju: index entry at %ju is beyond the image", + (uintmax_t)chunk_idx, (uintmax_t)idx_off)); + } + + error = erofs_read_metadata_range(sbi, vi->nid, idx_off, entry_size, + &buf); + if (error != 0) + return (error); + + idx = buf.data; + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) { + blkaddr = le32toh(idx->startblk_lo); + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0) + blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32; + raw_device_id = le16toh(idx->device_id); + addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ? + ((1ULL << 48) - 1) : UINT32_MAX; + } else { + blkaddr = le32dec(idx); + raw_device_id = 0; + addrmask = UINT32_MAX; + } + erofs_put_metabuf(&buf); + + if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) { + map->m_la = chunk_start; + map->m_pa = 0; + map->m_llen = MIN(chunk_size, vi->size - chunk_start); + map->m_plen = map->m_llen; + return (0); + } + + map->m_la = chunk_start; + map->m_llen = MIN(chunk_size, vi->size - chunk_start); + map->m_plen = map->m_llen; + map->m_deviceid = raw_device_id & sbi->device_id_mask; + /* blkaddr has at most 48 bits and blkszbits at most 16. */ + map->m_pa = blkaddr << sbi->blkszbits; + map->m_flags |= EROFS_MAP_MAPPED; + return (0); +} + +static erofs_off_t +erofs_inline_tail_start(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + if (vi->size == 0) + return (0); + return (roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size); +} + +static int +erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + erofs_off_t loff, tail_start; + uint64_t remain, block_rem; + + loff = map->m_la; + if (loff >= vi->size) + return (0); + + remain = vi->size - loff; + switch (vi->datalayout) { + case EROFS_INODE_CHUNK_BASED: + return (erofs_map_blocks_chunk(sbi, vi, map)); + case EROFS_INODE_FLAT_PLAIN: + map->m_llen = remain; + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff, + &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "start block %ju overflows", + (uintmax_t)vi->startblk)); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + case EROFS_INODE_FLAT_INLINE: + tail_start = erofs_inline_tail_start(sbi, vi); + if (loff < tail_start) { + map->m_llen = MIN(remain, tail_start - loff); + map->m_plen = map->m_llen; + if (vi->startblk == EROFS_NULL_ADDR) + return (0); + if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow( + vi->startblk << sbi->blkszbits, loff, &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "start block %ju overflows", + (uintmax_t)vi->startblk)); + map->m_flags |= EROFS_MAP_MAPPED; + return (0); + } + block_rem = sbi->block_size - + ((loff - tail_start) & (sbi->block_size - 1)); + map->m_llen = MIN(remain, block_rem); + map->m_plen = map->m_llen; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &map->m_pa) || + __builtin_add_overflow(map->m_pa, vi->xattr_isize, + &map->m_pa) || + __builtin_add_overflow(map->m_pa, loff - tail_start, + &map->m_pa)) + return (erofs_corrupt(sbi, vi->nid, + "inline tail position overflows")); + map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META; + return (0); + case EROFS_INODE_COMPRESSED_FULL: + case EROFS_INODE_COMPRESSED_COMPACT: + return (EOPNOTSUPP); + default: + return (EOPNOTSUPP); + } +} + +int +erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_map_blocks next = { .m_la = map->m_la }; + int err; + + EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, 0); + if (erofs_inode_is_data_compressed(vi->datalayout)) { +#if WITH_EROFS_ZIP + err = z_erofs_map_blocks(sbi, vi, &next); +#else + err = EOPNOTSUPP; +#endif + } else { + err = erofs_map_blocks_flatmode(sbi, vi, &next); + } + *map = next; + EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, err); + return (err); +} +static int +erofs_check_device_range(const struct erofs_sb_info *sbi, + const struct erofs_device_info *dif, erofs_off_t off, uint64_t len) +{ + erofs_off_t end, limit; + + if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) || + __builtin_add_overflow(off, len, &end)) + return (EINTEGRITY); + limit = dif->blocks << sbi->blkszbits; + return (end > limit ? EINTEGRITY : 0); +} + +static void +erofs_fill_from_devinfo(struct erofs_map_dev *map, + struct erofs_device_info *dif, erofs_off_t pa) +{ + map->m_dif = dif; + map->m_pa = pa; +} + +static int +erofs_map_dev_locate(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + struct erofs_device_info *dif; + uint64_t start; + unsigned int id; + int error; + + erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa); + if (map->m_deviceid != 0) { + if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL) + return (ENODEV); + dif = &sbi->devs[map->m_deviceid - 1]; + error = erofs_check_device_range(sbi, dif, map->m_pa, + map->m_plen); + if (error != 0) + return (error); + if (sbi->flatdev) { + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (__builtin_add_overflow(map->m_pa, start, + &map->m_pa)) + return (EINTEGRITY); + return (0); + } + if (!erofs_source_is_open(dif)) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa); + return (0); + } + + if (sbi->extra_devices == 0) + return (0); + if (sbi->flatdev) { + error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa, + map->m_plen); + if (error == 0) + return (0); + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0 || + dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + continue; + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa < start) + continue; + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error == 0) + return (0); + if (map->m_pa - start < + (dif->blocks << sbi->blkszbits)) + return (error); + } + return (EINTEGRITY); + } + for (id = 0; id < sbi->extra_devices; ++id) { + dif = &sbi->devs[id]; + if (dif->uniaddr == 0) + continue; + if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits)) + return (EINTEGRITY); + start = dif->uniaddr << sbi->blkszbits; + if (map->m_pa >= start && + map->m_pa - start < (dif->blocks << sbi->blkszbits)) { + error = erofs_check_device_range(sbi, dif, + map->m_pa - start, map->m_plen); + if (error != 0) + return (error); + if (!erofs_source_is_open(dif)) + return (ENODEV); + erofs_fill_from_devinfo(map, dif, map->m_pa - start); + break; + } + } + return (0); +} + +/* + * Resolve a physical range to a device. The lookup tries several devices + * and treats a miss on one as normal, so the corruption is reported here, + * once, when no device is able to serve the range. + */ +static int +erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map) +{ + uint64_t pa, plen; + unsigned int deviceid; + int error; + + pa = map->m_pa; + plen = map->m_plen; + deviceid = map->m_deviceid; + error = erofs_map_dev_locate(sbi, map); + if (error == EINTEGRITY) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "range %ju+%ju of device %u is not backed by any device", + (uintmax_t)pa, (uintmax_t)plen, deviceid)); + return (error); +} + +int +erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp) +{ + struct erofs_map_dev map; + erofs_blk_t blocks; + int error; + + map = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, &map); + if (error != 0) + return (error); + blocks = map.m_dif->blocks; + if (map.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len, + NULL, NULL, 0, bufp)); +} + +static int +erofs_read_physical_readahead(struct erofs_sb_info *sbi, + unsigned int device_id, erofs_off_t off, size_t len, + unsigned int rablocks, void **bufp) +{ + struct erofs_map_dev current, future; + daddr_t *rablkno; + int *rabsize; + erofs_off_t device_pa, step; + erofs_blk_t blocks; + unsigned int count; + int error, ret; + + current = (struct erofs_map_dev) { + .m_pa = off, + .m_deviceid = device_id, + .m_plen = len, + }; + error = erofs_map_dev(sbi, ¤t); + if (error != 0) + return (error); + blocks = current.m_dif->blocks; + if (current.m_dif == &sbi->dif0 && sbi->flatdev) + blocks = sbi->flatdev_blocks; + if (rablocks == 0) + return (erofs_bread_device(sbi, current.m_dif, blocks, + current.m_pa, len, NULL, NULL, 0, bufp)); + /* + * 256 slots are 2 KiB, which is a large part of a kernel stack inside + * the recursive read path, so keep the run lists off the stack. + */ + rablocks = MIN(rablocks, (unsigned int)EROFS_DIR_READAHEAD_SLOTS); + rablkno = malloc(rablocks * sizeof(*rablkno), M_EROFS, M_WAITOK); + rabsize = malloc(rablocks * sizeof(*rabsize), M_EROFS, M_WAITOK); + for (count = 0; count < rablocks; count++) { + step = (erofs_off_t)(count + 1) * sbi->block_size; + if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step) + break; + future = (struct erofs_map_dev) { + .m_pa = off + step, + .m_deviceid = device_id, + .m_plen = sbi->block_size, + }; + if (erofs_map_dev(sbi, &future) != 0 || + future.m_dif != current.m_dif || + future.m_pa != current.m_pa + step) + break; + if (__builtin_add_overflow(future.m_dif->fsoff, future.m_pa, + &device_pa) || device_pa > INT64_MAX) + break; + rablkno[count] = btodb(device_pa); + rabsize[count] = sbi->block_size; + } + ret = erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len, + rablkno, rabsize, count, bufp); + free(rabsize, M_EROFS); + free(rablkno, M_EROFS); + return (ret); +} + +static int +erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + char *out; + void *blk; + erofs_off_t mapoff, physical; + size_t done, want; + int error; + + if (bufp == NULL) + return (EINVAL); + *bufp = NULL; + if (len == 0) { + return (0); + } + EROFS_TRACE_READ_FOLIO(vi, loff, len, + !erofs_inode_is_data_compressed(vi->datalayout)); + if (loff > vi->size || (uint64_t)len > vi->size - loff) + return (erofs_corrupt(sbi, vi->nid, + "read of %zu bytes at %ju is beyond the size %ju", len, + (uintmax_t)loff, (uintmax_t)vi->size)); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) +#if WITH_EROFS_ZIP + return (z_erofs_read_data(sbi, vi, loff, len, bufp)); +#else + return (EOPNOTSUPP); +#endif + + /* Uncompressed file path */ + out = malloc(len, M_EROFS, M_WAITOK); + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) { + free(out, M_EROFS); + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju is empty or does not cover " + "it", + (uintmax_t)(loff + done))); + } + mapoff = loff + done - map.m_la; + want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS), + len - done); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + if ((map.m_flags & EROFS_MAP_META) != 0) { + if (mapoff > UINT64_MAX - map.m_pa) { + free(out, M_EROFS); + return (EOVERFLOW); + } + physical = map.m_pa + mapoff; + error = erofs_read_metadata_range(sbi, vi->nid, + physical, want, &buf); + } else if (done == 0 && mapoff == 0 && rablocks != 0 && + map.m_flags == EROFS_MAP_MAPPED) { + error = erofs_read_physical_readahead(sbi, + map.m_deviceid, map.m_pa, want, rablocks, + &blk); + } else { + if (mapoff > UINT64_MAX - map.m_pa) { + free(out, M_EROFS); + return (EOVERFLOW); + } + physical = map.m_pa + mapoff; + error = erofs_read_physical(sbi, map.m_deviceid, + physical, want, &blk); + } + if (error != 0) { + free(out, M_EROFS); + return (error); + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + memcpy(out + done, buf.data, want); + erofs_put_metabuf(&buf); + } else { + memcpy(out + done, blk, want); + erofs_brelse(blk); + } + } + done += want; + } + *bufp = out; + return (0); +} + +int +erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp)); +} + +int +erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, bool sequential, void **bufp) +{ + uint64_t dir_ra_bytes, total_blocks; + uint64_t remaining; + unsigned int rablocks; + int error; + + rablocks = 0; + if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN && + sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 && + len <= sbi->block_size && loff <= vi->size && + len <= vi->size - loff) { + dir_ra_bytes = erofs_dir_ra_bytes(sbi); + total_blocks = dir_ra_bytes / sbi->block_size; + if (total_blocks > 1) { + remaining = vi->size - loff - len; + rablocks = MIN(total_blocks - 1, + howmany(remaining, sbi->block_size)); + rablocks = MIN(rablocks, + (unsigned int)EROFS_DIR_READAHEAD_SLOTS); + } + } + EROFS_TRACE_READAHEAD(vi, loff >> PAGE_SHIFT, + howmany((uint64_t)len + + (uint64_t)rablocks * (uint64_t)sbi->block_size, PAGE_SIZE), + !erofs_inode_is_data_compressed(vi->datalayout)); + error = erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp); + if (error == 0 && rablocks != 0) + erofs_record_dir_readahead(sbi, rablocks); + return (error); +} + +static int +erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_map_blocks map; + void *blk; + erofs_off_t mapoff, physical; + size_t want, chunk; + int error; + + if (uio->uio_offset < 0) + return (EINVAL); + if ((uint64_t)uio->uio_offset >= vi->size) + return (0); + EROFS_TRACE_READ_FOLIO(vi, (uint64_t)uio->uio_offset, + (uint64_t)uio->uio_resid, + !erofs_inode_is_data_compressed(vi->datalayout)); + + /* Compressed file path */ + if (erofs_inode_is_data_compressed(vi->datalayout)) +#if WITH_EROFS_ZIP + return (z_erofs_read_uio(sbi, vi, uio)); +#else + return (EOPNOTSUPP); +#endif + + /* Uncompressed file path */ + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + map = (struct erofs_map_blocks) { .m_la = uio->uio_offset }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > (uint64_t)uio->uio_offset || + (uint64_t)uio->uio_offset - map.m_la >= map.m_llen) + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %jd is empty or does not cover " + "it", + (intmax_t)uio->uio_offset)); + mapoff = (uint64_t)uio->uio_offset - map.m_la; + want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS), + (size_t)uio->uio_resid); + if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + chunk = want; + while (chunk > 0) { + size_t zlen = MIN(chunk, ZERO_REGION_SIZE); + + error = uiomove(__DECONST(void *, zero_region), + zlen, uio); + if (error != 0) + return (error); + chunk -= zlen; + } + continue; + } + if ((map.m_flags & EROFS_MAP_META) != 0) { + if (mapoff > UINT64_MAX - map.m_pa) + return (EOVERFLOW); + physical = map.m_pa + mapoff; + error = erofs_read_metadata_range(sbi, vi->nid, + physical, want, &buf); + } else { + if (mapoff > UINT64_MAX - map.m_pa) + return (EOVERFLOW); + physical = map.m_pa + mapoff; + error = erofs_read_physical(sbi, map.m_deviceid, + physical, want, &blk); + } + if (error != 0) + return (error); + if ((map.m_flags & EROFS_MAP_META) != 0) { + error = uiomove(buf.data, want, uio); + erofs_put_metabuf(&buf); + } else { + error = uiomove(blk, want, uio); + erofs_brelse(blk); + } + if (error != 0) + return (error); + } + return (0); +} + +/* Read regular file data. */ +int +erofs_read_file(struct vnode *vp, struct uio *uio) +{ + return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio)); +} + +int +erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks map; + erofs_off_t pos, end; + bool data; + int error; + + if (vp->v_type != VREG) + return (ENOTTY); + vi = VTOE(vp); + if (*offset < 0 || (uint64_t)*offset >= vi->size) + return (ENXIO); + + sbi = MTOE(vp->v_mount); + pos = (erofs_off_t)*offset; + while (pos < vi->size) { + map = (struct erofs_map_blocks) { .m_la = pos }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_la > pos || map.m_llen == 0 || + map.m_la > UINT64_MAX - map.m_llen) { + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju is empty or does not cover " + "it", + (uintmax_t)pos)); + } + end = map.m_la + map.m_llen; + if (pos >= end) + return (erofs_corrupt(sbi, vi->nid, + "mapping of offset %ju ends at %ju", (uintmax_t)pos, + (uintmax_t)end)); + data = (map.m_flags & + (EROFS_MAP_MAPPED | EROFS_MAP_FRAGMENT)) != 0; + if (data == seek_data) { + *offset = (off_t)pos; + return (0); + } + pos = MIN(end, vi->size); + } + + if (seek_data) + return (ENXIO); + if (vi->size > OFF_MAX) + return (EOVERFLOW); + *offset = (off_t)vi->size; + return (0); +} + +/* Read symlink target string. */ +int +erofs_readlink_target(struct vnode *vp, struct uio *uio) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + if (vi->size == 0) + return (erofs_corrupt(MTOE(vp->v_mount), vi->nid, + "empty symlink")); + return (erofs_read_uio(MTOE(vp->v_mount), vi, uio)); +} diff --git a/decompressor.c b/decompressor.c new file mode 100644 index 0000000..82b3721 --- /dev/null +++ b/decompressor.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ + +#include +#include +#include +#include + +#include "compress.h" + +static int +z_erofs_load_lz4_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, const void *data, size_t size) +{ + const struct z_erofs_lz4_cfgs *lz4; + uint32_t max_pclusterblks; + uint16_t distance; + + if (data != NULL) { + if (size < sizeof(*lz4)) + return (EINTEGRITY); + lz4 = data; + max_pclusterblks = le16toh(lz4->max_pclusterblks); + if (max_pclusterblks > + (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits)) + return (EOPNOTSUPP); + } else { + distance = le16toh(dsb->u1.lz4_max_distance); + if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi)) + return (0); + sbi->available_compr_algs = 1U << Z_EROFS_COMPRESSION_LZ4; + } + return (0); +} + +static int +z_erofs_transform_plain(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *src; + uint8_t *dst; + size_t first, offset, span; + + if (rq->outputsize > rq->inputsize) + return (EINTEGRITY); + src = rq->in; + dst = rq->out; + if (rq->map->m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) { + memmove(dst, src, rq->outputsize); + return (0); + } + span = rq->sbi->block_size - + (rq->map->m_la & (rq->sbi->block_size - 1)); + if (span > rq->inputsize) + return (EINTEGRITY); + first = MIN(span, rq->outputsize); + offset = rq->inputsize - span; + if (offset > rq->inputsize || first > rq->inputsize - offset) + return (EINTEGRITY); + memmove(dst, src + offset, first); + if (first < rq->outputsize) + memmove(dst + first, src, rq->outputsize - first); + return (0); +} + +static const struct z_erofs_decompressor z_erofs_shifted_decomp = { + .decompress = z_erofs_transform_plain, +}; + +static const struct z_erofs_decompressor z_erofs_interlaced_decomp = { + .decompress = z_erofs_transform_plain, +}; + +const struct z_erofs_decompressor z_erofs_lz4_decomp = { + .config = z_erofs_load_lz4_config, + .decompress = z_erofs_lz4_buffer_decompress, +}; + +static const struct z_erofs_decompressor * const z_erofs_decomp[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp, + [Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp, + [Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp, +#if WITH_EROFS_ZIP_LZMA + [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, +#endif +#if WITH_EROFS_ZIP_DEFLATE + [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, +#endif +#if WITH_EROFS_ZIP_ZSTD + [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, +#endif +}; + +static int +z_erofs_read_cfg(struct erofs_sb_info *sbi, uint64_t *offset, + struct erofs_buf *buf, + size_t *sizep) +{ + struct erofs_buf metabuf = EROFS_BUF_INITIALIZER; + uint8_t length_buf[2]; + uint64_t aligned; + uint16_t length; + int error; + + /* + * At most one configuration per algorithm, each at most 64 KiB, follows + * the superblock, so the offset stays far below the overflow limit; the + * reads are range checked. + */ + aligned = roundup2(*offset, 4); + error = erofs_read_metadata_range(sbi, 0, aligned, sizeof(length_buf), + &metabuf); + if (error != 0) + return (error); + memcpy(length_buf, metabuf.data, sizeof(length_buf)); + erofs_put_metabuf(&metabuf); + length = le16dec(length_buf); + *sizep = length != 0 ? length : UINT16_MAX + 1U; + *offset = aligned + sizeof(length_buf); + error = erofs_read_metadata_range(sbi, 0, *offset, *sizep, buf); + if (error == 0) + *offset += *sizep; + return (error); +} + +int +z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf data = EROFS_BUF_INITIALIZER; + const struct z_erofs_decompressor *decompressor; + uint64_t offset; + uint16_t algorithms; + size_t size; + int algorithm, ret; + + if (!erofs_sb_has_compr_cfgs(sbi)) + return (z_erofs_lz4_decomp.config(sbi, dsb, NULL, 0)); + algorithms = le16toh(dsb->u1.available_compr_algs); + sbi->available_compr_algs = algorithms; + if ((algorithms & ~Z_EROFS_ALL_COMPR_ALGS) != 0) + return (EOPNOTSUPP); + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; + ++algorithm) { + if ((algorithms & (1U << algorithm)) == 0) + continue; + ret = z_erofs_read_cfg(sbi, &offset, &data, &size); + if (ret != 0) + return (ret); + decompressor = z_erofs_decomp[algorithm]; + if (decompressor == NULL || decompressor->config == NULL) + ret = EOPNOTSUPP; + else + ret = decompressor->config(sbi, dsb, data.data, size); + erofs_put_metabuf(&data); + if (ret != 0) + return (ret); + } + return (0); +} + +int +z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src0, size_t srclen, + void *dst, size_t dstlen, bool partial) +{ + const struct z_erofs_decompressor *decompressor; + struct z_erofs_decompress_req rq; + const uint8_t *src; + uint8_t algorithm; + size_t padding, padding_limit; + + algorithm = map->m_algorithmformat; + if (algorithm >= nitems(z_erofs_decomp) || + z_erofs_decomp[algorithm] == NULL || + z_erofs_decomp[algorithm]->decompress == NULL) + return (EOPNOTSUPP); + decompressor = z_erofs_decomp[algorithm]; + rq = (struct z_erofs_decompress_req) { + .sbi = sbi, + .map = map, + .in = src0, + .inputsize = srclen, + .out = dst, + .outputsize = dstlen, + .partial_decoding = partial, + }; + if (algorithm == Z_EROFS_COMPRESSION_SHIFTED || + algorithm == Z_EROFS_COMPRESSION_INTERLACED) + return (decompressor->decompress(&rq)); + + src = src0; + if (map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4 || + erofs_sb_has_lz4_0padding(sbi)) { + padding_limit = MIN(srclen, sbi->block_size - + (map->m_pa & (sbi->block_size - 1))); + for (padding = 0; padding < padding_limit && src[padding] == 0; + ++padding) + ; + if (padding == padding_limit) + return (EINTEGRITY); + src += padding; + srclen -= padding; + } + rq.in = src; + rq.inputsize = srclen; + return (decompressor->decompress(&rq)); +} diff --git a/decompressor_deflate.c b/decompressor_deflate.c new file mode 100644 index 0000000..5e49d6c --- /dev/null +++ b/decompressor_deflate.c @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal DEFLATE decompressor for EROFS FreeBSD */ + +#include +#include +#include + +#include + +#include "compress.h" + +struct z_erofs_deflate_ctx { + struct erofs_stream_ctx pool; + z_stream stream; + bool initialized; +}; + +_Static_assert(sizeof(struct z_erofs_deflate_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Deflate stream wrapper exceeds UMA item"); + +static voidpf +z_erofs_deflate_alloc(voidpf opaque, uInt items, uInt size) +{ + struct erofs_stream_ctx *pool; + pool = opaque; + /* Two 32-bit factors cannot overflow a 64-bit size_t. */ + return (z_erofs_stream_ctx_alloc(pool, (size_t)items * size)); +} + +static void +z_erofs_deflate_free(voidpf opaque, voidpf address) +{ + + z_erofs_stream_ctx_free(opaque, address); +} + +static int +z_erofs_load_deflate_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_deflate_cfgs *deflate; + + if (size < sizeof(*deflate)) + return (EINTEGRITY); + deflate = data; + if (deflate->windowbits > MAX_WBITS) + return (EOPNOTSUPP); + return (0); +} + +static int +z_erofs_deflate_error(int ret) +{ + + switch (ret) { + case Z_MEM_ERROR: + return (ENOMEM); + case Z_VERSION_ERROR: + return (EOPNOTSUPP); + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_BUF_ERROR: + return (EINTEGRITY); + default: + return (EIO); + } +} + +static void +z_erofs_deflate_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + + ctx = (struct z_erofs_deflate_ctx *)pool; + if (ctx->initialized) { + (void)inflateEnd(&ctx->stream); + ctx->initialized = false; + } +} + +static int +z_erofs_deflate_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_deflate_ctx *ctx; + int ret; + + ctx = (struct z_erofs_deflate_ctx *)pool; + bzero(&ctx->stream, sizeof(ctx->stream)); + ctx->stream.zalloc = z_erofs_deflate_alloc; + ctx->stream.zfree = z_erofs_deflate_free; + ctx->stream.opaque = pool; + ret = inflateInit2(&ctx->stream, -MAX_WBITS); + if (ret != Z_OK) + return (z_erofs_deflate_error(ret)); + ctx->initialized = true; + return (0); +} + +static int +z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret, + uInt avail_in) +{ + + if (rq->partial_decoding) + return (0); + if (ret != Z_STREAM_END || avail_in != 0) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_deflate_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_deflate_ctx *ctx; + z_stream *strm; + uInt in_before, out_before; + int error, ret; + + /* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */ + EROFS_DEBUG_ASSERT(rq->inputsize <= UINT_MAX && + rq->outputsize <= UINT_MAX, ("erofs: deflate sizes")); + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_DEFLATE, + sizeof(*ctx), z_erofs_deflate_ctx_init, z_erofs_deflate_ctx_fini, + &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_deflate_ctx *)pool; + strm = &ctx->stream; + pool->allocation_failed = false; + ret = inflateReset2(strm, -MAX_WBITS); + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + z_erofs_stream_ctx_put(pool, false); + return (error); + } + strm->next_in = __DECONST(void *, rq->in); + strm->avail_in = rq->inputsize; + strm->next_out = rq->out; + strm->avail_out = rq->outputsize; + + error = 0; + ret = Z_OK; + while (strm->avail_out != 0) { + in_before = strm->avail_in; + out_before = strm->avail_out; + ret = inflate(strm, Z_SYNC_FLUSH); + if (ret == Z_STREAM_END) + break; + if (ret != Z_OK) { + error = z_erofs_deflate_error(ret); + break; + } + if (strm->avail_in == in_before && + strm->avail_out == out_before) { + error = EINTEGRITY; + break; + } + } + if (error == 0 && strm->avail_out != 0) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_deflate_finish(rq, ret, strm->avail_in); + if (pool->allocation_failed) + error = ENOMEM; + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, +}; diff --git a/decompressor_lz4.c b/decompressor_lz4.c new file mode 100644 index 0000000..06549b2 --- /dev/null +++ b/decompressor_lz4.c @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Contiguous-buffer LZ4 backend for EROFS FreeBSD. */ + +#include +#include +#include + +#include "compress.h" + +#define EROFS_LZ4_TOKEN_LITERAL_SHIFT 4 +#define EROFS_LZ4_TOKEN_MATCH_MASK 0x0f +#define EROFS_LZ4_MAX_RUN 15 +#define EROFS_LZ4_EXT_SENTINEL 255 +#define EROFS_LZ4_MIN_MATCH 4 +#define EROFS_LZ4_OFFSET_BYTES 2 + +static int +lz4_finish(const uint8_t *ip, const uint8_t *iend, int partial) +{ + if (partial) + return (0); + while (ip < iend) { + if (*ip++ != 0) + return (EINTEGRITY); + } + return (0); +} + +int +z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq) +{ + const uint8_t *input; + uint8_t *output; + unsigned int token; + size_t length, copylen; + size_t offset; + size_t input_pos, output_pos; + + if (rq == NULL || (rq->inputsize != 0 && rq->in == NULL) || + (rq->outputsize != 0 && rq->out == NULL)) + return (EINTEGRITY); + if (rq->inputsize == 0) + return (rq->outputsize == 0 ? 0 : EINTEGRITY); + /* + * A length grows by at most 255 per input byte, so it cannot reach + * SIZE_MAX however long the input is. + */ + input = rq->in; + output = rq->out; + input_pos = 0; + output_pos = 0; + + while (input_pos < rq->inputsize) { + token = input[input_pos++]; + length = token >> EROFS_LZ4_TOKEN_LITERAL_SHIFT; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (input_pos >= rq->inputsize) + return (EINTEGRITY); + value = input[input_pos++]; + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + if (length > rq->inputsize - input_pos) + return (EINTEGRITY); + if (!rq->partial_decoding && + length > rq->outputsize - output_pos) + return (EINTEGRITY); + copylen = MIN(length, rq->outputsize - output_pos); + memcpy(output + output_pos, input + input_pos, copylen); + input_pos += length; + output_pos += copylen; + if (output_pos == rq->outputsize) + return (lz4_finish(input + input_pos, + input + rq->inputsize, rq->partial_decoding)); + if (input_pos >= rq->inputsize) + break; + if (EROFS_LZ4_OFFSET_BYTES > rq->inputsize - input_pos) + return (EINTEGRITY); + offset = le16dec(input + input_pos); + input_pos += EROFS_LZ4_OFFSET_BYTES; + if (offset == 0 || offset > output_pos) + return (EINTEGRITY); + length = token & EROFS_LZ4_TOKEN_MATCH_MASK; + if (length == EROFS_LZ4_MAX_RUN) { + unsigned int value; + do { + if (input_pos >= rq->inputsize) + return (EINTEGRITY); + value = input[input_pos++]; + length += value; + } while (value == EROFS_LZ4_EXT_SENTINEL); + } + length += EROFS_LZ4_MIN_MATCH; + if (!rq->partial_decoding && + length > rq->outputsize - output_pos) + return (EINTEGRITY); + copylen = MIN(length, rq->outputsize - output_pos); + while (copylen-- != 0) { + output[output_pos] = output[output_pos - offset]; + ++output_pos; + } + if (output_pos == rq->outputsize) + return (lz4_finish(input + input_pos, + input + rq->inputsize, rq->partial_decoding)); + } + return (output_pos == rq->outputsize ? + lz4_finish(input + input_pos, input + rq->inputsize, + rq->partial_decoding) : EINTEGRITY); +} diff --git a/decompressor_lzma.c b/decompressor_lzma.c new file mode 100644 index 0000000..44eded7 --- /dev/null +++ b/decompressor_lzma.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * EROFS MicroLZMA wrapper around FreeBSD's bundled XZ Embedded decoder. + * The decoder source is compiled with private symbol names because the + * stock xz.ko does not enable its optional MicroLZMA entry points. + */ + +#include +#include +#include + +#include "compress.h" + +#define XZ_DEC_MICROLZMA +#define xz_dec_lzma2_create erofs_xz_dec_lzma2_create +#define xz_dec_lzma2_reset erofs_xz_dec_lzma2_reset +#define xz_dec_lzma2_run erofs_xz_dec_lzma2_run +#define xz_dec_lzma2_end erofs_xz_dec_lzma2_end +#define xz_dec_microlzma_alloc erofs_xz_dec_microlzma_alloc +#define xz_dec_microlzma_reset erofs_xz_dec_microlzma_reset +#define xz_dec_microlzma_run erofs_xz_dec_microlzma_run +#define xz_dec_microlzma_end erofs_xz_dec_microlzma_end +#define xz_malloc erofs_xz_malloc +#define xz_free erofs_xz_free + +static void * +erofs_xz_malloc(unsigned long size) +{ + return (malloc(size, M_EROFS, M_WAITOK)); +} + +static void +erofs_xz_free(void *ptr) +{ + free(ptr, M_EROFS); +} + +#include + + +#undef bool +#undef false +#undef true +#undef min + +struct z_erofs_lzma_ctx { + struct erofs_stream_ctx pool; + struct xz_dec_microlzma *state; + size_t state_bytes; +}; + +_Static_assert(sizeof(struct z_erofs_lzma_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "LZMA stream wrapper exceeds UMA item"); + +static void +z_erofs_lzma_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + + ctx = (struct z_erofs_lzma_ctx *)pool; + if (ctx->state != NULL) { + xz_dec_microlzma_end(ctx->state); + ctx->state = NULL; + } + if (ctx->state_bytes != 0) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + } +} + +static int +z_erofs_lzma_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_lzma_ctx *ctx; + int error; + + ctx = (struct z_erofs_lzma_ctx *)pool; + ctx->state_bytes = sizeof(*ctx->state); + error = z_erofs_stream_ctx_charge(pool, ctx->state_bytes); + if (error != 0) { + ctx->state_bytes = 0; + return (error); + } + ctx->state = xz_dec_microlzma_alloc(XZ_SINGLE, + pool->sbi->lzma_dict_size); + if (ctx->state == NULL) { + z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes); + ctx->state_bytes = 0; + pool->allocation_failed = 1; + return (ENOMEM); + } + return (0); +} + +static int +z_erofs_load_lzma_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_lzma_cfgs *lzma; + uint32_t dict_size; + + if (size < sizeof(*lzma)) + return (EINTEGRITY); + lzma = data; + if (le16toh(lzma->format) != 0) + return (EOPNOTSUPP); + dict_size = le32toh(lzma->dict_size); + if (dict_size < 4096) + return (EINTEGRITY); + if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) + return (EOPNOTSUPP); + sbi->lzma_dict_size = dict_size; + return (0); +} + +static int +z_erofs_lzma_error(enum xz_ret ret) +{ + + switch (ret) { + case XZ_MEM_ERROR: + return (ENOMEM); + case XZ_MEMLIMIT_ERROR: + case XZ_OPTIONS_ERROR: + case XZ_UNSUPPORTED_CHECK: + return (EOPNOTSUPP); + default: + return (EINTEGRITY); + } +} + +static int +z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret, + size_t input_pos) +{ + + if (rq->partial_decoding && + (ret == XZ_OK || ret == XZ_STREAM_END)) + return (0); + if (!rq->partial_decoding && ret == XZ_STREAM_END && + input_pos == rq->inputsize) + return (0); + return (z_erofs_lzma_error(ret)); +} + +static int +z_erofs_lzma_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_lzma_ctx *ctx; + struct xz_buf buffer; + enum xz_ret ret; + int error; + + /* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */ + EROFS_DEBUG_ASSERT(rq->inputsize <= UINT32_MAX && + rq->outputsize <= UINT32_MAX, ("erofs: lzma sizes")); + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_LZMA, + sizeof(*ctx), z_erofs_lzma_ctx_init, z_erofs_lzma_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_lzma_ctx *)pool; + bzero(&buffer, sizeof(buffer)); + buffer.in = rq->in; + buffer.in_size = rq->inputsize; + buffer.out = rq->out; + buffer.out_size = rq->outputsize; + xz_dec_microlzma_reset(ctx->state, (uint32_t)rq->inputsize, + (uint32_t)rq->outputsize, !rq->partial_decoding); + ret = xz_dec_microlzma_run(ctx->state, &buffer); + if (buffer.out_pos != rq->outputsize) + error = EINTEGRITY; + else + error = z_erofs_lzma_finish(rq, ret, buffer.in_pos); + z_erofs_stream_ctx_put(pool, error == 0); + return (error); +} + +const struct z_erofs_decompressor z_erofs_lzma_decomp = { + .config = z_erofs_load_lzma_config, + .decompress = z_erofs_lzma_decompress, +}; diff --git a/decompressor_zstd.c b/decompressor_zstd.c new file mode 100644 index 0000000..01ab746 --- /dev/null +++ b/decompressor_zstd.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* Minimal zstd decompressor for EROFS FreeBSD */ +#include +#include + +#include "compress.h" + +#ifdef ZSTDIO +#define ZSTD_STATIC_LINKING_ONLY +#include + + +struct z_erofs_zstd_ctx { + struct erofs_stream_ctx pool; + ZSTD_DCtx *dctx; +}; + +_Static_assert(sizeof(struct z_erofs_zstd_ctx) <= + EROFS_STREAM_CTX_WRAPPER_SIZE, "Zstd stream wrapper exceeds UMA item"); +#endif + +static bool +erofs_zstd_available(void) +{ +#ifdef ZSTDIO + return (true); +#else + return (false); +#endif +} + +static int +z_erofs_load_zstd_config(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb __unused, const void *data, size_t size) +{ + const struct z_erofs_zstd_cfgs *zstd; + + if (!erofs_zstd_available()) { + vfs_mount_error(sbi->mnt, + "erofs: ZSTD compression requires ZSTDIO support"); + return (EOPNOTSUPP); + } + if (data == NULL || size < sizeof(*zstd)) + return (EINTEGRITY); + zstd = data; + if (zstd->format != 0 || zstd->windowlog > 10) + return (EOPNOTSUPP); + sbi->zstd_windowlog = zstd->windowlog; + return (0); +} + +#ifdef ZSTDIO +static void * +zstd_alloc(void *opaque, size_t size) +{ + return (z_erofs_stream_ctx_alloc(opaque, size)); +} + +static void +zstd_free(void *opaque, void *address) +{ + z_erofs_stream_ctx_free(opaque, address); +} + +static void +z_erofs_zstd_ctx_fini(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + + ctx = (struct z_erofs_zstd_ctx *)pool; + if (ctx->dctx != NULL) { + (void)ZSTD_freeDCtx(ctx->dctx); + ctx->dctx = NULL; + } +} + +static int +z_erofs_zstd_ctx_init(struct erofs_stream_ctx *pool) +{ + struct z_erofs_zstd_ctx *ctx; + ZSTD_customMem alloc; + size_t ret; + + ctx = (struct z_erofs_zstd_ctx *)pool; + alloc = (ZSTD_customMem) { + .customAlloc = zstd_alloc, + .customFree = zstd_free, + .opaque = pool, + }; + ctx->dctx = ZSTD_createDCtx_advanced(alloc); + if (ctx->dctx == NULL) + return (ENOMEM); + ret = ZSTD_DCtx_setParameter(ctx->dctx, ZSTD_d_windowLogMax, + pool->sbi->zstd_windowlog + 10); + if (ZSTD_isError(ret)) + return (pool->allocation_failed ? ENOMEM : EOPNOTSUPP); + return (0); +} + +static int +z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret, + size_t input_pos, size_t input_size) +{ + + if (rq->partial_decoding) + return (0); + if (ret != 0 || input_pos != input_size) + return (EINTEGRITY); + return (0); +} + +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq) +{ + struct erofs_stream_ctx *pool; + struct z_erofs_zstd_ctx *ctx; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t in_before, out_before, ret; + int error; + + error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_ZSTD, + sizeof(*ctx), z_erofs_zstd_ctx_init, z_erofs_zstd_ctx_fini, &pool); + if (error != 0) + return (error); + ctx = (struct z_erofs_zstd_ctx *)pool; + pool->allocation_failed = false; + + input = (ZSTD_inBuffer) { + .src = rq->in, + .size = rq->inputsize, + }; + output = (ZSTD_outBuffer) { + .dst = rq->out, + .size = rq->outputsize, + }; + error = 0; + ret = 1; + while (output.pos != output.size) { + in_before = input.pos; + out_before = output.pos; + ret = ZSTD_decompressStream(ctx->dctx, &output, &input); + if (ZSTD_isError(ret)) { + error = pool->allocation_failed ? ENOMEM : EINTEGRITY; + break; + } + if (input.pos == in_before && output.pos == out_before) { + error = EINTEGRITY; + break; + } + if (ret == 0) + break; + } + if (error == 0 && output.pos != output.size) + error = EINTEGRITY; + else if (error == 0) + error = z_erofs_zstd_finish(rq, ret, input.pos, input.size); + z_erofs_stream_ctx_put(pool, error == 0 && !rq->partial_decoding); + return (error); +} +#else +static int +z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq __unused) +{ + return (EOPNOTSUPP); +} +#endif + +const struct z_erofs_decompressor z_erofs_zstd_decomp = { + .config = z_erofs_load_zstd_config, + .decompress = z_erofs_zstd_decompress, +}; diff --git a/dir.c b/dir.c new file mode 100644 index 0000000..522d373 --- /dev/null +++ b/dir.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static unsigned char +erofs_ftype_to_dtype(uint8_t ftype) +{ + switch (ftype) { + case EROFS_FT_REG_FILE: + return (DT_REG); + case EROFS_FT_DIR: + return (DT_DIR); + case EROFS_FT_CHRDEV: + return (DT_CHR); + case EROFS_FT_BLKDEV: + return (DT_BLK); + case EROFS_FT_FIFO: + return (DT_FIFO); + case EROFS_FT_SOCK: + return (DT_SOCK); + case EROFS_FT_SYMLINK: + return (DT_LNK); + default: + return (DT_UNKNOWN); + } +} + +/* + * Directory block checks. + * + * A block is trusted only as far as it is used: its header is checked once + * per block read, and each entry is bounds-checked when it is looked at. + * Nothing here requires names to be sorted, unique, or ordered across + * blocks. That is fsck.erofs's job, and a directory that violates it can + * only make a lookup miss; the binary search stays within the block either + * way. + */ + +/* + * The first name offset is also the size of the dirent array, so it must + * leave room for at least one dirent and lie inside the block. + */ +int +erofs_dirblock_ndirents(const char *blk, uint32_t maxsize, uint32_t *ndirentsp) +{ + uint32_t first_nameoff; + + if (maxsize < sizeof(struct erofs_dirent)) + return (EINTEGRITY); + first_nameoff = le16toh(((const struct erofs_dirent *)blk)->nameoff); + if (first_nameoff < sizeof(struct erofs_dirent) || + first_nameoff >= maxsize || + (first_nameoff % sizeof(struct erofs_dirent)) != 0) + return (EINTEGRITY); + *ndirentsp = first_nameoff / sizeof(struct erofs_dirent); + return (0); +} + +/* + * Bounds of the name of the idx'th entry: [*nameoffp, *endoffp) is non-empty + * and inside the block. A name ends where the next one starts; the last + * one ends at the block end and may be NUL terminated before that. + */ +int +erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents, + uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp) +{ + const struct erofs_dirent *de; + uint32_t endoff, nameoff; + + de = (const struct erofs_dirent *)blk; + nameoff = le16toh(de[idx].nameoff); + endoff = idx + 1 < ndirents ? le16toh(de[idx + 1].nameoff) : maxsize; + if (endoff > maxsize || endoff <= nameoff) + return (EINTEGRITY); + *nameoffp = nameoff; + *endoffp = endoff; + return (0); +} + +/* + * The name of an entry that readdir is about to hand to userland must be a + * plain file name that fits d_name: no NUL inside it (so d_namlen matches + * strlen(d_name)), no '/', and at most EROFS_NAME_LEN bytes. Lookup does + * not need any of this. + */ +static int +erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff, + bool trailing, size_t *namelenp) +{ + size_t namelen, span; + + span = endoff - nameoff; + if (trailing) { + namelen = strnlen(blk + nameoff, span); + } else { + namelen = span; + if (memchr(blk + nameoff, '\0', span) != NULL) + return (EINTEGRITY); + } + if (namelen == 0 || namelen > EROFS_NAME_LEN) + return (EINTEGRITY); + for (size_t i = 0; i < namelen; i++) { + if (blk[nameoff + i] == '/') + return (EINTEGRITY); + } + *namelenp = namelen; + return (0); +} + +/* Extract name, nid, and type of the idx'th directory entry from a block. */ +static int +erofs_dirent_name(const char *blk, uint32_t maxsize, + uint32_t idx, uint32_t ndirents, char *name, size_t namesz __unused, + erofs_nid_t *nid, uint8_t *ftype, size_t *namelenp) +{ + const struct erofs_dirent *de; + uint32_t nameoff, endoff; + size_t namelen; + int error; + + error = erofs_dirent_span(blk, maxsize, ndirents, idx, &nameoff, + &endoff); + if (error != 0) + return (error); + error = erofs_dirent_namelen(blk, nameoff, endoff, + idx + 1 == ndirents, &namelen); + if (error != 0) + return (error); + /* erofs_dirent_namelen() caps the name at EROFS_NAME_LEN. */ + EROFS_DEBUG_ASSERT(namelen < namesz, + ("erofs: dirent name of %zu bytes exceeds buffer", namelen)); + de = (const struct erofs_dirent *)blk; + memcpy(name, blk + nameoff, namelen); + name[namelen] = '\0'; + *nid = le64toh(de[idx].nid); + *ftype = de[idx].file_type; + *namelenp = namelen; + return (0); +} + +/* Per-call state for readdir dirent/cookie output. */ +struct erofs_uiodir { + struct dirent *dirent; + uint64_t *cookies; + uint64_t last_cookie; + int ncookies; + int acookies; + int eofflag; +}; + +enum erofs_uiodir_result { + EROFS_UIODIR_BUFFER_FULL = -1, + EROFS_UIODIR_OK = 0, +}; + +/* Push a dirent and its cookie to the caller, modelled after UDF. */ +static int +erofs_uiodir(struct erofs_uiodir *uiodir, int de_size, struct uio *uio, + uint64_t cookie) +{ + int error; + + /* + * Cookies are block offsets we computed, and grow with the entry + * index whatever the block contains. + */ + EROFS_DEBUG_ASSERT(cookie > uiodir->last_cookie, + ("erofs: readdir cookie %ju not above %ju", (uintmax_t)cookie, + (uintmax_t)uiodir->last_cookie)); + if (uio->uio_resid < de_size || + (uiodir->cookies != NULL && + uiodir->acookies >= uiodir->ncookies)) { + return (EROFS_UIODIR_BUFFER_FULL); + } + error = uiomove(uiodir->dirent, de_size, uio); + if (error != 0) + return (error); + uiodir->last_cookie = cookie; + if (uiodir->cookies != NULL) + uiodir->cookies[uiodir->acookies++] = cookie; + return (EROFS_UIODIR_OK); +} + +/* + * Process directory entries within a single block and output them to uio. + * Returns 0 on success (all entries consumed), -1 if uio is full, or a + * positive error code on corruption. + */ +static int +erofs_fill_dentries(struct erofs_sb_info *sbi, erofs_nid_t dirnid, + struct erofs_uiodir *uiodir, struct uio *uio, struct dirent *d, + const char *blk, uint32_t maxsize, + uint32_t start_idx, uint32_t ndirents, erofs_off_t block_off, + uint64_t *logical_offp) +{ + char name[EROFS_NAME_LEN + 1]; + uint32_t idx; + uint64_t curpos, nextoff; + erofs_nid_t nid; + size_t namelen; + uint8_t ftype; + int error; + + for (idx = start_idx; idx < ndirents; idx++) { + curpos = block_off + idx * sizeof(struct erofs_dirent); + nextoff = (idx + 1 < ndirents) ? + (curpos + sizeof(struct erofs_dirent)) : + (block_off + maxsize); + error = erofs_dirent_name(blk, maxsize, idx, ndirents, name, + sizeof(name), &nid, &ftype, &namelen); + if (error != 0) + return (erofs_corrupt(sbi, dirnid, + "invalid name of dirent %u in the block at offset " + "%ju", + idx, (uintmax_t)block_off)); + /* + * The nid is only reported as d_fileno; lookup checks it + * (erofs_read_inode()) before it is used for anything else. + */ + bzero(d, sizeof(*d)); + d->d_fileno = erofs_nid_to_ino64(sbi, nid); + d->d_type = erofs_ftype_to_dtype(ftype); + d->d_namlen = namelen; + d->d_reclen = GENERIC_DIRSIZ(d); + d->d_off = nextoff; + strlcpy(d->d_name, name, sizeof(d->d_name)); + error = erofs_uiodir(uiodir, d->d_reclen, uio, d->d_off); + if (error != 0) + return (error); + *logical_offp = nextoff; + uio->uio_offset = *logical_offp; + } + return (0); +} + +/* + * Read directory contents and output a FreeBSD dirent stream to uio. + * + * Key points: + * - On-disk entries use their logical file offsets as cookies; + * - A dot_omitted directory appends a synthetic "." at i_size, matching + * Linux, so existing on-disk cookies are not shifted; + * - The dirent array occupies only the front portion of a block, so after + * scanning all entries offset must jump to maxsize (the block end), + * otherwise the loop would get stuck on the same block; + * - Supports a_ncookies / a_cookies for NFS and other callers that need + * resumable iteration. + */ +int +erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies) +{ + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct erofs_uiodir uiodir; + struct dirent d; + uint64_t *cookiebuf; + size_t cookie_count; + char *blk; + erofs_off_t block_off; + uint64_t logical_off; + uint32_t block_pos, blksz, ndirents, start_idx, maxsize; + bool sequential; + int error; + + dir = VTOE(vp); + sbi = MTOE(vp->v_mount); + blksz = sbi->block_size; + error = 0; + cookiebuf = NULL; + cookie_count = 0; + uiodir.eofflag = 0; + uiodir.acookies = 0; + uiodir.dirent = &d; + uiodir.cookies = NULL; + uiodir.ncookies = 0; + if (cookies != NULL && ncookies != NULL) { + *cookies = NULL; + *ncookies = 0; + if (uio->uio_resid > 0) { + cookie_count = (size_t)uio->uio_resid / + GENERIC_MINDIRSIZ; + cookie_count = MIN(cookie_count, (size_t)INT_MAX); + } + uiodir.ncookies = (int)cookie_count; + if (cookie_count != 0) + cookiebuf = malloc(sizeof(*cookiebuf) * cookie_count, + M_TEMP, M_WAITOK); + uiodir.cookies = cookiebuf; + } + + if (uio->uio_offset < 0) { + error = EINVAL; + goto out; + } + if (dir->dot_omitted && dir->size == (uint64_t)OFF_MAX) { + error = erofs_corrupt(sbi, dir->nid, + "directory size leaves no room for '.'"); + goto out; + } + + logical_off = uio->uio_offset; + sequential = logical_off == 0; + uiodir.last_cookie = logical_off; + uio->uio_offset = logical_off; + + while (logical_off < dir->size) { + block_off = rounddown2(logical_off, (uint64_t)blksz); + maxsize = MIN((uint64_t)blksz, dir->size - block_off); + block_pos = logical_off - block_off; + if ((block_pos % sizeof(struct erofs_dirent)) != 0) { + block_pos = roundup(block_pos, + sizeof(struct erofs_dirent)); + logical_off = block_off + block_pos; + uio->uio_offset = logical_off; + } + error = erofs_read_data_readahead(sbi, dir, block_off, + maxsize, sequential, (void **)&blk); + if (error != 0) + goto out; + error = erofs_dirblock_ndirents(blk, maxsize, &ndirents); + if (error != 0) { + erofs_brelse(blk); + error = erofs_corrupt(sbi, dir->nid, + "invalid dirent array in the block at offset %ju", + (uintmax_t)block_off); + goto out; + } + start_idx = block_pos / sizeof(struct erofs_dirent); + if (start_idx >= ndirents) { + logical_off = block_off + maxsize; + uio->uio_offset = logical_off; + erofs_brelse(blk); + continue; + } + error = erofs_fill_dentries(sbi, dir->nid, &uiodir, uio, &d, + blk, maxsize, start_idx, ndirents, block_off, &logical_off); + erofs_brelse(blk); + if (error != 0) + goto out; + } + if (dir->dot_omitted && logical_off == dir->size) { + bzero(&d, sizeof(d)); + d.d_fileno = erofs_nid_to_ino64(sbi, dir->nid); + d.d_type = DT_DIR; + d.d_namlen = 1; + d.d_reclen = GENERIC_DIRSIZ(&d); + d.d_off = dir->size + 1; + d.d_name[0] = '.'; + d.d_name[1] = '\0'; + error = erofs_uiodir(&uiodir, d.d_reclen, uio, d.d_off); + if (error != 0) + goto out; + logical_off++; + uio->uio_offset = logical_off; + } + uiodir.eofflag = 1; +out: + if (error == EROFS_UIODIR_BUFFER_FULL) + error = 0; + if (eofflag != NULL && error == 0) + *eofflag = uiodir.eofflag; + if (cookies != NULL && ncookies != NULL) { + if (error != 0) { + if (cookiebuf != NULL) + free(cookiebuf, M_TEMP); + } else { + *ncookies = uiodir.acookies; + *cookies = cookiebuf; + } + } + return (error); +} diff --git a/erofs_compat.h b/erofs_compat.h new file mode 100644 index 0000000..f302244 --- /dev/null +++ b/erofs_compat.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * What erofs_fs.h, which is kept as it is in Linux, takes from Linux headers: + * the integer and little-endian types, a few macros, and the file type + * and magic definitions that Linux keeps elsewhere. + */ +#ifndef _FS_EROFS_COMPAT_H_ +#define _FS_EROFS_COMPAT_H_ + +#include +#include + +typedef uint8_t u8; +typedef uint8_t __u8; +typedef uint32_t __u32; +typedef uint16_t __le16; +typedef uint32_t __le32; +typedef uint64_t __le64; + +#define le16_to_cpu(x) le16toh(x) +#define cpu_to_le64(x) htole64(x) + +#define BIT(n) (1UL << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define round_up(x, y) roundup2((x), (y)) + +/* Fails to compile when the condition is true, as in Linux. */ +#define BUILD_BUG_ON(cond) ((void)sizeof(char[1 - 2 * !!(cond)])) + +/* include/uapi/linux/magic.h */ +#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2 + +/* + * File types in directory entries; Linux uses the generic FT_* values from + * include/linux/fs_types.h. + */ +#define EROFS_FT_UNKNOWN 0 +#define EROFS_FT_REG_FILE 1 +#define EROFS_FT_DIR 2 +#define EROFS_FT_CHRDEV 3 +#define EROFS_FT_BLKDEV 4 +#define EROFS_FT_FIFO 5 +#define EROFS_FT_SOCK 6 +#define EROFS_FT_SYMLINK 7 + +#endif diff --git a/erofs_fs.h b/erofs_fs.h new file mode 100644 index 0000000..7871b16 --- /dev/null +++ b/erofs_fs.h @@ -0,0 +1,467 @@ +/* SPDX-License-Identifier: MIT */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +/* to allow for x86 boot sectors and other oddities. */ +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 +#define EROFS_FEATURE_COMPAT_XATTR_FILTER 0x00000004 +#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX 0x00000008 +#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX 0x00000010 +#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS 0x00000020 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 +#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 +#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES 0x00000040 +#define EROFS_FEATURE_INCOMPAT_48BIT 0x00000080 +#define EROFS_FEATURE_INCOMPAT_METABOX 0x00000100 +#define EROFS_ALL_FEATURE_INCOMPAT \ + ((EROFS_FEATURE_INCOMPAT_METABOX << 1) - 1) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +struct erofs_deviceslot { + u8 tag[64]; /* digest(sha256), etc. */ + __le32 blocks_lo; /* total blocks count of this device */ + __le32 uniaddr_lo; /* unified starting block of this device */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 uniaddr_hi; /* unified starting block MSB */ + u8 reserved[52]; +}; +#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) + +/* erofs on-disk super block (currently 144 bytes at maximum) */ +struct erofs_super_block { + __le32 magic; /* file system magic number */ + __le32 checksum; /* crc32c to avoid unexpected on-disk overlap */ + __le32 feature_compat; + __u8 blkszbits; /* filesystem block size in bit shift */ + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */ + union { + __le16 rootnid_2b; /* nid of root directory */ + __le16 blocks_hi; /* (48BIT on) blocks count MSB */ + } __packed rb; + __le64 inos; /* total valid ino # (== f_files - f_favail) */ + __le64 epoch; /* base seconds used for compact inodes */ + __le32 fixed_nsec; /* fixed nanoseconds for compact inodes */ + __le32 blocks_lo; /* blocks count LSB */ + __le32 meta_blkaddr; /* start block address of metadata area */ + __le32 xattr_blkaddr; /* start block address of shared xattr area */ + __u8 uuid[16]; /* 128-bit uuid for volume */ + __u8 volume_name[16]; /* volume name */ + __le32 feature_incompat; + union { + /* bitmap for available compression algorithms */ + __le16 available_compr_algs; + /* customized sliding window size instead of 64k by default */ + __le16 lz4_max_distance; + } __packed u1; + __le16 extra_devices; /* # of devices besides the primary device */ + __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */ + __u8 dirblkbits; /* directory block size in bit shift */ + __u8 xattr_prefix_count; /* # of long xattr name prefixes */ + __le32 xattr_prefix_start; /* start of long xattr prefixes */ + __le64 packed_nid; /* nid of the special packed inode */ + __u8 xattr_filter_reserved; /* reserved for xattr name filter */ + __u8 ishare_xattr_prefix_id; + __u8 reserved[2]; + __le32 build_time; /* seconds added to epoch for mkfs time */ + __le64 rootnid_8b; /* (48BIT on) nid of root directory */ + __le64 reserved2; + __le64 metabox_nid; /* (METABOX on) nid of the metabox inode */ + __le64 reserved3; /* [align to extslot 1] */ +}; + +/* + * EROFS inode datalayout (i_format in on-disk inode): + * 0 - uncompressed flat inode without tail-packing inline data: + * 1 - compressed inode with non-compact indexes: + * 2 - uncompressed flat inode with tail-packing inline data: + * 3 - compressed inode with compact indexes: + * 4 - chunk-based inode with (optional) multi-device support: + * 5~7 - reserved + */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_COMPRESSED_FULL = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_COMPRESSED_COMPACT = 3, + EROFS_INODE_CHUNK_BASED = 4, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool erofs_inode_is_data_compressed(unsigned int datamode) +{ + return datamode == EROFS_INODE_COMPRESSED_COMPACT || + datamode == EROFS_INODE_COMPRESSED_FULL; +} + +/* bit definitions of inode i_format */ +#define EROFS_I_VERSION_MASK 0x01 +#define EROFS_I_DATALAYOUT_MASK 0x07 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_NLINK_1_BIT 4 /* non-directory compact inodes only */ +#define EROFS_I_DOT_OMITTED_BIT 4 /* (directories) omit the `.` dirent */ +#define EROFS_I_ALL ((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1) + +/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS_MASK 0x001F +/* with chunk indexes or just a 4-byte block array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 +#define EROFS_CHUNK_FORMAT_48BIT 0x0040 + +#define EROFS_CHUNK_FORMAT_ALL ((EROFS_CHUNK_FORMAT_48BIT << 1) - 1) + +/* 32-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +/* 64-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_EXTENDED 1 + +struct erofs_inode_chunk_info { + __le16 format; /* chunk blkbits, etc. */ + __le16 reserved; +}; + +union erofs_inode_i_u { + __le32 blocks_lo; /* total blocks count (if compressed inodes) */ + __le32 startblk_lo; /* starting block number (if flat inodes) */ + __le32 rdev; /* device ID (if special inodes) */ + struct erofs_inode_chunk_info c; +}; + +union erofs_inode_i_nb { + __le16 nlink; /* if EROFS_I_NLINK_1_BIT is unset */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 startblk_hi; /* starting block number MSB */ +} __packed; + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le32 i_size; + __le32 i_mtime; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved; +}; + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le64 i_size; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le32 i_uid; + __le32 i_gid; + __le64 i_mtime; + __le32 i_mtime_nsec; + __le32 i_nlink; + __u8 i_reserved2[16]; +}; + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_name_filter; /* bit value 1 indicates not-present */ + __u8 h_shared_count; + __u8 h_reserved2[7]; + __le32 h_shared_xattrs[]; /* shared xattr id array */ +}; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* + * bit 7 of e_name_index is set when it refers to a long xattr name prefix, + * while the remained lower bits represent the index of the prefix. + */ +#define EROFS_XATTR_LONG_PREFIX 0x80 +#define EROFS_XATTR_LONG_PREFIX_MASK 0x7f + +#define EROFS_XATTR_FILTER_BITS 32 +#define EROFS_XATTR_FILTER_DEFAULT UINT32_MAX +#define EROFS_XATTR_FILTER_SEED 0x25BBE08F + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_size; /* size of attribute value */ + /* followed by e_name and e_value */ + char e_name[]; /* attribute name */ +}; + +/* long xattr name prefix */ +struct erofs_xattr_long_prefix { + __u8 base_index; /* short xattr name prefix index */ + char infix[]; /* infix apart from short prefix */ +}; + +static inline unsigned int erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + return sizeof(struct erofs_xattr_ibody_header) + + sizeof(__u32) * (le16_to_cpu(i_xattr_icount) - 1); +} + +#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry)) + +static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) +{ + return EROFS_XATTR_ALIGN(sizeof(struct erofs_xattr_entry) + + e->e_name_len + le16_to_cpu(e->e_value_size)); +} + +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR -1 + +/* 4-byte block address array */ +#define EROFS_BLOCK_MAP_ENTRY_SIZE sizeof(__le32) + +/* 8-byte inode chunk index */ +struct erofs_inode_chunk_index { + __le16 startblk_hi; /* starting block number MSB */ + __le16 device_id; /* back-end storage id (with bits masked) */ + __le32 startblk_lo; /* starting block number of this chunk */ +}; + +#define EROFS_DIRENT_NID_METABOX_BIT 63 +#define EROFS_DIRENT_NID_MASK (BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT) - 1) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; /* node number */ + __le16 nameoff; /* start offset of file name */ + __u8 file_type; /* file type */ + __u8 reserved; /* reserved */ +} __packed; + +/* + * EROFS file types should match generic FT_* types and + * it seems no need to add BUILD_BUG_ONs since potential + * unmatchness will break other fses as well... + */ + +#define EROFS_NAME_LEN 255 + +/* maximum supported encoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) + +/* maximum supported decoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) + +/* available compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_LZMA = 1, + Z_EROFS_COMPRESSION_DEFLATE = 2, + Z_EROFS_COMPRESSION_ZSTD = 3, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS ((1 << Z_EROFS_COMPRESSION_MAX) - 1) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + u8 reserved[10]; +} __packed; + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lzma_cfgs { + __le32 dict_size; + __le16 format; + u8 reserved[8]; +} __packed; + +#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE) + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_deflate_cfgs { + u8 windowbits; /* 8..15 for DEFLATE */ + u8 reserved[5]; +} __packed; + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_zstd_cfgs { + u8 format; + u8 windowlog; /* windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN(10) */ + u8 reserved[4]; +} __packed; + +#define Z_EROFS_ZSTD_MAX_DICT_SIZE Z_EROFS_PCLUSTER_MAX_SIZE + +/* + * Enable COMPACTED_2B for EROFS_INODE_COMPRESSED_COMPACT inodes: + * 4B (disabled) vs 4B+2B+4B (enabled) + */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +/* Enable extent metadata for EROFS_INODE_COMPRESSED_FULL inodes */ +#define Z_EROFS_ADVISE_EXTENTS 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +/* Indicate the record size for each extent if extent metadata is used */ +#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 +#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 + +#define Z_EROFS_FRAGMENT_INODE_BIT 7 +struct z_erofs_map_header { + union { + /* fragment data offset in the packed inode */ + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + /* indicates the encoded size of tailpacking data */ + __le16 h_idata_size; + }; + __le32 h_extents_lo; /* extent count LSB */ + }; + __le16 h_advise; + union { + struct { + /* algorithm type (bit 0-3: HEAD1; bit 4-7: HEAD2) */ + __u8 h_algorithmtype; + /* + * bit 0-3 : logical cluster bits - blkszbits + * bit 4-6 : reserved + * bit 7 : pack the whole file into packed inode + */ + __u8 h_clusterbits; + } __packed; + __le16 h_extents_hi; /* extent count MSB */ + } __packed; +}; + +enum { + Z_EROFS_LCLUSTER_TYPE_PLAIN = 0, + Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1, + Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3, + Z_EROFS_LCLUSTER_TYPE_MAX +}; + +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) + +/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ +#define Z_EROFS_LI_PARTIAL_REF (1 << 15) + +/* Set on 1st non-head lcluster to store compressed block counti (in blocks) */ +#define Z_EROFS_LI_D0_CBLKCNT (1 << 11) + +struct z_erofs_lcluster_index { + __le16 di_advise; + /* where to decompress in the head lcluster */ + __le16 di_clusterofs; + + union { + __le32 blkaddr; /* for the HEAD lclusters */ + /* + * [0] - distance to its HEAD lcluster + * [1] - distance to the next HEAD lcluster + */ + __le16 delta[2]; /* for the NONHEAD lclusters */ + } di_u; +}; + +#define Z_EROFS_MAP_HEADER_END(end) \ + (ALIGN(end, 8) + sizeof(struct z_erofs_map_header)) +#define Z_EROFS_FULL_INDEX_START(end) (Z_EROFS_MAP_HEADER_END(end) + 8) + +#define Z_EROFS_EXTENT_PLEN_PARTIAL BIT(27) +#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 +#define Z_EROFS_EXTENT_PLEN_MASK ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) +struct z_erofs_extent { + __le32 plen; /* encoded length */ + __le32 pstart_lo; /* physical offset */ + __le32 pstart_hi; /* physical offset MSB */ + __le32 lstart_lo; /* logical offset */ + __le32 lstart_hi; /* logical offset MSB (>= 4GiB inodes) */ + __u8 reserved[12]; /* for future use */ +}; + +static inline int z_erofs_extent_recsize(unsigned int advise) +{ + return 4 << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & + Z_EROFS_ADVISE_EXTRECSZ_MASK); +} + +/* check the EROFS on-disk layout strictly at compile time */ +static inline void erofs_check_ondisk_layout_definitions(void) +{ + const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) { + .h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT + }; + + BUILD_BUG_ON(sizeof(struct erofs_super_block) != 144); + BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32); + BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64); + BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12); + BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_lcluster_index) != 8); + BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12); + /* keep in sync between 2 index structures for better extendibility */ + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != + sizeof(struct z_erofs_lcluster_index)); + BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128); + + /* exclude old compiler versions like gcc 7.5.0 */ + BUILD_BUG_ON(__builtin_constant_p(fmh) ? + fmh != cpu_to_le64(1ULL << 63) : 0); +} + +#endif diff --git a/erofs_vnops.c b/erofs_vnops.c new file mode 100644 index 0000000..e7b0552 --- /dev/null +++ b/erofs_vnops.c @@ -0,0 +1,681 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if WITH_EROFS_POSIX_ACL +#include +#endif + +#include +#include + +#include "internal.h" +#include "xattr.h" + +static vop_inactive_t erofs_inactive; +static vop_reclaim_t erofs_reclaim; + +static vop_readdir_t erofs_readdir; +static vop_readlink_t erofs_readlink; + +static vop_open_t erofs_open; +static vop_read_t erofs_read; +static vop_bmap_t erofs_bmap; +static vop_ioctl_t erofs_ioctl; + +static vop_getattr_t erofs_getattr; +static vop_setattr_t erofs_setattr; +static vop_access_t erofs_access; +static vop_pathconf_t erofs_pathconf; + +static vop_getextattr_t erofs_getextattr; +static vop_listextattr_t erofs_listextattr; +static vop_deleteextattr_t erofs_deleteextattr; +static vop_setextattr_t erofs_setextattr; +#if WITH_EROFS_POSIX_ACL +static vop_getacl_t erofs_vop_getacl; +static vop_aclcheck_t erofs_aclcheck; +static vop_setacl_t erofs_setacl; +#endif + +/* vop_fhtovp removed in FreeBSD 15.0 */ +static vop_vptofh_t erofs_vptofh; + +/* + * Access check: data nodes are read-only, but device/FIFO nodes are not + * denied writes. + */ +static int +erofs_access(struct vop_access_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; +#if WITH_EROFS_POSIX_ACL + struct acl *acl; +#endif + accmode_t accmode; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + accmode = ap->a_accmode; + if ((accmode & VMODIFY_PERMS) != 0) { + switch (vp->v_type) { + case VDIR: + case VLNK: + case VREG: + return (EROFS); + default: + break; + } + } + error = vfs_unixify_accmode(&accmode); + if (error != 0) + return (error); +#if WITH_EROFS_POSIX_ACL + if ((vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); + + acl = acl_alloc(M_WAITOK); + error = erofs_get_acl(vp, ACL_TYPE_ACCESS, acl); + if (error == 0) + error = vaccess_acl_posix1e(vp->v_type, vi->uid, vi->gid, acl, + accmode, ap->a_cred); + acl_free(acl); + return (error); +#else + return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid, + vi->gid, accmode, ap->a_cred)); +#endif +} + +/* + * Tell the generic pager that EROFS does not provide block-level bmap. + * + * Returning EOPNOTSUPP prevents the pager from assuming a bufobj/strategy is + * available, avoiding "No strategy for buffer" errors. VM will correctly + * fall back to the VOP_READ-based page-in path. The default async VOP wrapper + * must remain in use so synchronous fallback completion still calls iodone. + */ +static int +erofs_bmap(struct vop_bmap_args *ap) +{ + if (ap->a_bop != NULL) + *ap->a_bop = NULL; + if (ap->a_bnp != NULL) + *ap->a_bnp = -1; + if (ap->a_runp != NULL) + *ap->a_runp = 0; + if (ap->a_runb != NULL) + *ap->a_runb = 0; + return (EOPNOTSUPP); +} + +/* No dirty writeback on last ref release, so inactive is a no-op. */ +static int +erofs_inactive(struct vop_inactive_args *ap __unused) +{ + return (0); +} + +static int +erofs_open(struct vop_open_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + int error; + + vp = ap->a_vp; + vi = VTOE(vp); + if (VN_ISDEV(vp)) + return (EOPNOTSUPP); + if (vp->v_type == VREG) { + error = vnode_create_vobject(vp, vi->size, ap->a_td); + return (error); + } + return (0); +} + +static int +erofs_getattr(struct vop_getattr_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + struct erofs_sb_info *sbi; + struct vattr *vap; + + vp = ap->a_vp; + vi = VTOE(vp); + sbi = MTOE(vp->v_mount); + vap = ap->a_vap; + VATTR_NULL(vap); + /* The image has no creation time; keep the vop_stat() default. */ + vap->va_birthtime.tv_sec = -1; + vap->va_birthtime.tv_nsec = 0; + vap->va_type = vp->v_type; + vap->va_mode = vi->mode & ALLPERMS; + vap->va_nlink = vi->nlink; + vap->va_uid = vi->uid; + vap->va_gid = vi->gid; + vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0]; + vap->va_fileid = erofs_nid_to_ino64(sbi, vi->nid); + vap->va_size = vi->size; + vap->va_blocksize = sbi->block_size; + vap->va_atime.tv_sec = vi->mtime; + vap->va_mtime.tv_sec = vi->mtime; + vap->va_ctime.tv_sec = vi->mtime; + vap->va_atime.tv_nsec = vi->mtime_nsec; + vap->va_mtime.tv_nsec = vi->mtime_nsec; + vap->va_ctime.tv_nsec = vi->mtime_nsec; + vap->va_gen = vi->generation; + vap->va_flags = 0; + vap->va_rdev = VN_ISDEV(vp) ? vi->rdev : NODEV; + /* A 48-bit block count of 64 KiB blocks, or about size bytes. */ + vap->va_bytes = vi->data_blocks << sbi->blkszbits; + vap->va_filerev = 0; + return (0); +} + +/* + * Read-only xattr get entry point. + * + * Delegates to erofs_getxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes) + */ +static int +erofs_getextattr(struct vop_getextattr_args *ap __unused) +{ +#if WITH_EROFS_XATTR + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER && + !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + if (ap->a_name == NULL || ap->a_name[0] == '\0') + return (EINVAL); + if (strlen(ap->a_name) > EXTATTR_MAXNAMELEN) + return (EINVAL); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_getxattr(ap->a_vp, ap->a_attrnamespace, ap->a_name, + ap->a_uio, ap->a_size)); +#else + return (EOPNOTSUPP); +#endif +} + +/* + * Read-only xattr list entry point. + * + * Delegates to erofs_listxattr() for two namespaces: + * - EXTATTR_NAMESPACE_USER + * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes) + */ +static int +erofs_listextattr(struct vop_listextattr_args *ap __unused) +{ +#if WITH_EROFS_XATTR + int error; + + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER && + !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER)) + return (EOPNOTSUPP); + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, + ap->a_td, VREAD); + if (error != 0) + return (error); + + switch (ap->a_attrnamespace) { + case EXTATTR_NAMESPACE_USER: + case EXTATTR_NAMESPACE_SYSTEM: + break; + default: + return (EOPNOTSUPP); + } + + return (erofs_listxattr(ap->a_vp, ap->a_attrnamespace, ap->a_uio, + ap->a_size)); +#else + return (EOPNOTSUPP); +#endif +} + +static int +erofs_deleteextattr(struct vop_deleteextattr_args *ap __unused) +{ +#if !WITH_EROFS_XATTR + return (EOPNOTSUPP); +#else + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +#endif +} + +static int +erofs_setextattr(struct vop_setextattr_args *ap __unused) +{ +#if !WITH_EROFS_XATTR + return (EOPNOTSUPP); +#else + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +#endif +} + +/* + * EROFS is read-only; mutations on regular files/dirs/symlinks are denied, size + * changes on special vnodes are treated as no-ops per read-only convention. + */ +static int +erofs_setattr(struct vop_setattr_args *ap) +{ + struct vnode *vp; + struct vattr *vap; + + vp = ap->a_vp; + vap = ap->a_vap; + if (vap->va_mode != (mode_t)VNOVAL || vap->va_uid != (uid_t)VNOVAL || + vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL || + vap->va_atime.tv_nsec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL || + vap->va_mtime.tv_nsec != VNOVAL || vap->va_flags != VNOVAL) + return (EROFS); + if (vap->va_size != VNOVAL) { + switch (vp->v_type) { + case VDIR: + return (EISDIR); + case VLNK: + case VREG: + return (EROFS); + case VCHR: + case VBLK: + case VSOCK: + case VFIFO: + case VNON: + case VBAD: + case VMARKER: + return (0); + } + } + return (0); +} + +static int +erofs_read(struct vop_read_args *ap) +{ + switch (ap->a_vp->v_type) { + case VREG: + return (erofs_read_file(ap->a_vp, ap->a_uio)); + case VDIR: + return (EISDIR); + default: + return (EINVAL); + } +} + +static int +erofs_ioctl(struct vop_ioctl_args *ap) +{ + struct vnode *vp; + bool seek_data; + int error; + + switch (ap->a_command) { + case FIOSEEKDATA: + seek_data = true; + break; + case FIOSEEKHOLE: + seek_data = false; + break; + default: + return (ENOTTY); + } + + vp = ap->a_vp; + error = vn_lock(vp, LK_SHARED); + if (error != 0) + return (EBADF); + error = erofs_file_llseek(vp, (off_t *)ap->a_data, seek_data); + VOP_UNLOCK(vp); + return (error); +} + +static int +erofs_readdir(struct vop_readdir_args *ap) +{ + if (ap->a_vp->v_type != VDIR) + return (ENOTDIR); + return (erofs_readdir_block(ap->a_vp, ap->a_uio, ap->a_eofflag, + ap->a_ncookies, ap->a_cookies)); +} + +static int +erofs_readlink(struct vop_readlink_args *ap) +{ + if (ap->a_vp->v_type != VLNK) + return (EINVAL); + return (erofs_readlink_target(ap->a_vp, ap->a_uio)); +} + +static int +erofs_pathconf(struct vop_pathconf_args *ap) +{ + switch (ap->a_name) { + case _PC_NAME_MAX: + *ap->a_retval = EROFS_NAME_LEN; + return (0); + case _PC_PATH_MAX: + *ap->a_retval = PATH_MAX; + return (0); + case _PC_FILESIZEBITS: + *ap->a_retval = 64; + return (0); + case _PC_LINK_MAX: + *ap->a_retval = INT_MAX; + return (0); + case _PC_CHOWN_RESTRICTED: + case _PC_NO_TRUNC: + *ap->a_retval = 1; + return (0); +#if WITH_EROFS_POSIX_ACL + case _PC_ACL_EXTENDED: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? 1 : 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = + ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? + ACL_MAX_ENTRIES : 3; + return (0); + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); +#else + case _PC_ACL_EXTENDED: + case _PC_ACL_NFS4: + *ap->a_retval = 0; + return (0); + case _PC_ACL_PATH_MAX: + *ap->a_retval = 3; + return (0); +#endif + default: + return (vop_stdpathconf(ap)); + } +} + +#if WITH_EROFS_POSIX_ACL +static int +erofs_vop_getacl(struct vop_getacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + return (erofs_get_acl(ap->a_vp, ap->a_type, ap->a_aclp)); +} + +static int +erofs_aclcheck(struct vop_aclcheck_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0) + return (EOPNOTSUPP); + if (ap->a_aclp == NULL) + return (EINVAL); + switch (ap->a_type) { + case ACL_TYPE_ACCESS: + break; + case ACL_TYPE_DEFAULT: + if (ap->a_vp->v_type != VDIR) + return (EINVAL); + break; + default: + return (EINVAL); + } + return (acl_posix1e_check(ap->a_aclp)); +} + +static int +erofs_setacl(struct vop_setacl_args *ap) +{ + if (VN_ISDEV(ap->a_vp)) + return (EOPNOTSUPP); + return (EROFS); +} +#endif + +static u_int +erofs_vfs_hash(erofs_nid_t nid) +{ + + return (fnv_32_buf(&nid, sizeof(nid), FNV1_32_INIT)); +} + +static int +erofs_vfs_hash_cmp(struct vnode *vp, void *pnid) +{ + struct erofs_inode *vi; + + vi = VTOE(vp); + return (vi == NULL || vi->nid != *(erofs_nid_t *)pnid); +} + +static void +erofs_fill_vnode(struct erofs_sb_info *sbi, struct vnode *vp, + const struct erofs_inode *vi) +{ + vp->v_type = vi->vtype; + if (vp->v_type == VFIFO) + vp->v_op = &erofs_fifoops; + else + VN_LOCK_ASHARE(vp); + if (vi->nid == sbi->root_nid) + vp->v_vflag |= VV_ROOT; +} + +int +erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags, + struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct thread *td; + struct vnode *vp; + u_int hash; + bool shared; + int error; + + *vpp = NULL; + td = curthread; + shared = (flags & LK_TYPE_MASK) == LK_SHARED; + hash = erofs_vfs_hash(nid); + error = vfs_hash_get(mp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + sbi = MTOE(mp); + vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO); + error = getnewvnode("erofs", mp, &erofs_vnodeops, &vp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + vp->v_data = vi; + vi->nid = nid; + lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); + error = insmntque(vp, mp); + if (error != 0) { + free(vi, M_EROFS); + *vpp = NULL; + return (error); + } + error = vfs_hash_insert(vp, hash, flags, td, vpp, erofs_vfs_hash_cmp, + &nid); + if (error != 0 || *vpp != NULL) + return (error); + + error = erofs_read_inode(sbi, nid, vi); + if (error != 0) { + *vpp = NULL; + vgone(vp); + vput(vp); + return (error); + } + erofs_xattr_cache_init(vi); + erofs_fill_vnode(sbi, vp, vi); + vn_set_state(vp, VSTATE_CONSTRUCTED); + if (shared) + VOP_LOCK(vp, LK_DOWNGRADE); + *vpp = vp; + return (0); +} + +int +erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags, + struct vnode **vpp) +{ + return (erofs_vget_raw(mp, *(erofs_nid_t *)arg, flags, vpp)); +} + +int +erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp) +{ + struct erofs_sb_info *sbi; + erofs_nid_t nid; + int error; + + *vpp = NULL; + sbi = MTOE(mp); + error = erofs_ino_to_nid(sbi, ino, &nid); + if (error != 0) + return (error); + if (!erofs_nid_is_valid(sbi, nid)) + return (ESTALE); + return (erofs_vget_raw(mp, nid, flags, vpp)); +} + +static int +erofs_reclaim(struct vop_reclaim_args *ap) +{ + struct vnode *vp; + struct erofs_inode *vi; + + vp = ap->a_vp; + vi = VTOE(vp); + if (vi != NULL) { + vfs_hash_remove(vp); + erofs_xattr_cache_fini(MTOE(vp->v_mount), vi); + free(vi, M_EROFS); + vp->v_data = NULL; + } + return (0); +} + +/* Vnode pointer to persistent EROFS file handle. */ +static int +erofs_vptofh(struct vop_vptofh_args *ap) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + + vi = VTOE(ap->a_vp); + bzero(&efid, sizeof(efid)); + efid.len = sizeof(efid); + efid.nid_hi = vi->nid >> 32; + efid.nid_lo = vi->nid; + efid.gen = vi->generation; + memcpy(ap->a_fhp, &efid, sizeof(efid)); + return (0); +} + +struct vop_vector erofs_vnodeops = { + .vop_default = &default_vnodeops, + .vop_inactive = erofs_inactive, + .vop_reclaim = erofs_reclaim, + + .vop_lookup = vfs_cache_lookup, + .vop_cachedlookup = erofs_lookup, + .vop_readdir = erofs_readdir, + .vop_readlink = erofs_readlink, + + .vop_open = erofs_open, + .vop_read = erofs_read, + .vop_bmap = erofs_bmap, + .vop_ioctl = erofs_ioctl, + .vop_getpages = vnode_pager_local_getpages, + + .vop_getattr = erofs_getattr, + .vop_setattr = erofs_setattr, + .vop_access = erofs_access, + .vop_pathconf = erofs_pathconf, + + .vop_getextattr = erofs_getextattr, + .vop_listextattr = erofs_listextattr, + .vop_deleteextattr = erofs_deleteextattr, + .vop_setextattr = erofs_setextattr, +#if WITH_EROFS_POSIX_ACL + .vop_getacl = erofs_vop_getacl, + .vop_aclcheck = erofs_aclcheck, + .vop_setacl = erofs_setacl, +#endif + + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_vnodeops); + +struct vop_vector erofs_fifoops = { + .vop_default = &fifo_specops, + .vop_access = erofs_access, +#if WITH_EROFS_POSIX_ACL + .vop_aclcheck = erofs_aclcheck, +#endif + .vop_deleteextattr = erofs_deleteextattr, +#if WITH_EROFS_POSIX_ACL + .vop_getacl = erofs_vop_getacl, +#endif + .vop_getextattr = erofs_getextattr, + .vop_getattr = erofs_getattr, + .vop_listextattr = erofs_listextattr, + .vop_pathconf = erofs_pathconf, + .vop_reclaim = erofs_reclaim, +#if WITH_EROFS_POSIX_ACL + .vop_setacl = erofs_setacl, +#endif + .vop_setattr = erofs_setattr, + .vop_setextattr = erofs_setextattr, + .vop_vptofh = erofs_vptofh, +}; +VFS_VOP_VECTOR_REGISTER(erofs_fifoops); diff --git a/fileio.c b/fileio.c new file mode 100644 index 0000000..999b637 --- /dev/null +++ b/fileio.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2024, Alibaba Cloud + */ + +#include +#include +#include + +#include "internal.h" + +#if WITH_EROFS_BACKED_BY_FILE +static int +erofs_read_file_source_vnrdwr(struct erofs_device_info *dif, erofs_off_t off, + size_t len, void *out, int ioflag) +{ + size_t done, want; + ssize_t resid; + int error; + + /* The credential is taken when the file source is opened. */ + EROFS_DEBUG_ASSERT(dif->source_cred != NULL, + ("erofs: file source without a credential")); + done = 0; + while (done < len) { + want = MIN(len - done, (size_t)MAXPHYS); + /* erofs_bread_device() keeps off + len within INT64_MAX + 1. */ + error = vn_rdwr(UIO_READ, dif->devvp, (char *)out + done, + (int)want, (off_t)(off + done), UIO_SYSSPACE, ioflag, + dif->source_cred, NOCRED, &resid, curthread); + if (error != 0) + return (error); + if (resid != 0) + return (EIO); + done += want; + } + return (0); +} + +static int +erofs_read_file_source_direct(struct erofs_device_info *dif, erofs_off_t off, + size_t len, void *out) +{ + return (erofs_read_file_source_vnrdwr(dif, off, len, out, IO_DIRECT)); +} + +int +erofs_read_file_source(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out) +{ + if (test_opt(&sbi->opt, DIRECT_IO)) + return (erofs_read_file_source_direct(dif, off, len, out)); + return (erofs_read_file_source_vnrdwr(dif, off, len, out, 0)); +} +#endif diff --git a/inode.c b/inode.c new file mode 100644 index 0000000..c19d913 --- /dev/null +++ b/inode.c @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "xattr.h" + +static uint64_t +erofs_addrmask(const struct erofs_sb_info *sbi) +{ + if (erofs_sb_has_48bit(sbi)) + return ((1ULL << 48) - 1); + return (UINT32_MAX); +} + +static dev_t +erofs_decode_dev(uint32_t dev) +{ + unsigned int major, minor; + + major = (dev & 0xfff00) >> 8; + minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); + return (makedev(major, minor)); +} + +static uint32_t +erofs_inode_generation(const struct erofs_sb_info *sbi, erofs_nid_t nid, + const void *inode, size_t inode_size) +{ + uint8_t encoded_nid[sizeof(nid)]; + uint32_t generation; + + le64enc(encoded_nid, nid); + generation = fnv_32_buf(encoded_nid, sizeof(encoded_nid), + sbi->generation_seed); + generation = fnv_32_buf(inode, inode_size, generation); + return (generation != 0 ? generation : 1); +} + +static int +erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds, + uint32_t nanoseconds) +{ + /* Keep va_mtime a valid timespec; mkfs.erofs never writes this. */ + if (nanoseconds >= 1000000000) + return (EINTEGRITY); + vi->mtime = seconds; + vi->mtime_nsec = nanoseconds; + return (0); +} + +static void +erofs_set_data_blocks(const struct erofs_sb_info *sbi, struct erofs_inode *vi, + uint64_t compressed_blocks) +{ + if (erofs_inode_is_data_compressed(vi->datalayout)) + vi->data_blocks = compressed_blocks; + else if (vi->size == 0) + vi->data_blocks = 0; + else + /* size is at most OFF_MAX (checked by the caller). */ + vi->data_blocks = roundup2(vi->size, + (uint64_t)sbi->block_size) >> sbi->blkszbits; +} + +static int +erofs_validate_inline_data(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi) +{ + uint64_t image_size, inline_end, inline_off, inline_size, tail_start; + + if (vi->datalayout != EROFS_INODE_FLAT_INLINE || vi->size == 0) + return (0); + tail_start = roundup2(vi->size, (uint64_t)sbi->block_size) - + sbi->block_size; + inline_size = vi->size - tail_start; + if (__builtin_add_overflow(vi->inode_off, vi->inode_isize, + &inline_off) || + __builtin_add_overflow(inline_off, vi->xattr_isize, &inline_off) || + __builtin_add_overflow(inline_off, inline_size, &inline_end)) + return (erofs_corrupt(sbi, vi->nid, + "inline tail offset overflows")); + if ((inline_off & (sbi->block_size - 1)) + inline_size > + sbi->block_size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail of %ju bytes crosses its block", + (uintmax_t)inline_size)); + if (erofs_nid_in_metabox(vi->nid)) { + if (sbi->metabox_en == NULL || + inline_end > sbi->metabox_en->size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail ends at %ju, beyond the metabox", + (uintmax_t)inline_end)); + return (0); + } + /* The image fits its provider; see erofs_nid_is_valid(). */ + image_size = sbi->blocks << sbi->blkszbits; + if (inline_end > image_size) + return (erofs_corrupt(sbi, vi->nid, + "inline tail ends at %ju, beyond the image", + (uintmax_t)inline_end)); + return (0); +} + +/* + * Convert a logical nid to its inode-table byte offset. Normal NIDs are + * relative to the primary metadata area. For metabox NIDs, bit 63 selects + * the metabox backing inode and the remaining bits are relative to its data. + * EROFS_NULL_ADDR is returned when the address cannot be represented. + */ +static erofs_off_t +erofs_iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t meta_offset; + erofs_nid_t nid_lo; + erofs_off_t result; + bool in_metabox; + + in_metabox = erofs_nid_in_metabox(nid); + if (in_metabox && !erofs_sb_has_metabox(sbi)) + return (EROFS_NULL_ADDR); + nid_lo = nid & EROFS_DIRENT_NID_MASK; + if (nid_lo > (UINT64_MAX >> 5)) + return (EROFS_NULL_ADDR); + result = nid_lo << 5; + if (in_metabox) + return (result); + + meta_offset = (uint64_t)sbi->meta_blkaddr << sbi->blkszbits; + if (result > UINT64_MAX - meta_offset) + return (EROFS_NULL_ADDR); + + return (meta_offset + result); +} + +/* + * Check that a NID can address at least one compact inode slot without + * crossing the declared primary image or metabox backing-file boundary. + */ +bool +erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + erofs_off_t image_size, off; + + off = erofs_iloc(sbi, nid); + if (off == EROFS_NULL_ADDR) + return (false); + if (erofs_nid_in_metabox(nid)) { + if (sbi->metabox_en == NULL || off > sbi->metabox_en->size) + return (false); + return (sizeof(struct erofs_inode_compact) <= + sbi->metabox_en->size - off); + } + /* + * The image fits its provider (erofs_validate_device_size()), and a + * block count has at most 48 bits, so neither the shift nor the + * provider size needs a check here. + */ + image_size = sbi->blocks << sbi->blkszbits; + return (off <= image_size && + sizeof(struct erofs_inode_compact) <= image_size - off); +} + +int +erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi) +{ + struct erofs_inode_compact *dic; + struct erofs_inode_extended *die; + struct erofs_inode_chunk_info chunk_info; + union erofs_inode_i_nb inode_nb; + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint64_t addrmask; + int64_t mtime; + erofs_off_t off; + erofs_blk_t startblk; + uint64_t compressed_blocks; + uint32_t raw_rdev, startblk_lo; + uint16_t ifmt, startblk_hi; + int err; + + if (!erofs_nid_is_valid(sbi, nid)) + return (erofs_corrupt(sbi, nid, "invalid nid")); + off = erofs_iloc(sbi, nid); + EROFS_TRACE_FILL_INODE(nid, off >> sbi->blkszbits, + off & (sbi->block_size - 1)); + err = erofs_read_metadata_range(sbi, nid, off, + sizeof(struct erofs_inode_compact), &buf); + if (err != 0) + return (err); + + bzero(&vi->size, sizeof(*vi) - offsetof(struct erofs_inode, size)); + vi->nid = nid; + vi->inode_off = off; + ifmt = le16dec(buf.data); + if ((ifmt & ~EROFS_I_ALL) != 0) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported inode format %#x @ nid %ju", ifmt, + (uintmax_t)nid); + return (EOPNOTSUPP); + } + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported data layout %u @ nid %ju", + vi->datalayout, (uintmax_t)nid); + return (EOPNOTSUPP); + } + vi->compact_inode = (erofs_inode_version(ifmt) == 0); + if (!vi->compact_inode) { + erofs_put_metabuf(&buf); + err = erofs_read_metadata_range(sbi, nid, off, + sizeof(struct erofs_inode_extended), &buf); + if (err != 0) + return (err); + } + addrmask = erofs_addrmask(sbi); + startblk = EROFS_NULL_ADDR; + startblk_lo = 0; + startblk_hi = 0; + compressed_blocks = 0; + raw_rdev = 0; + bzero(&inode_nb, sizeof(inode_nb)); + dic = buf.data; + if (vi->compact_inode) { + vi->inode_isize = sizeof(struct erofs_inode_compact); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(dic->i_mode); + vi->size = le32toh(dic->i_size); + vi->uid = le16toh(dic->i_uid); + vi->gid = le16toh(dic->i_gid); + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + if (__builtin_add_overflow(sbi->epoch, + (int64_t)le32toh(dic->i_mtime), &mtime)) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "mtime overflows the superblock epoch")); + } + err = erofs_set_timestamp(vi, mtime, sbi->fixed_nsec); + if (err != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "invalid mtime nsec %u", sbi->fixed_nsec)); + } + startblk_lo = le32toh(dic->i_u.startblk_lo); + compressed_blocks = le32toh(dic->i_u.blocks_lo); + raw_rdev = le32toh(dic->i_u.rdev); + if (!S_ISDIR(vi->mode) && + ((ifmt >> EROFS_I_NLINK_1_BIT) & 0x1) != 0) { + vi->nlink = 1; + inode_nb = dic->i_nb; + } else { + vi->nlink = le16toh(dic->i_nb.nlink); + addrmask = UINT32_MAX; + } + } else { + die = buf.data; + vi->inode_isize = sizeof(struct erofs_inode_extended); + vi->generation = erofs_inode_generation(sbi, nid, buf.data, + vi->inode_isize); + vi->mode = le16toh(die->i_mode); + vi->size = le64toh(die->i_size); + vi->uid = le32toh(die->i_uid); + vi->gid = le32toh(die->i_gid); + vi->nlink = le32toh(die->i_nlink); + inode_nb = die->i_nb; + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + err = erofs_set_timestamp(vi, + (int64_t)le64toh(die->i_mtime), + le32toh(die->i_mtime_nsec)); + if (err != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "invalid mtime nsec %u", + le32toh(die->i_mtime_nsec))); + } + startblk_lo = le32toh(die->i_u.startblk_lo); + compressed_blocks = le32toh(die->i_u.blocks_lo); + raw_rdev = le32toh(die->i_u.rdev); + } + startblk_hi = le16toh(inode_nb.startblk_hi); + compressed_blocks |= (uint64_t)le16toh(inode_nb.blocks_hi) << 32; + if (vi->size > (uint64_t)OFF_MAX) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "size %ju is too large", + (uintmax_t)vi->size)); + } + + vi->vtype = IFTOVT(vi->mode); + /* A mode with an unknown file type cannot be given a vnode type. */ + if (vi->mode != 0 && vi->vtype == VNON) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "invalid mode %#o", vi->mode)); + } + if (vi->vtype == VLNK) { + if (vi->size == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "empty symlink")); + } + } + vi->dot_omitted = (vi->vtype == VDIR) && + (((ifmt >> EROFS_I_DOT_OMITTED_BIT) & 0x1) != 0); + + if (erofs_inode_is_data_compressed(vi->datalayout)) { +#if WITH_EROFS_ZIP + err = z_erofs_fill_inode(sbi, vi); +#else + err = EOPNOTSUPP; +#endif + if (err != 0) { + erofs_put_metabuf(&buf); + return (err); + } + } else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + if (vi->compact_inode) + chunk_info = dic->i_u.c; + else + chunk_info = die->i_u.c; + if (le16toh(chunk_info.reserved) != 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "reserved chunk field %#x is set", + le16toh(chunk_info.reserved))); + } + vi->chunkformat = le16toh(chunk_info.format); + if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported chunk format %#x @ nid %ju", + vi->chunkformat, (uintmax_t)nid); + return (EOPNOTSUPP); + } + if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 && + (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, + "chunk format %#x: 48-bit without chunk indexes", + vi->chunkformat)); + } + vi->chunkbits = sbi->blkszbits + + (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); + } else if (vi->datalayout != EROFS_INODE_FLAT_PLAIN && + vi->datalayout != EROFS_INODE_FLAT_INLINE) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported data layout %u @ nid %ju", + vi->datalayout, (uintmax_t)nid); + return (EOPNOTSUPP); + } + + switch (vi->vtype) { + case VREG: + case VDIR: + case VLNK: + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + } + startblk = startblk_lo | ((uint64_t)startblk_hi << 32); + if (vi->datalayout == EROFS_INODE_FLAT_PLAIN && + ((startblk ^ EROFS_NULL_ADDR) & addrmask) == 0) + startblk = EROFS_NULL_ADDR; + vi->startblk = startblk; + vi->rdev = NODEV; + break; + case VCHR: + case VBLK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = erofs_decode_dev(raw_rdev); + break; + case VFIFO: + case VSOCK: + vi->startblk = EROFS_NULL_ADDR; + vi->rdev = NODEV; + break; + default: + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, nid, "unknown file type, mode %#o", + vi->mode)); + } + erofs_set_data_blocks(sbi, vi, compressed_blocks); + err = erofs_validate_inline_data(sbi, vi); + if (err != 0) { + erofs_put_metabuf(&buf); + return (err); + } + + erofs_put_metabuf(&buf); + return (0); +} diff --git a/internal.h b/internal.h new file mode 100644 index 0000000..aef896f --- /dev/null +++ b/internal.h @@ -0,0 +1,721 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#ifndef _FS_EROFS_INTERNAL_H_ +#define _FS_EROFS_INTERNAL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "erofs_compat.h" +#include "erofs_fs.h" + +/* + * Linux checks the sizes of the on-disk structures in + * erofs_check_ondisk_layout_definitions(); these also check the offsets. + */ +_Static_assert(sizeof(struct erofs_deviceslot) == 128, + "EROFS device slot ABI size"); +_Static_assert(sizeof(struct erofs_super_block) == 144, + "EROFS super block ABI size"); +_Static_assert(sizeof(struct erofs_inode_compact) == 32, + "EROFS compact inode ABI size"); +_Static_assert(sizeof(struct erofs_inode_extended) == 64, + "EROFS extended inode ABI size"); +_Static_assert(sizeof(struct erofs_xattr_ibody_header) == 12, + "EROFS xattr ibody header ABI size"); +_Static_assert(sizeof(struct erofs_xattr_entry) == 4, + "EROFS xattr entry ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_info) == 4, + "EROFS chunk info ABI size"); +_Static_assert(sizeof(struct erofs_inode_chunk_index) == 8, + "EROFS chunk index ABI size"); +_Static_assert(sizeof(struct z_erofs_map_header) == 8, + "EROFS zmap header ABI size"); +_Static_assert(sizeof(struct z_erofs_lcluster_index) == 8, + "EROFS lcluster index ABI size"); +_Static_assert(sizeof(struct z_erofs_extent) == 32, + "EROFS compression extent ABI size"); +_Static_assert(sizeof(struct erofs_dirent) == 12, + "EROFS dirent ABI size"); +_Static_assert(sizeof(struct erofs_deviceslot) == EROFS_DEVT_SLOT_SIZE, + "EROFS device slot ABI macro"); +_Static_assert( + __builtin_offsetof(struct erofs_super_block, extra_devices) == 86, + "EROFS extra device count ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, devt_slotoff) == 88, + "EROFS device table slot offset ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, rootnid_8b) == 112, + "EROFS 48-bit root nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_super_block, metabox_nid) == 128, + "EROFS metabox nid ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_compact, i_u) == 16, + "EROFS compact inode union ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_u) == 16, + "EROFS extended inode union ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_extended, i_reserved2) == 48, + "EROFS extended inode reserved ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_chunk_index, device_id) == 2, + "EROFS chunk device id ABI offset"); +_Static_assert( + __builtin_offsetof(struct erofs_inode_chunk_index, startblk_lo) == 4, + "EROFS chunk start block ABI offset"); +_Static_assert(__builtin_offsetof(struct z_erofs_map_header, h_advise) == 4, + "EROFS zmap advise ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_lo) == 64, + "EROFS device blocks ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_lo) == 68, + "EROFS device unified address ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_hi) == 72, + "EROFS device blocks high ABI offset"); +_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_hi) == 74, + "EROFS device unified address high ABI offset"); + +MALLOC_DECLARE(M_EROFS); + +#ifndef WITH_EROFS_DEBUG +#define WITH_EROFS_DEBUG 1 +#endif + +#if WITH_EROFS_DEBUG +#define EROFS_DEBUG_ASSERT(exp, msg) KASSERT((exp), msg) +#else +#define EROFS_DEBUG_ASSERT(exp, msg) do { } while (0) +#endif + +/* + * Diagnostics. erofs_corrupt() goes where bad on-disk data is first + * noticed, not in the callers that pass the error up. It logs one + * rate-limited line (mount point, function, what was wrong, inode), fires + * the erofs:::corrupt DTrace probe (arguments: nid, source line, error) + * and returns EINTEGRITY, so a check reads + * + * return (erofs_corrupt(sbi, nid, "bad foo %u", foo)); + * + * erofs_err() is the same without the probe, for errors that are not + * corruption, such as an unsupported feature. + */ +struct erofs_sb_info; +/* The error is not about an inode. */ +#define EROFS_NID_NONE ((erofs_nid_t)-1) +void erofs_log_err(const struct erofs_sb_info *sbi, const char *func, + const char *fmt, ...) __printflike(3, 4); +int erofs_log_corrupt(const struct erofs_sb_info *sbi, uint64_t nid, + const char *func, int line, const char *fmt, ...) __printflike(5, 6); +#define erofs_err(sbi, fmt, ...) \ + erofs_log_err((sbi), __func__, fmt, ##__VA_ARGS__) +#define erofs_corrupt(sbi, nid, fmt, ...) \ + erofs_log_corrupt((sbi), (nid), __func__, __LINE__, fmt, ##__VA_ARGS__) + +struct cdev; +struct g_consumer; +struct erofs_device_info; + +typedef uint64_t erofs_nid_t; +typedef uint64_t erofs_off_t; +typedef uint64_t erofs_blk_t; + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) +#error EROFS requires compiler byte-order definitions +#elif __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error EROFS requires a little-endian FreeBSD kernel ABI +#endif +/* + * The module is LP64-only (see also the architecture list in the Makefile), + * so size_t, off_t, ino_t, time_t and erofs_blk_t are all 64 bits wide. + * Code below relies on this: do not add 32-bit overflow guards. + */ +_Static_assert(sizeof(uintptr_t) == sizeof(uint64_t), + "EROFS requires a 64-bit pointer ABI"); +_Static_assert(sizeof(ino_t) >= sizeof(uint64_t), + "EROFS requires a 64-bit inode number ABI"); +_Static_assert(sizeof(off_t) >= sizeof(uint64_t), + "EROFS requires a 64-bit file offset ABI"); +_Static_assert(sizeof(time_t) >= sizeof(int64_t), + "EROFS requires a 64-bit timestamp ABI"); +_Static_assert(sizeof(u_long) == sizeof(uint64_t), + "EROFS requires 64-bit FreeBSD atomic long operations"); +/* + * The block count of the image and of each device has at most 48 bits, so + * (blocks << blkszbits) fits in 64 bits. This does not hold for + * flatdev_blocks, a sum of two such counts: erofs_bread_device() checks it. + */ +_Static_assert(PAGE_SHIFT <= 16, "EROFS block sizes are at most 64 KiB"); + +#define EROFS_MOUNT_POSIX_ACL 0x00000020 +#define EROFS_MOUNT_XATTR_USER 0x00000080 +#define EROFS_MOUNT_DIRECT_IO 0x00000100 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +struct erofs_sb_info_opts { + unsigned int mount_opt; + uint8_t cache_strategy; + uint8_t worker_queue_priority; +}; + +enum erofs_sync_decompress { + EROFS_SYNC_DECOMPRESS_AUTO, + EROFS_SYNC_DECOMPRESS_FORCE_ON, + EROFS_SYNC_DECOMPRESS_FORCE_OFF, +}; + +enum erofs_zip_cache_strategy { + EROFS_ZIP_CACHE_DISABLED, + EROFS_ZIP_CACHE_READAHEAD, + EROFS_ZIP_CACHE_READAROUND, +}; + +struct erofs_device_handle; + +enum erofs_source_kind { + EROFS_SOURCE_NONE, + EROFS_SOURCE_BLOCK, + EROFS_SOURCE_FILE, +}; + +struct erofs_device_info { + struct vnode *devvp; + struct cdev *dev; + struct g_consumer *cp; + struct erofs_device_handle *handle; + struct ucred *source_cred; + enum erofs_source_kind source_kind; + uint64_t mediasize; + uint32_t sectorsize; + uint64_t fsoff; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +static inline bool +erofs_source_is_open(const struct erofs_device_info *dif) +{ + return (dif != NULL && dif->devvp != NULL && dif->handle != NULL && + (dif->source_kind == EROFS_SOURCE_BLOCK || + dif->source_kind == EROFS_SOURCE_FILE)); +} + +static inline bool +erofs_source_is_file(const struct erofs_device_info *dif) +{ + return (erofs_source_is_open(dif) && + dif->source_kind == EROFS_SOURCE_FILE); +} + +struct erofs_xattr_prefix_item { + uint8_t base_index; + uint8_t infix_len; + char *infix; +}; + +struct erofs_map_blocks { + erofs_off_t m_pa, m_la; + uint64_t m_plen, m_llen; + + unsigned short m_deviceid; + uint8_t m_algorithmformat; + unsigned int m_flags; +}; + +enum erofs_zextent_cache_state { + EROFS_ZCACHE_EMPTY, + EROFS_ZCACHE_INFLIGHT, + EROFS_ZCACHE_READY, + EROFS_ZCACHE_FAILED, +}; + +#define EROFS_ZCACHE_MAX_ENTRIES 8 + +struct erofs_zextent_key { + struct erofs_map_blocks map; + erofs_nid_t nid; + size_t decoded_size; +}; + +struct erofs_zextent_cache_metrics { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + uint64_t waiter_merges; + size_t resident_bytes; +}; + +struct erofs_cache_stats { + uint64_t hits; + uint64_t misses; + uint64_t bypasses; + uint64_t evictions; + uint64_t reclaims; + uint64_t waiter_merges; + uint64_t entries; + uint64_t bytes; +}; + +struct erofs_zextent_cache_entry { + struct erofs_zextent_key key; + void *data; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_zextent_cache_state state; + struct cv cv; + uint64_t lru; + bool active; +}; + +struct erofs_zextent_cache { + struct erofs_zextent_cache_entry entries[EROFS_ZCACHE_MAX_ENTRIES]; + size_t charged_bytes; + size_t budget_bytes; + uint64_t minimum_decode_work; + struct erofs_zextent_cache_metrics metrics[Z_EROFS_COMPRESSION_MAX]; + uint64_t lru_clock; + unsigned int entry_count; + bool closing; +}; + +struct erofs_decompress_task { + struct task task; + struct mtx lock; + struct cv cv; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks map; + size_t decoded_len; + bool partial; + void *decoded; + int error; + bool done; +}; + +enum erofs_xattr_cache_state { + EROFS_XATTR_CACHE_EMPTY, + EROFS_XATTR_CACHE_INFLIGHT, + EROFS_XATTR_CACHE_READY, + EROFS_XATTR_CACHE_FAILED, +}; + +struct erofs_xattr_cache { + struct mtx lock; + struct cv cv; + void *data; + size_t charged_bytes; + unsigned int waiters; + int error; + enum erofs_xattr_cache_state state; + bool closing; + bool initialized; +}; + +#define EROFS_STREAM_CTX_WRAPPER_SIZE 256 + +struct erofs_stream_ctx; +typedef int erofs_stream_ctx_init_t(struct erofs_stream_ctx *ctx); +typedef void erofs_stream_ctx_fini_t(struct erofs_stream_ctx *ctx); + +struct erofs_stream_ctx { + STAILQ_ENTRY(erofs_stream_ctx) link; + struct erofs_sb_info *sbi; + erofs_stream_ctx_fini_t *fini; + size_t charged_bytes; + uint8_t algorithm; + bool cached; + bool allocation_failed; +}; + +STAILQ_HEAD(erofs_stream_ctx_head, erofs_stream_ctx); + +struct erofs_stream_pool_codec { + struct erofs_stream_ctx_head idle; + unsigned int contexts; + unsigned int cached; + unsigned int borrowed; + unsigned int idle_count; +}; + +struct erofs_stream_pool { + struct erofs_stream_pool_codec codec[Z_EROFS_COMPRESSION_MAX]; + size_t resident_bytes; + unsigned int waiters; + struct cv cv; + bool closing; +}; + +struct erofs_sb_info { + struct mount *mnt; + struct erofs_device_info dif0; + struct mtx control_lock; + struct sysctl_ctx_list sysctl_ctx; + uint64_t control_id; + uint64_t dir_ra_bytes; + uint64_t dir_ra_max_bytes; + uint64_t dir_ra_requests; + uint64_t dir_ra_blocks; + bool control_initialized; + bool sysctl_registered; + bool control_closing; + + uint32_t block_size; + uint32_t sb_size; + uint8_t blkszbits; + uint32_t meta_blkaddr; + uint32_t xattr_blkaddr; + uint32_t xattr_prefix_start; + uint8_t xattr_prefix_count; + uint8_t xattr_filter_reserved; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + struct erofs_inode *metabox_en; + struct erofs_inode *packed_inode; + struct erofs_xattr_prefix_item *xattr_prefixes; + erofs_blk_t blocks; + uint64_t inos; + erofs_nid_t root_nid; + int64_t epoch; + uint32_t fixed_nsec; + uint32_t generation_seed; + uint32_t feature_compat; + uint32_t feature_incompat; + + struct erofs_sb_info_opts opt; + uint16_t available_compr_algs; + uint32_t lzma_dict_size; + uint8_t zstd_windowlog; + unsigned int sync_decompress; + + /* Device table */ + uint16_t extra_devices; + uint16_t device_id_mask; + bool flatdev; + erofs_blk_t total_blocks; + erofs_blk_t flatdev_blocks; + struct erofs_device_info *devs; + struct mtx z_extent_cache_lock; + struct erofs_zextent_cache z_extent_cache; + LIST_ENTRY(erofs_sb_info) z_extent_cache_link; + bool z_extent_cache_initialized; + struct taskqueue *decompress_taskqueue; + struct cv decompress_taskqueue_cv; + unsigned int decompress_taskqueue_users; + unsigned int decompress_worker_count; + bool decompress_worker_cpu_bound; + uint8_t decompress_queue_priority; + uint64_t decompress_submissions; + uint64_t decompress_worker_runs; + uint64_t decompress_queue_fallbacks; + struct erofs_stream_pool stream_pool; + LIST_ENTRY(erofs_sb_info) stream_pool_link; + bool stream_pool_initialized; + volatile u_long xattr_cache_resident; +}; + +#define MTOE(mp) ((struct erofs_sb_info *)(mp)->mnt_data) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ + static inline bool \ + erofs_sb_has_##name(const struct erofs_sb_info *sbi) \ + { \ + return ( \ + (sbi->feature_##compat & EROFS_FEATURE_##feature) != 0); \ + } + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) + +static inline bool +erofs_sb_has_xattr_filter_v1(const struct erofs_sb_info *sbi) +{ + return (erofs_sb_has_xattr_filter(sbi) && + sbi->xattr_filter_reserved == 0); +} + +struct erofs_inode { + erofs_nid_t nid; + uint64_t size; + uint64_t data_blocks; + /* Byte offset in the device, or in the metabox for a metabox nid. */ + erofs_off_t inode_off; + erofs_blk_t startblk; + uint32_t generation; + uint32_t nlink; + uid_t uid; + gid_t gid; + mode_t mode; + __enum_uint8(vtype) vtype; + dev_t rdev; + time_t mtime; + uint32_t mtime_nsec; + uint8_t datalayout; + uint8_t inode_isize; + uint32_t xattr_isize; + bool compact_inode; + bool dot_omitted; + /* Compression fields */ + uint16_t z_advise; + uint8_t z_algorithmtype[2]; + uint8_t z_lclusterbits; + uint16_t z_idata_size; + erofs_off_t z_fragmentoff; + uint64_t z_tailextent_headlcn; + uint64_t z_extents; + bool z_initialized; + /* Chunk-based fields */ + uint16_t chunkformat; + uint8_t chunkbits; + /* Fragment fields */ + struct erofs_xattr_cache xattr_cache; +}; + +struct erofs_fid { + uint16_t len; + uint16_t pad; + uint32_t nid_hi; + uint32_t nid_lo; + uint32_t gen; +}; + +_Static_assert(sizeof(struct erofs_fid) == 16, + "EROFS file handle ABI must be 16 bytes"); +_Static_assert(sizeof(struct erofs_fid) <= sizeof(struct fid), + "struct erofs_fid must fit within struct fid"); + +#define VTOE(vp) ((struct erofs_inode *)(vp)->v_data) + +static inline unsigned int +erofs_inode_version(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK); +} + +static inline unsigned int +erofs_inode_datalayout(unsigned int ifmt) +{ + return ((ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK); +} + +static inline bool +erofs_nid_in_metabox(erofs_nid_t nid) +{ + return ((nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) != 0); +} + +static inline uint64_t +erofs_nid_to_ino64(const struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + if (!erofs_sb_has_metabox(sbi)) + return (nid); + return ((nid << 1) & 0xffffffff00000000ULL) | + (nid & 0x000000007fffffffULL) | + ((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31); +} + +static inline int +erofs_ino_to_nid(const struct erofs_sb_info *sbi, ino_t ino, + erofs_nid_t *nidp) +{ + uint64_t ino64; + erofs_nid_t nid; + + ino64 = (uint64_t)ino; + if (!erofs_sb_has_metabox(sbi)) + nid = ino64; + else + nid = ((ino64 >> 1) & UINT64_C(0x7fffffff80000000)) | + (ino64 & UINT64_C(0x000000007fffffff)) | + ((ino64 & UINT64_C(0x0000000080000000)) << 32); + if (erofs_nid_to_ino64(sbi, nid) != ino64) + return (EOVERFLOW); + *nidp = nid; + return (0); +} + +/* Allocated on disk at m_pa. */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata. */ +#define EROFS_MAP_META 0x0002 +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +#define EROFS_MAP_PARTIAL_REF 0x0008 +#define EROFS_MAP_FRAGMENT 0x0010 +#define EROFS_MAP_FULL(f) \ + (!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF))) + +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +_Static_assert(Z_EROFS_COMPRESSION_RUNTIME_MAX <= UINT8_MAX, + "algorithm format must fit in uint8_t"); + +struct erofs_map_dev { + struct erofs_device_info *m_dif; + + erofs_off_t m_pa; + unsigned int m_deviceid; + uint64_t m_plen; +}; + +struct erofs_buf { + void *data; + void (*release)(void *); +}; + +#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL } + +/* Buffer and device I/O. */ +void erofs_put_metabuf(struct erofs_buf *buf); +int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, + void **bufp); +int erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id, + erofs_off_t off, size_t len, void **bufp); +void erofs_brelse(void *buf); +int erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid, + erofs_off_t off, size_t len, struct erofs_buf *buf); +/* Logical mapping and file data. */ +int erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int erofs_read_data_readahead(struct erofs_sb_info *sbi, + struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential, + void **bufp); +uint64_t erofs_dir_ra_bytes(struct erofs_sb_info *sbi); +int erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value); +void erofs_record_dir_readahead(struct erofs_sb_info *sbi, + unsigned int blocks); +int erofs_read_file(struct vnode *vp, struct uio *uio); +int erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data); +int erofs_readlink_target(struct vnode *vp, struct uio *uio); + +/* File-backed source I/O is owned by fileio.c. */ +#if WITH_EROFS_BACKED_BY_FILE +int erofs_read_file_source(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out); +#else +static inline int +erofs_read_file_source(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + erofs_off_t off, size_t len, void *out) +{ + (void)sbi; + (void)dif; + (void)off; + (void)len; + (void)out; + return (EOPNOTSUPP); +} +#endif + +/* Inode and vnode lifecycle. */ +bool erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid); +int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid, + struct erofs_inode *vi); +int erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags, + struct vnode **vpp); +int erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags, + struct vnode **vpp); +int erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp); + +/* Directory operations. */ +int erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag, + int *ncookies, uint64_t **cookies); +int erofs_dirblock_ndirents(const char *blk, uint32_t maxsize, + uint32_t *ndirentsp); +int erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents, + uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp); +int erofs_lookup(struct vop_cachedlookup_args *ap); + +/* Compressed mapping and data. */ +int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi); +int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map); +void z_erofs_extent_cache_init(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_fini(struct erofs_sb_info *sbi); +int z_erofs_extent_cache_drop(struct erofs_sb_info *sbi); +void z_erofs_extent_cache_stats(struct erofs_sb_info *sbi, + struct erofs_cache_stats *stats); +void z_erofs_stream_pool_init(struct erofs_sb_info *sbi); +void z_erofs_stream_pool_fini(struct erofs_sb_info *sbi); +int z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp); +void z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable); +int z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes); +void *z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes); +void z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address); +int z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp); +int z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio); +int z_erofs_decompress(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map, const void *src, size_t srclen, + void *dst, size_t dstlen, bool partial); +int z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, + void **bufp); +int z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len, + bool partial, void **bufp); + +/* Compression configuration. */ +int z_erofs_parse_cfgs(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb); + +#if WITH_EROFS_ZIP +int erofs_parse_zip_options(struct mount *mp, + struct erofs_sb_info_opts *opts, unsigned int *syncp, + bool *cache_presentp, bool *sync_presentp); +#endif + +int erofs_parse_dir_ra_option(struct erofs_sb_info *sbi, + struct vfsoptlist *opts); +int erofs_parse_dir_ra_value(struct erofs_sb_info *sbi, + struct vfsoptlist *opts, uint64_t *valuep, bool *presentp); +void erofs_management_init(struct erofs_sb_info *sbi); +void erofs_management_fini(struct erofs_sb_info *sbi); +int erofs_register_sysfs(struct erofs_sb_info *sbi); +int erofs_unregister_sysfs(struct erofs_sb_info *sbi); + +/* VOP vectors. */ +extern struct vop_vector erofs_vnodeops; +extern struct vop_vector erofs_fifoops; + +#endif /* _FS_EROFS_INTERNAL_H_ */ diff --git a/namei.c b/namei.c new file mode 100644 index 0000000..ac08e2b --- /dev/null +++ b/namei.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "xattr.h" + +struct erofs_qstr { + const unsigned char *name; + const unsigned char *end; +}; + +/* + * Compare two directory entry names using an already-matched prefix. + * qn: search key (not necessarily null-terminated). + * qd: on-disk name range (may not be null-terminated). + * matched: in/out count of prefix characters already known to match. + * + * Returns 0 if equal, 1 if qn > qd, -1 if qn < qd. + */ +static int +erofs_dirnamecmp(const struct erofs_qstr *qn, const struct erofs_qstr *qd, + unsigned int *matched) +{ + size_t dname_span, qname_span; + unsigned int i; + + qname_span = qn->end - qn->name; + dname_span = qd->end - qd->name; + i = MIN(*matched, qname_span); + i = MIN(i, dname_span); + while (i < qname_span && i < dname_span && qd->name[i] != '\0') { + if (qn->name[i] != qd->name[i]) { + *matched = i; + return (qn->name[i] > qd->name[i] ? 1 : -1); + } + ++i; + } + *matched = i; + if (i == qname_span) + return (i == dname_span || qd->name[i] == '\0' ? 0 : -1); + return (1); +} + +/* + * Name of the idx'th dirent of a block, as a range inside the block. Only + * the bounds are checked; see erofs_dirent_span(). + */ +static int +erofs_dirent_qstr(const char *data, uint32_t datasize, uint32_t ndirents, + uint32_t idx, struct erofs_qstr *name) +{ + uint32_t endoff, nameoff; + int error; + + error = erofs_dirent_span(data, datasize, ndirents, idx, &nameoff, + &endoff); + if (error != 0) + return (error); + name->name = (const unsigned char *)data + nameoff; + name->end = (const unsigned char *)data + endoff; + return (0); +} + +/* + * Binary search within a directory block for the target name. + * + * Sets *dep to the matching dirent, or NULL on a miss. Only the entries + * the search visits are looked at, and each is bounds-checked as it is + * visited, so an unsorted or otherwise odd block makes the search miss but + * cannot take it outside the block. + */ +static int +find_target_dirent(const struct erofs_qstr *name, char *data, + uint32_t datasize, uint32_t ndirents, struct erofs_dirent **dep) +{ + uint32_t head, back; + unsigned int startprfx, endprfx; + struct erofs_dirent *const de = (struct erofs_dirent *)data; + int error; + + *dep = NULL; + /* The 1st dirent has already been evaluated by the caller. */ + head = 1; + back = ndirents - 1; + startprfx = endprfx = 0; + + while (head <= back) { + const uint32_t mid = head + (back - head) / 2; + unsigned int matched = MIN(startprfx, endprfx); + struct erofs_qstr dname; + int ret; + + error = erofs_dirent_qstr(data, datasize, ndirents, mid, + &dname); + if (error != 0) + return (error); + + /* String comparison without already matched prefix */ + ret = erofs_dirnamecmp(name, &dname, &matched); + + if (ret == 0) { + *dep = de + mid; + return (0); + } else if (ret > 0) { + head = mid + 1; + startprfx = matched; + } else { + back = mid - 1; + endprfx = matched; + } + } + + return (0); +} + +static int +erofs_read_dirblock(struct erofs_sb_info *sbi, struct erofs_inode *dir, + uint64_t block, char **datap, uint32_t *datasizep, uint32_t *ndirentsp) +{ + erofs_off_t block_off; + uint32_t datasize; + char *data; + int error; + + *datap = NULL; + /* The only caller searches the blocks that hold dir->size bytes. */ + block_off = block * sbi->block_size; + EROFS_DEBUG_ASSERT(block_off < dir->size, + ("erofs: directory block %ju beyond the size", (uintmax_t)block)); + datasize = MIN((uint64_t)sbi->block_size, dir->size - block_off); + error = erofs_read_data(sbi, dir, block_off, datasize, (void **)&data); + if (error != 0) + return (error); + error = erofs_dirblock_ndirents(data, datasize, ndirentsp); + if (error != 0) { + erofs_brelse(data); + return (erofs_corrupt(sbi, dir->nid, + "invalid dirent array in block %ju", (uintmax_t)block)); + } + *datap = data; + *datasizep = datasize; + return (0); +} + +/* + * Find the directory block most likely to contain the target name. + * + * Uses two-level binary search: first across blocks, then within the + * candidate block via find_target_dirent(). + * + * Returns the block buffer on success (caller must erofs_brelse), + * or NULL if no block matches, in which case *errorp is 0, or on error, + * when *errorp is set to a positive errno. *_ndirents is set to the number + * of dirents in the returned block (0 means the first entry is the match). + */ +static char * +erofs_find_target_block(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, uint32_t *_ndirents, uint32_t *_datasize, + int *errorp) +{ + uint32_t bsz = sbi->block_size; + uint64_t head, back; + unsigned int startprfx = 0, endprfx = 0; + char *candidate = NULL; + int error; + + *errorp = 0; + *_ndirents = 0; + *_datasize = 0; + + if (dir->size == 0) + return (NULL); + + head = 0; + back = (dir->size - 1) / bsz; + + while (head <= back) { + const uint64_t mid = head + (back - head) / 2; + uint32_t maxsize; + char *blk; + int diff; + uint32_t ndirents; + unsigned int matched; + struct erofs_qstr dname; + + error = erofs_read_dirblock(sbi, dir, mid, &blk, &maxsize, + &ndirents); + if (error != 0) { + *errorp = error; + goto out; + } + error = erofs_dirent_qstr(blk, maxsize, ndirents, 0, &dname); + if (error != 0) { + erofs_brelse(blk); + *errorp = erofs_corrupt(sbi, dir->nid, + "invalid name span of the first dirent in block " + "%ju", + (uintmax_t)mid); + goto out; + } + + matched = MIN(startprfx, endprfx); + /* String comparison without already matched prefix */ + diff = erofs_dirnamecmp(name, &dname, &matched); + + if (diff < 0) { + erofs_brelse(blk); + if (mid == 0) + break; + back = mid - 1; + endprfx = matched; + continue; + } + + /* diff >= 0: this block is a candidate. */ + if (candidate != NULL) + erofs_brelse(candidate); + candidate = blk; + if (diff == 0) { + *_ndirents = 0; + *_datasize = maxsize; + return (candidate); + } + head = mid + 1; + startprfx = matched; + *_ndirents = ndirents; + *_datasize = maxsize; + } + return (candidate); +out: + if (candidate != NULL) + erofs_brelse(candidate); + return (NULL); +} + +/* + * Look up a name in a directory and return its nid. + */ +static int +erofs_namei(struct erofs_sb_info *sbi, struct erofs_inode *dir, + const struct erofs_qstr *name, erofs_nid_t *nid) +{ + int error; + uint32_t ndirents; + uint32_t datasize; + char *blk; + struct erofs_dirent *de; + + blk = erofs_find_target_block(sbi, dir, name, &ndirents, + &datasize, &error); + if (blk == NULL) + return (error != 0 ? error : ENOENT); + + de = (struct erofs_dirent *)blk; + if (ndirents > 0) { + error = find_target_dirent(name, blk, datasize, ndirents, &de); + if (error != 0) + error = erofs_corrupt(sbi, dir->nid, + "invalid name span in a dirent block"); + } + + if (error == 0 && de != NULL) { + erofs_nid_t found_nid; + + found_nid = le64toh(de->nid); + if (!erofs_nid_is_valid(sbi, found_nid)) { + error = erofs_corrupt(sbi, dir->nid, + "dirent points to invalid nid %ju", + (uintmax_t)found_nid); + } else { + *nid = found_nid; + } + } + erofs_brelse(blk); + if (error != 0) + return (error); + return (de != NULL ? 0 : ENOENT); +} + +/* + * Directory name lookup (VOP_CACHEDLOOKUP entry point). + * + * FreeBSD-side API requirements: + * - "." must be returned under the caller's requested lock mode; + * - ".." must go through vn_vget_ino() to avoid holding a child lock while + * acquiring the parent directory lock in reverse; + * - Both hit and miss must correctly update the namecache. + */ +int +erofs_lookup(struct vop_cachedlookup_args *ap) +{ + struct erofs_qstr qname; + struct vnode *dvp, *vp; + struct erofs_inode *dir; + struct erofs_sb_info *sbi; + struct componentname *cnp; + erofs_nid_t nid; + int error, ltype; + + dvp = ap->a_dvp; + cnp = ap->a_cnp; + *ap->a_vpp = NULL; + if ((cnp->cn_flags & ISLASTCN) != 0 && + (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) + return (EROFS); + if (cnp->cn_namelen > EROFS_NAME_LEN) + return (ENAMETOOLONG); + erofs_trace_lookup(VTOE(dvp), cnp->cn_nameptr, + (size_t)cnp->cn_namelen, (uint32_t)cnp->cn_flags); + if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') { + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + + dir = VTOE(dvp); + sbi = MTOE(dvp->v_mount); + qname.name = (const unsigned char *)cnp->cn_nameptr; + qname.end = qname.name + cnp->cn_namelen; + error = erofs_namei(sbi, dir, &qname, &nid); + if (error != 0) { + if (error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, NULL, cnp); + if (error == ENOENT && (cnp->cn_flags & ISLASTCN) != 0 && + cnp->cn_nameiop == CREATE) + return (EROFS); + return (error); + } + if ((cnp->cn_flags & ISDOTDOT) != 0 && nid == dir->nid) { + if (nid != sbi->root_nid) + return (erofs_corrupt(sbi, dir->nid, + "'..' of a non-root directory points to itself")); + vref(dvp); + ltype = cnp->cn_lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else if (ltype == LK_SHARED) + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + } + *ap->a_vpp = dvp; + return (0); + } + if ((cnp->cn_flags & ISDOTDOT) == 0 && nid == dir->nid) + return (erofs_corrupt(sbi, dir->nid, + "directory contains an entry pointing to itself")); + + if ((cnp->cn_flags & ISDOTDOT) != 0) + error = vn_vget_ino_gen(dvp, erofs_vget_raw_alloc, &nid, + cnp->cn_lkflags, &vp); + else + error = erofs_vget_raw(dvp->v_mount, nid, cnp->cn_lkflags, &vp); + if (error != 0) + return (error); + if ((cnp->cn_flags & ISDOTDOT) != 0 && VTOE(vp)->vtype != VDIR) { + vput(vp); + return (erofs_corrupt(sbi, dir->nid, + "'..' does not lead to a directory (nid %ju)", + (uintmax_t)nid)); + } + *ap->a_vpp = vp; + if ((cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, vp, cnp); + return (0); +} diff --git a/super.c b/super.c new file mode 100644 index 0000000..10cfa7d --- /dev/null +++ b/super.c @@ -0,0 +1,1643 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "trace.h" +#include "xattr.h" + +MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem"); + +static const uint32_t erofs_crc32c_seed = 0x5045b54aU; + +/* + * At most EROFS_ERR_MAXPPS lines a second for the whole module, so a bad + * directory cannot flood the log. The DTrace probe is not rate limited. + */ +#define EROFS_ERR_MAXPPS 5 +static struct timeval erofs_err_lasttime; +static int erofs_err_curpps; + +static const char * +erofs_mntonname(const struct erofs_sb_info *sbi) +{ + if (sbi == NULL || sbi->mnt == NULL) + return ("?"); + return (sbi->mnt->mnt_stat.f_mntonname); +} + +void +erofs_log_err(const struct erofs_sb_info *sbi, const char *func, + const char *fmt, ...) +{ + char msg[128]; + va_list ap; + + if (ppsratecheck(&erofs_err_lasttime, &erofs_err_curpps, + EROFS_ERR_MAXPPS) == 0) + return; + va_start(ap, fmt); + vsnprintf(msg, sizeof(msg), fmt, ap); + va_end(ap); + log(LOG_ERR, "erofs (%s): %s: %s\n", erofs_mntonname(sbi), func, msg); +} + +int +erofs_log_corrupt(const struct erofs_sb_info *sbi, uint64_t nid, + const char *func, int line, const char *fmt, ...) +{ + char msg[128]; + va_list ap; + + EROFS_TRACE_CORRUPT(nid, line, EINTEGRITY); + if (ppsratecheck(&erofs_err_lasttime, &erofs_err_curpps, + EROFS_ERR_MAXPPS) != 0) { + va_start(ap, fmt); + vsnprintf(msg, sizeof(msg), fmt, ap); + va_end(ap); + if (nid == EROFS_NID_NONE) + log(LOG_ERR, "erofs (%s): %s: %s\n", + erofs_mntonname(sbi), func, msg); + else + log(LOG_ERR, "erofs (%s): %s: %s @ nid %ju\n", + erofs_mntonname(sbi), func, msg, (uintmax_t)nid); + } + return (EINTEGRITY); +} + +static int +erofs_superblock_csum_verify(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t expected, crc; + size_t len; + int error; + + if (!erofs_sb_has_sb_chksum(sbi)) + return (0); + + len = 1u << dsb->blkszbits; + if (len > EROFS_SUPER_OFFSET) + len -= EROFS_SUPER_OFFSET; + + error = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, len, + &buf); + if (error != 0) + return (error); + + crc = calculate_crc32c(erofs_crc32c_seed, + (const uint8_t *)buf.data + + offsetof(struct erofs_super_block, checksum) + + sizeof(dsb->checksum), + len - offsetof(struct erofs_super_block, checksum) - + sizeof(dsb->checksum)); + expected = le32toh(dsb->checksum); + erofs_put_metabuf(&buf); + + if (crc != expected) { + vfs_mount_error(sbi->mnt, + "erofs: invalid superblock checksum 0x%08x, " + "0x%08x expected", crc, expected); + return (EBADMSG); + } + return (0); +} + + +static const char *erofs_opts[] = { + "acl", + "device", + "export", + "from", + "dir_ra_bytes", + "directio", + "cache_strategy", + "sync_decompress", + "worker_queue_priority", + "fsoffset", + "nodirectio", + "user_xattr", + /* Generic VFS options that other file systems accept. */ + "async", + "noatime", + "noclusterr", + "noclusterw", + "nosymfollow", + "sync", + "union", + NULL, +}; + +static vfs_mount_t erofs_mount; +static vfs_root_t erofs_root; +static vfs_statfs_t erofs_statfs; +static vfs_unmount_t erofs_unmount; +static vfs_fhtovp_t erofs_fhtovp; + +static int +erofs_parse_fsoffset(struct mount *mp, uint64_t *fsoffsetp, bool *presentp) +{ + struct vfsopt *opt; + const unsigned char *value; + uint64_t fsoffset; + size_t i; + + *fsoffsetp = 0; + *presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "fsoffset") != 0) + continue; + *presentp = true; + if (opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') + goto invalid; + value = opt->value; + fsoffset = 0; + for (i = 0; i < (size_t)opt->len - 1; ++i) { + if (value[i] < '0' || value[i] > '9') + goto invalid; + if (fsoffset > (UINT64_MAX - (value[i] - '0')) / 10) + goto overflow; + fsoffset = fsoffset * 10 + (value[i] - '0'); + } + *fsoffsetp = fsoffset; + } + return (0); + +invalid: + vfs_mount_error(mp, "erofs: fsoffset requires an unsigned byte offset"); + return (EINVAL); +overflow: + vfs_mount_error(mp, "erofs: fsoffset value is too large"); + return (ERANGE); +} + +static int +erofs_parse_directio_options(struct mount *mp, bool *enabledp, bool *presentp) +{ + struct vfsopt *opt; + + *enabledp = false; + *presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "directio") != 0 && + strcmp(opt->name, "nodirectio") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, + "erofs: directio/nodirectio take no value"); + return (EINVAL); + } +#if !WITH_EROFS_BACKED_BY_FILE + vfs_mount_error(mp, + "erofs: directio support is not built"); + return (EOPNOTSUPP); +#endif + *presentp = true; + *enabledp = strcmp(opt->name, "directio") == 0; + } + return (0); +} + +static int +erofs_parse_zip_mount_options(struct mount *mp, + struct erofs_sb_info_opts *opts __unused, unsigned int *syncp __unused, + bool *cache_presentp, bool *sync_presentp, bool *priority_presentp) +{ +#if WITH_EROFS_ZIP + struct vfsopt *opt; + int error; + + *priority_presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "worker_queue_priority") == 0) + *priority_presentp = true; + } + error = erofs_parse_zip_options(mp, opts, syncp, + cache_presentp, sync_presentp); + if (error != 0) { + vfs_mount_error(mp, + "erofs: invalid cache_strategy, sync_decompress, or " + "worker_queue_priority option"); + return (error); + } + return (0); +#else + struct vfsopt *opt; + + *cache_presentp = false; + *sync_presentp = false; + *priority_presentp = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "cache_strategy") != 0 && + strcmp(opt->name, "sync_decompress") != 0 && + strcmp(opt->name, "worker_queue_priority") != 0) + continue; + vfs_mount_error(mp, + "erofs: compression options require ZIP support"); + return (EOPNOTSUPP); + } + return (0); +#endif +} + +#define EROFS_DEVICE_OPT_PREFIX "device." + +struct erofs_device_arg { + uint16_t slot; + char *path; +}; + +static int +erofs_load_generation_seed(struct erofs_sb_info *sbi, uint32_t sb_size, + uint32_t *seedp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t seed; + int error; + + error = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, sb_size, + &buf); + if (error != 0) + return (error); + seed = fnv_32_buf(buf.data, sb_size, FNV1_32_INIT); + erofs_put_metabuf(&buf); + *seedp = seed != 0 ? seed : 1; + return (0); +} + +static void +erofs_free_device_args(struct erofs_device_arg *args, unsigned int count) +{ + unsigned int i; + + if (args == NULL) + return; + for (i = 0; i < count; ++i) + free(args[i].path, M_EROFS); + free(args, M_EROFS); +} + +static int +erofs_parse_device_slot(const char *name, uint16_t *slotp) +{ + const char *p; + unsigned int slot; + + if (strncmp(name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + return (ENOENT); + p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1; + if (*p < '1' || *p > '9') + return (EINVAL); + slot = 0; + for (; *p != '\0'; ++p) { + if (*p < '0' || *p > '9' || + slot > (unsigned int)(UINT16_MAX - (*p - '0')) / 10) + return (EINVAL); + slot = slot * 10 + (*p - '0'); + } + *slotp = slot; + return (0); +} + +static int +erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp, + unsigned int *countp) +{ + struct erofs_device_arg *args; + struct vfsopt *opt; + char name[32]; + unsigned int count, device_count, slot_count, i; + uint16_t slot; + int error; + + *argsp = NULL; + *countp = 0; + count = 0; + device_count = 0; + slot_count = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") == 0) { + if (opt->value == NULL || opt->len == 0 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, + "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (device_count == UINT16_MAX) + return (E2BIG); + ++device_count; + continue; + } + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + if (error != 0 || opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0') { + vfs_mount_error(mp, + "erofs: invalid external device option %s", + opt->name); + return (EINVAL); + } + if (slot_count == UINT16_MAX) + return (E2BIG); + ++slot_count; + } + if (device_count != 0 && slot_count != 0) { + vfs_mount_error(mp, + "erofs: device and device.N options cannot be combined"); + return (EINVAL); + } + count = device_count != 0 ? device_count : slot_count; + if (count == 0) + return (0); + + args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO); + if (device_count != 0) { + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") != 0) + continue; + args[i].slot = i + 1; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + vfs_deleteopt(mp->mnt_optnew, "device"); + } else { + i = 0; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + error = erofs_parse_device_slot(opt->name, &slot); + if (error == ENOENT) + continue; + EROFS_DEBUG_ASSERT(error == 0, + ("validated EROFS device option changed")); + args[i].slot = slot; + args[i].path = malloc(opt->len, M_EROFS, M_WAITOK); + memcpy(args[i].path, opt->value, opt->len); + ++i; + } + for (i = 0; i < count; ++i) { + unsigned int j; + + for (j = 0; j < i; ++j) { + if (args[i].slot != args[j].slot) + continue; + vfs_mount_error(mp, + "erofs: external device slot %u was " + "specified more than once", + args[i].slot); + erofs_free_device_args(args, count); + return (EINVAL); + } + } + for (i = 0; i < count; ++i) { + snprintf(name, sizeof(name), + EROFS_DEVICE_OPT_PREFIX "%u", args[i].slot); + vfs_deleteopt(mp->mnt_optnew, name); + } + } + *argsp = args; + *countp = count; + return (0); +} + +static int +erofs_reject_device_update(struct mount *mp) +{ + struct vfsopt *opt; + + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "device") != 0 && + strncmp(opt->name, EROFS_DEVICE_OPT_PREFIX, + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0) + continue; + vfs_mount_error(mp, + "erofs: external devices cannot be changed on update"); + return (EOPNOTSUPP); + } + return (0); +} + +static int +erofs_parse_acl_options(struct mount *mp, bool default_enabled, bool *enabledp) +{ + struct vfsopt *opt; + bool enabled; + + enabled = default_enabled; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "acl") != 0 && + strcmp(opt->name, "noacl") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, "erofs: acl/noacl take no value"); + return (EINVAL); + } +#if WITH_EROFS_POSIX_ACL + enabled = strcmp(opt->name, "acl") == 0; +#else + vfs_mount_error(mp, "erofs: POSIX ACL support is not built"); + return (EOPNOTSUPP); +#endif + } +#if !WITH_EROFS_POSIX_ACL + enabled = false; +#endif + *enabledp = enabled; + return (0); +} + +static int +erofs_parse_xattr_options(struct mount *mp, bool default_enabled, + bool *enabledp) +{ + struct vfsopt *opt; + bool enabled; + + enabled = default_enabled; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "user_xattr") != 0 && + strcmp(opt->name, "nouser_xattr") != 0) + continue; + if (opt->value != NULL || opt->len != 0) { + vfs_mount_error(mp, + "erofs: user_xattr/nouser_xattr take no value"); + return (EINVAL); + } +#if !WITH_EROFS_XATTR + vfs_mount_error(mp, + "erofs: extended attribute support is not built"); + return (EOPNOTSUPP); +#endif + enabled = strcmp(opt->name, "user_xattr") == 0; + } +#if !WITH_EROFS_XATTR + enabled = false; +#endif + *enabledp = enabled; + return (0); +} + +static void +erofs_set_xattr_state(struct erofs_sb_info *sbi, bool enabled) +{ + if (enabled) + set_opt(&sbi->opt, XATTR_USER); + else + clear_opt(&sbi->opt, XATTR_USER); +} + +static void +erofs_set_acl_state(struct erofs_sb_info *sbi, bool enabled) +{ + struct mount *mp; + + mp = sbi->mnt; + if (enabled) { + set_opt(&sbi->opt, POSIX_ACL); + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_ACLS; + MNT_IUNLOCK(mp); + } else { + MNT_ILOCK(mp); + mp->mnt_flag &= ~MNT_ACLS; + MNT_IUNLOCK(mp); + clear_opt(&sbi->opt, POSIX_ACL); + } +} + +struct erofs_device_handle { + struct vnode *devvp; /* block: private mntfs vnode; file: the file */ + struct vnode *odevvp; /* block: the devfs vnode, reference only */ + struct cdev *dev; + struct g_consumer *cp; + struct mount *source_mp; + struct mount_upper_node source_upper; + volatile u_int refs; +}; + +/* + * A mount reads through VOP_READ on another mount whenever any of its devices + * is file-backed, not just the primary one: FreeBSD allows a block-backed + * primary device together with file-backed extra devices. + */ +static bool +erofs_mount_has_file_source(const struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (erofs_source_is_file(&sbi->dif0)) + return (true); + for (i = 0; i < sbi->extra_devices; ++i) { + if (erofs_source_is_file(&sbi->devs[i])) + return (true); + } + return (false); +} + +/* + * The registration pins source_mp. Judge the mount that really holds the + * data: nullfs forwards VOP_READ to the vnode below it, and + * VOP_GETLOWVNODE() looks through it. unionfs does not implement that, so it + * is refused by name, as Linux refuses any stacked filesystem. + */ +static int +erofs_check_source_stacking(struct erofs_sb_info *sbi, struct vnode *devvp) +{ + struct erofs_sb_info *lower; + struct vnode *lowvp; + struct mount *lowmp; + int error; + + error = VOP_GETLOWVNODE(devvp, &lowvp, FREAD); + if (error != 0) + return (error); + lowmp = lowvp->v_mount; + error = 0; + if (lowmp == NULL) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source filesystem is being unmounted"); + error = EBUSY; + } else if (strcmp(lowmp->mnt_vfc->vfc_name, "unionfs") == 0) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed mounts cannot be applied to stacked " + "fses"); + error = ENOTBLK; + } else if (lowmp->mnt_vfc == sbi->mnt->mnt_vfc) { + lower = lowmp->mnt_data; + if (lower != NULL && erofs_mount_has_file_source(lower)) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed mounts cannot be applied to " + "file-backed EROFS mounts"); + error = ENOTBLK; + } + } + vrele(lowvp); + return (error); +} + +static int +erofs_register_source_upper(struct erofs_sb_info *sbi, + struct erofs_device_info *dif) +{ + struct mount *source_mnt, *covered_mnt; + struct mount *source_mp; + int error; + + if (!erofs_source_is_file(dif)) + return (0); + if (sbi == NULL || sbi->mnt == NULL || dif->handle == NULL || + dif->devvp == NULL) + return (EINVAL); + /* + * These run before registration because vfs_register_upper_from_vp() + * asserts that the lower mount is not the one being built, so the + * self-referential case has to be rejected first. + */ + source_mnt = dif->devvp->v_mount; + if (source_mnt == sbi->mnt) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source resolves to the mount being " + "built"); + return (EDEADLK); + } + covered_mnt = sbi->mnt->mnt_vnodecovered != NULL ? + sbi->mnt->mnt_vnodecovered->v_mount : NULL; + if (covered_mnt != NULL && source_mnt == covered_mnt && + source_mnt->mnt_vfc == sbi->mnt->mnt_vfc) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source and mount target are on the " + "same mount"); + return (ELOOP); + } + source_mp = vfs_register_upper_from_vp(dif->devvp, sbi->mnt, + &dif->handle->source_upper); + if (source_mp == NULL) { + vfs_mount_error(sbi->mnt, + "erofs: file-backed source filesystem is being unmounted"); + return (EBUSY); + } + dif->handle->source_mp = source_mp; + /* + * Registering as an upper mount pins source_mp until this mount is torn + * down, so the mounts below it (and their erofs_sb_info) stay valid for + * the check. + * + * Reading a file-backed source re-enters VOP_READ on the lower mount + * while this frame is still live. Refuse another file-backed EROFS + * level, matching Linux: a block-backed EROFS mount may still carry one + * file-backed mount, so the depth stays 2. + */ + error = erofs_check_source_stacking(sbi, dif->devvp); + if (error != 0) { + vfs_unregister_upper(source_mp, &dif->handle->source_upper); + dif->handle->source_mp = NULL; + return (error); + } + return (0); +} + +static void +erofs_destroy_device_handle(struct erofs_device_handle *handle) +{ + if (handle == NULL) + return; + if (handle->source_mp != NULL) { + vfs_unregister_upper(handle->source_mp, &handle->source_upper); + handle->source_mp = NULL; + } + if (handle->odevvp != NULL) { + /* + * Block device: close the consumer and free the private vnode + * with the vnode locked, then drop the devfs vnode reference. + */ + if (handle->devvp != NULL) { + vn_lock(handle->devvp, LK_EXCLUSIVE | LK_RETRY); + if (handle->cp != NULL) { + g_topology_lock(); + g_vfs_close(handle->cp); + g_topology_unlock(); + } + mntfs_freevp(handle->devvp); + } + vrele(handle->odevvp); + } else if (handle->devvp != NULL) { + vrele(handle->devvp); + } + if (handle->dev != NULL) + dev_rel(handle->dev); + free(handle, M_EROFS); +} + +static void +erofs_release_device_info(struct erofs_device_info *dif) +{ + struct erofs_device_handle *handle; + + handle = dif->handle; + dif->handle = NULL; + dif->cp = NULL; + dif->devvp = NULL; + dif->dev = NULL; + if (dif->source_cred != NULL) { + crfree(dif->source_cred); + dif->source_cred = NULL; + } + dif->source_kind = EROFS_SOURCE_NONE; + if (handle == NULL || !refcount_release(&handle->refs)) + return; + erofs_destroy_device_handle(handle); +} + +static struct erofs_device_handle * +erofs_find_device_handle(struct erofs_sb_info *sbi, struct g_provider *pp) +{ + unsigned int i; + + if (sbi == NULL) + return (NULL); + if (sbi->dif0.cp != NULL && sbi->dif0.cp->provider == pp) + return (sbi->dif0.handle); + for (i = 0; i < sbi->extra_devices; ++i) { + if (sbi->devs[i].cp != NULL && sbi->devs[i].cp->provider == pp) + return (sbi->devs[i].handle); + } + return (NULL); +} + +static int +erofs_open_device(struct mount *mp, struct erofs_sb_info *sbi, + const char *path, struct erofs_device_info *dif) +{ + struct erofs_device_handle *handle, *shared; + struct g_provider *pp; + struct nameidata nd; + struct vnode *devvp, *mdevvp; + struct cdev *dev; + bool reused, need_open; + int error; + + bzero(dif, sizeof(*dif)); + handle = malloc(sizeof(*handle), M_EROFS, M_WAITOK | M_ZERO); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path); + error = namei(&nd); + if (error != 0) { + free(handle, M_EROFS); + return (error); + } + devvp = nd.ni_vp; + NDFREE_PNBUF(&nd); + if (devvp->v_type == VREG) { +#if !WITH_EROFS_BACKED_BY_FILE + vput(devvp); + free(handle, M_EROFS); + return (EOPNOTSUPP); +#else + struct vattr va; + + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, + curthread); + if (error == 0) + error = VOP_GETATTR(devvp, &va, curthread->td_ucred); + if (error != 0) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + dif->source_cred = crhold(curthread->td_ucred); + handle->devvp = devvp; + refcount_init(&handle->refs, 1); + dif->handle = handle; + dif->devvp = devvp; + dif->source_kind = EROFS_SOURCE_FILE; + dif->mediasize = (uint64_t)va.va_size; + dif->sectorsize = 1; + VOP_UNLOCK(devvp); + return (0); +#endif + } + if (devvp->v_type != VCHR) { + vput(devvp); + free(handle, M_EROFS); + return (EINVAL); + } + if (!vn_isdisk_error(devvp, &error)) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread); + if (error != 0) + error = priv_check(curthread, PRIV_VFS_MOUNT_PERM); + if (error != 0) { + vput(devvp); + free(handle, M_EROFS); + return (error); + } + dev = devvp->v_rdev; + dev_ref(dev); + handle->odevvp = devvp; + handle->dev = dev; + refcount_init(&handle->refs, 1); + reused = false; + need_open = false; + g_topology_lock(); + pp = g_dev_getprovider(dev); + if (pp == NULL) + error = ENXIO; + else if ((shared = erofs_find_device_handle(sbi, pp)) != NULL) { + refcount_acquire(&shared->refs); + dif->handle = shared; + dif->cp = shared->cp; + dif->devvp = shared->devvp; + dif->dev = shared->dev; + dif->source_kind = EROFS_SOURCE_BLOCK; + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + reused = true; + error = 0; + } else { + need_open = true; + error = 0; + } + g_topology_unlock(); + if (need_open) { + /* + * Use a private vnode for the device, not the devfs one. When + * the device goes away devfs vgone()s its own vnode, and with + * it the buffer cache state that this mount is still using. + * mntfs_allocvp() unlocks the devfs vnode and returns the new + * one locked. + */ + mdevvp = mntfs_allocvp(mp, devvp); + /* + * Readahead takes this lock around breadn(), which waits for + * I/O. devfs vnodes allow shared locking, so keep that. + */ + VN_LOCK_ASHARE(mdevvp); + g_topology_lock(); + error = g_vfs_open(mdevvp, &handle->cp, "erofs", 0); + if (error == 0) { + handle->devvp = mdevvp; + dif->handle = handle; + dif->cp = handle->cp; + dif->devvp = mdevvp; + dif->dev = handle->dev; + dif->source_kind = EROFS_SOURCE_BLOCK; + dif->mediasize = dif->cp->provider->mediasize; + dif->sectorsize = dif->cp->provider->sectorsize; + } + g_topology_unlock(); + if (error == 0) + VOP_UNLOCK(mdevvp); + else + mntfs_freevp(mdevvp); + } else { + VOP_UNLOCK(devvp); + } + if (error != 0 || reused) { + erofs_destroy_device_handle(handle); + } + if (error != 0) { + return (error); + } + if (dif->sectorsize == 0 || + (dif->sectorsize & (dif->sectorsize - 1)) != 0) { + erofs_release_device_info(dif); + return (EINVAL); + } + return (0); +} + +static void +erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif) +{ + u_long iosize; + + iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ? + dif->dev->si_iosize_max : MAXPHYS; + mp->mnt_iosize_max = (int)MIN((u_long)mp->mnt_iosize_max, + MIN(iosize, (u_long)MAXPHYS)); +} + +static void +erofs_free_dev_context(struct erofs_sb_info *sbi) +{ + unsigned int i; + + if (sbi->devs != NULL) { + for (i = sbi->extra_devices; i > 0; --i) + erofs_release_device_info(&sbi->devs[i - 1]); + free(sbi->devs, M_EROFS); + } +} + +static void +erofs_drop_internal_inodes(struct erofs_sb_info *sbi) +{ + if (sbi->metabox_en != NULL) + free(sbi->metabox_en, M_EROFS); + if (sbi->packed_inode != NULL) + free(sbi->packed_inode, M_EROFS); +} + +static int +erofs_sb_free(struct erofs_sb_info *sbi) +{ + int error; + + if (sbi == NULL) + return (0); + error = erofs_unregister_sysfs(sbi); + if (error != 0) + return (error); +#if WITH_EROFS_ZIP + z_erofs_extent_cache_fini(sbi); + z_erofs_stream_pool_fini(sbi); +#endif + erofs_xattr_prefixes_cleanup(sbi); + erofs_drop_internal_inodes(sbi); + erofs_free_dev_context(sbi); + erofs_release_device_info(&sbi->dif0); + erofs_management_fini(sbi); + free(sbi, M_EROFS); + return (0); +} + +static void +erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat, + uint64_t *blocks, erofs_nid_t *root_nid) +{ + *blocks = le32toh(dsb->blocks_lo); + if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 && + dsb->rootnid_8b != 0) { + *blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32; + *root_nid = le64toh(dsb->rootnid_8b); + } else { + *root_nid = le16toh(dsb->rb.rootnid_2b); + } +} + +static int +erofs_validate_device_size(struct erofs_sb_info *sbi, + struct erofs_device_info *dif, erofs_blk_t blocks) +{ + uint64_t bytes, end; + + /* + * The sector size is a power of two (erofs_open_device()), so a block + * that is not smaller is also a multiple of it. A block count has at + * most 48 bits and blkszbits is at most 16 (internal.h), so the shift + * cannot overflow. + */ + if (sbi->block_size < dif->sectorsize) + return (EINVAL); + bytes = blocks << sbi->blkszbits; + if (__builtin_add_overflow(dif->fsoff, bytes, &end)) + return (EOVERFLOW); + if (end > dif->mediasize) + return (ENXIO); + return (0); +} + +static int +erofs_init_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif, + const char *path) +{ + struct erofs_device_info opened; + erofs_blk_t blocks, uniaddr; + int error; + + blocks = dif->blocks; + uniaddr = dif->uniaddr; + error = erofs_open_device(sbi->mnt, sbi, path, &opened); + if (error != 0) + return (error); + erofs_update_iosize_max(sbi->mnt, &opened); + opened.blocks = blocks; + opened.uniaddr = uniaddr; + error = erofs_validate_device_size(sbi, &opened, opened.blocks); + if (error != 0) { + erofs_release_device_info(&opened); + return (error); + } + error = erofs_register_source_upper(sbi, &opened); + if (error != 0) { + erofs_release_device_info(&opened); + return (error); + } + *dif = opened; + return (0); +} + +static const char * +erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count, + unsigned int slot) +{ + unsigned int i; + + for (i = 0; i < count; ++i) { + if (args[i].slot == slot) + return (args[i].path); + } + return (NULL); +} + +static int +erofs_scan_devices(struct erofs_sb_info *sbi, + const struct erofs_super_block *dsb, + const struct erofs_device_arg *args, unsigned int arg_count) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_deviceslot *slots; + struct erofs_device_info *dif; + const char *path; + erofs_off_t devt_off; + uint64_t devt_size, image_size, end; + erofs_blk_t maxend; + unsigned int i, mask; + int err; + + sbi->total_blocks = sbi->dif0.blocks; + sbi->flatdev_blocks = sbi->dif0.blocks; + if (sbi->extra_devices == 0) { + if (arg_count != 0) { + vfs_mount_error(sbi->mnt, + "erofs: external devices given without a device " + "table"); + return (EINVAL); + } + return (0); + } + devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE; + devt_size = (uint64_t)sbi->extra_devices * EROFS_DEVT_SLOT_SIZE; + if (sbi->dif0.blocks > (UINT64_MAX >> sbi->blkszbits)) { + vfs_mount_error(sbi->mnt, "erofs: image size overflows"); + return (EINTEGRITY); + } + image_size = sbi->dif0.blocks << sbi->blkszbits; + if (devt_off > image_size || devt_size > image_size - devt_off) { + vfs_mount_error(sbi->mnt, + "erofs: device table of %ju bytes at %ju is outside the " + "image", (uintmax_t)devt_size, (uintmax_t)devt_off); + return (EINTEGRITY); + } + err = erofs_read_metadata_range(sbi, 0, devt_off, (size_t)devt_size, + &buf); + if (err != 0) + return (err); + slots = buf.data; + sbi->devs = mallocarray(sbi->extra_devices, sizeof(*sbi->devs), M_EROFS, + M_WAITOK | M_ZERO); + maxend = sbi->dif0.blocks; + for (i = 0; i < sbi->extra_devices; ++i) { + dif = &sbi->devs[i]; + dif->blocks = le32toh(slots[i].blocks_lo); + dif->uniaddr = le32toh(slots[i].uniaddr_lo); + if (erofs_sb_has_48bit(sbi)) { + dif->blocks |= + (uint64_t)le16toh(slots[i].blocks_hi) << 32; + dif->uniaddr |= + (uint64_t)le16toh(slots[i].uniaddr_hi) << 32; + } + /* + * blocks == 0 is valid: mkfs.erofs --blobdev leaves an empty + * device when no chunk data goes to it. Every read from such a + * device fails the range check in erofs_check_device_range(). + */ + if (__builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) { + vfs_mount_error(sbi->mnt, + "erofs: device %u: unified address plus size " + "overflows", i + 1); + err = EINTEGRITY; + goto out; + } + /* + * The slots are not checked against each other or against the + * primary image: overlapping or out-of-range slots only make + * the address lookup ambiguous, and every resolved range is + * checked against its device in erofs_check_device_range() and + * erofs_bread_device(). A pairwise scan would cost O(n^2) at + * mount for up to 65535 slots. + */ + if (__builtin_add_overflow(sbi->total_blocks, dif->blocks, + &sbi->total_blocks)) { + err = EOVERFLOW; + goto out; + } + maxend = MAX(maxend, (erofs_blk_t)end); + } + erofs_put_metabuf(&buf); + sbi->flatdev_blocks = maxend; + mask = 1; + while (mask < (unsigned int)sbi->extra_devices + 1) + mask <<= 1; + sbi->device_id_mask = mask - 1; + sbi->flatdev = arg_count == 0; + if (sbi->flatdev) { + /* + * The device table addresses a unified space that is larger + * than the primary image whenever the other devices are not + * part of it, e.g. a split image mounted without device + * options. Such an image still mounts, as on Linux; a read + * that needs a range beyond the provider fails with ENXIO in + * erofs_bread_device(). + */ + return (0); + } + if (arg_count != sbi->extra_devices) { + vfs_mount_error(sbi->mnt, + "erofs: external devices don't match (ondisk %u, given %u)", + sbi->extra_devices, arg_count); + return (arg_count < sbi->extra_devices ? ENXIO : EINVAL); + } + for (i = 0; i < arg_count; ++i) { + if (args[i].slot == 0 || args[i].slot > sbi->extra_devices) + return (EINVAL); + } + for (i = 0; i < sbi->extra_devices; ++i) { + path = erofs_device_arg_path(args, arg_count, i + 1); + if (path == NULL) + return (ENXIO); + err = erofs_init_device(sbi, &sbi->devs[i], path); + if (err != 0) + return (err); + } + return (0); +out: + erofs_put_metabuf(&buf); + return (err); +} + +static int +erofs_init_packed_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* Load the packed carrier before any fragment-backed metabox inode. */ + if (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) { + sbi->packed_inode = malloc(sizeof(*sbi->packed_inode), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->packed_nid, + sbi->packed_inode); + if (error != 0) { + free(sbi->packed_inode, M_EROFS); + sbi->packed_inode = NULL; + return (error); + } + /* + * The type of the packed inode is not checked, and neither is + * whether it has fragments of its own: every read of it is + * bounded, and z_erofs_do_read() refuses a fragment that points + * into the packed inode itself. + */ + } + return (0); +} + +static int +erofs_init_metabox_inode(struct erofs_sb_info *sbi) +{ + int error; + + /* + * METABOX NIDs address inode slots in this backing inode's data. The + * packed carrier is ready first so a compressed metabox may legally end + * in a fragment pcluster without reading an uninitialized dependency. + */ + if (erofs_sb_has_metabox(sbi)) { + sbi->metabox_en = malloc(sizeof(*sbi->metabox_en), M_EROFS, + M_WAITOK | M_ZERO); + error = erofs_read_inode(sbi, sbi->metabox_nid, + sbi->metabox_en); + if (error != 0) + return (error); + } + return (0); +} + +static int +erofs_read_superblock(struct erofs_sb_info *sbi, struct erofs_super_block *dsb) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + uint32_t unsupported; + int ret; + + ret = erofs_read_metadata_range(sbi, 0, EROFS_SUPER_OFFSET, + sizeof(*dsb), &buf); + if (ret != 0) + return (ret); + memcpy(dsb, buf.data, sizeof(*dsb)); + erofs_put_metabuf(&buf); + + if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1) + return (EINVAL); + if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT) + return (EINVAL); + sbi->blkszbits = dsb->blkszbits; + sbi->block_size = 1u << sbi->blkszbits; + if ((sbi->dif0.fsoff & (sbi->block_size - 1)) != 0) { + vfs_mount_error(sbi->mnt, + "erofs: fsoffset %ju is not aligned to block size %u", + (uintmax_t)sbi->dif0.fsoff, sbi->block_size); + return (EINVAL); + } + if (dsb->dirblkbits != 0) { + vfs_mount_error(sbi->mnt, + "erofs: dirblkbits %u isn't supported", dsb->dirblkbits); + return (EINVAL); + } + sbi->feature_compat = le32toh(dsb->feature_compat); + ret = erofs_superblock_csum_verify(sbi, dsb); + if (ret != 0) + return (ret); + + sbi->feature_incompat = le32toh(dsb->feature_incompat); + sbi->packed_nid = le64toh(dsb->packed_nid); + sbi->extra_devices = erofs_sb_has_device_table(sbi) ? + le16toh(dsb->extra_devices) : 0; + unsupported = sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT; + if (unsupported != 0) { + vfs_mount_error(sbi->mnt, + "erofs: unidentified incompatible feature %x, please " + "upgrade kernel", + unsupported); + return (EINVAL); + } + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) + return (EINVAL); + sbi->meta_blkaddr = le32toh(dsb->meta_blkaddr); +#if WITH_EROFS_XATTR + sbi->xattr_blkaddr = le32toh(dsb->xattr_blkaddr); + sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start); + sbi->xattr_prefix_count = dsb->xattr_prefix_count; + sbi->xattr_filter_reserved = dsb->xattr_filter_reserved; +#endif + erofs_sb_blocks_root(dsb, sbi->feature_incompat, &sbi->blocks, + &sbi->root_nid); + sbi->dif0.blocks = sbi->blocks; + if (sbi->dif0.blocks == 0) { + vfs_mount_error(sbi->mnt, "erofs: the image declares 0 blocks"); + return (EINTEGRITY); + } + ret = erofs_validate_device_size(sbi, &sbi->dif0, sbi->dif0.blocks); + if (ret != 0) + return (ret); + sbi->inos = le64toh(dsb->inos); + sbi->epoch = (int64_t)le64toh(dsb->epoch); + /* Not checked here: erofs_set_timestamp() checks it for each inode. */ + sbi->fixed_nsec = le32toh(dsb->fixed_nsec); + ret = erofs_load_generation_seed(sbi, sbi->sb_size, + &sbi->generation_seed); + if (ret != 0) + return (ret); + if (erofs_sb_has_metabox(sbi)) { + if (sbi->sb_size <= offsetof(struct erofs_super_block, + metabox_nid)) { + vfs_mount_error(sbi->mnt, + "erofs: the superblock has no room for " + "metabox_nid"); + return (EINTEGRITY); + } + sbi->metabox_nid = le64toh(dsb->metabox_nid); + if (erofs_nid_in_metabox(sbi->metabox_nid)) { + vfs_mount_error(sbi->mnt, + "erofs: metabox nid %ju is in the metabox", + (uintmax_t)sbi->metabox_nid); + return (EINTEGRITY); + } + } + +#if WITH_EROFS_ZIP + return (z_erofs_parse_cfgs(sbi, dsb)); +#else + if (dsb->u1.available_compr_algs != 0 || + erofs_sb_has_lz4_0padding(sbi)) { + vfs_mount_error(sbi->mnt, + "erofs: compression disabled, unable to mount compressed " + "EROFS"); + return (EOPNOTSUPP); + } + return (0); +#endif +} + +static int +erofs_mountfs(struct erofs_device_info *primary, struct mount *mp, + const struct erofs_device_arg *args, unsigned int arg_count, + bool xattr_user_enabled, bool acl_enabled, bool directio_enabled) +{ + struct erofs_sb_info *sbi; + struct erofs_inode root; + struct erofs_super_block dsb; + bool cache_present, sync_present, priority_present; + int cleanup_error, error; + + sbi = malloc(sizeof(*sbi), M_EROFS, M_WAITOK | M_ZERO); + sbi->mnt = mp; + erofs_management_init(sbi); + sbi->opt.cache_strategy = EROFS_ZIP_CACHE_READAROUND; + sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_AUTO; + error = erofs_parse_zip_mount_options(mp, &sbi->opt, + &sbi->sync_decompress, &cache_present, &sync_present, + &priority_present); + if (error != 0) + goto fail; + mtx_lock(&sbi->control_lock); + sbi->decompress_queue_priority = sbi->opt.worker_queue_priority; + mtx_unlock(&sbi->control_lock); +#if WITH_EROFS_ZIP + z_erofs_extent_cache_init(sbi); + z_erofs_stream_pool_init(sbi); +#endif + sbi->dif0 = *primary; + bzero(primary, sizeof(*primary)); + if (directio_enabled) + set_opt(&sbi->opt, DIRECT_IO); + error = erofs_register_source_upper(sbi, &sbi->dif0); + if (error != 0) + goto fail; + + error = erofs_read_superblock(sbi, &dsb); + if (error != 0) + goto fail; + error = erofs_parse_dir_ra_option(sbi, mp->mnt_optnew); + if (error != 0) { + vfs_mount_error(mp, "erofs: invalid dir_ra_bytes option"); + goto fail; + } + vfs_deleteopt(mp->mnt_optnew, "dir_ra_bytes"); + error = erofs_scan_devices(sbi, &dsb, args, arg_count); + if (error != 0) + goto fail; + + error = erofs_init_packed_inode(sbi); + if (error != 0) + goto fail; + error = erofs_init_metabox_inode(sbi); + if (error != 0) + goto fail; + error = erofs_read_inode(sbi, sbi->root_nid, &root); + if (error != 0) + goto fail; + if (root.vtype != VDIR) { + vfs_mount_error(mp, + "erofs: root inode nid=%ju is not a directory", + (uintmax_t)sbi->root_nid); + error = EINVAL; + goto fail; + } + error = erofs_xattr_prefixes_init(sbi); + if (error != 0) + goto fail; + + mp->mnt_data = sbi; + if (sbi->dif0.source_kind == EROFS_SOURCE_FILE) + vfs_getnewfsid(mp); + else { + mp->mnt_stat.f_fsid.val[0] = dev2udev(sbi->dif0.devvp->v_rdev); + mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum; + } + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY; + mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED | + MNTK_USES_BCACHE; + MNT_IUNLOCK(mp); + erofs_set_xattr_state(sbi, xattr_user_enabled); + erofs_set_acl_state(sbi, acl_enabled); + error = erofs_register_sysfs(sbi); + if (error != 0) + goto fail; + return (0); +fail: + cleanup_error = erofs_sb_free(sbi); + if (mp->mnt_data == sbi) + mp->mnt_data = NULL; + return (cleanup_error != 0 ? cleanup_error : error); +} + +static int +erofs_mount(struct mount *mp) +{ + struct erofs_device_arg *args; + struct erofs_device_info primary; + struct erofs_sb_info *sbi; + struct erofs_sb_info_opts next_opt; + char *fspec; + uint64_t fsoffset, new_dir_ra; + unsigned int next_sync; + unsigned int arg_count; + bool directio_enabled, directio_present; + bool fsoffset_present, xattr_user_enabled, acl_enabled; + bool cache_present, sync_present, priority_present, dir_ra_present; + int error, len; + + MNT_ILOCK(mp); + mp->mnt_flag |= MNT_RDONLY; + MNT_IUNLOCK(mp); + if (mp->mnt_flag & MNT_UPDATE) { + sbi = MTOE(mp); + if (sbi == NULL) + return (EINVAL); + if (mp->mnt_flag & MNT_RELOAD) + return (EOPNOTSUPP); + error = erofs_reject_device_update(mp); + if (error != 0) + return (error); + error = erofs_parse_fsoffset(mp, &fsoffset, &fsoffset_present); + if (error != 0) + goto update_fail; + error = erofs_parse_directio_options(mp, &directio_enabled, + &directio_present); + if (error != 0) + goto update_fail; + if (fsoffset_present && fsoffset != sbi->dif0.fsoff) { + vfs_mount_error(mp, + "erofs: fsoffset cannot be changed on update"); + error = EOPNOTSUPP; + goto update_fail; + } + error = vfs_filteropt(mp->mnt_optnew, erofs_opts); + if (error != 0) + goto update_fail; + mtx_lock(&sbi->control_lock); + bzero(&next_opt, sizeof(next_opt)); + next_opt.cache_strategy = sbi->opt.cache_strategy; + next_opt.worker_queue_priority = + sbi->opt.worker_queue_priority; + next_sync = sbi->sync_decompress; + mtx_unlock(&sbi->control_lock); + error = erofs_parse_zip_mount_options(mp, &next_opt, + &next_sync, &cache_present, &sync_present, + &priority_present); + if (error != 0) + goto update_fail; + error = erofs_parse_acl_options(mp, + test_opt(&sbi->opt, POSIX_ACL) != 0, &acl_enabled); + if (error != 0) + goto update_fail; + error = erofs_parse_xattr_options(mp, + test_opt(&sbi->opt, XATTR_USER) != 0, &xattr_user_enabled); + if (error != 0) + goto update_fail; + new_dir_ra = 0; + error = erofs_parse_dir_ra_value(sbi, mp->mnt_optnew, + &new_dir_ra, &dir_ra_present); + if (error != 0) { + vfs_mount_error(mp, + "erofs: invalid dir_ra_bytes option"); + goto update_fail; + } + vfs_deleteopt(mp->mnt_optnew, "dir_ra_bytes"); + if (dir_ra_present) { + error = erofs_set_dir_ra_bytes(sbi, new_dir_ra); + if (error != 0) + goto update_fail; + } + mtx_lock(&sbi->control_lock); + if (cache_present) + sbi->opt.cache_strategy = next_opt.cache_strategy; + if (sync_present) + sbi->sync_decompress = next_sync; + if (priority_present) { + sbi->opt.worker_queue_priority = + next_opt.worker_queue_priority; + sbi->decompress_queue_priority = + next_opt.worker_queue_priority; + } + mtx_unlock(&sbi->control_lock); + erofs_set_xattr_state(sbi, xattr_user_enabled); + erofs_set_acl_state(sbi, acl_enabled); + if (directio_present) { + if (directio_enabled) + set_opt(&sbi->opt, DIRECT_IO); + else + clear_opt(&sbi->opt, DIRECT_IO); + } + return (0); +update_fail: + return (error); + } + args = NULL; + arg_count = 0; + error = erofs_parse_device_options(mp, &args, &arg_count); + if (error != 0) + return (error); + if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) { + error = EINVAL; + goto out_args; + } + error = erofs_parse_fsoffset(mp, &fsoffset, &fsoffset_present); + if (error != 0) + goto out_args; + error = erofs_parse_directio_options(mp, &directio_enabled, + &directio_present); + if (error != 0) + goto out_args; + error = erofs_parse_xattr_options(mp, true, &xattr_user_enabled); + if (error != 0) + goto out_args; + error = erofs_parse_acl_options(mp, true, &acl_enabled); + if (error != 0) + goto out_args; + fspec = NULL; + error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len); + if (error != 0 || fspec == NULL || len == 0 || + fspec[len - 1] != '\0') { + error = EINVAL; + goto out_args; + } + mp->mnt_iosize_max = MAXPHYS; + error = erofs_open_device(mp, NULL, fspec, &primary); + if (error != 0) + goto out_args; + primary.fsoff = fsoffset; + erofs_update_iosize_max(mp, &primary); + error = erofs_mountfs(&primary, mp, args, arg_count, xattr_user_enabled, + acl_enabled, directio_enabled); + if (error != 0) + erofs_release_device_info(&primary); +out_args: + erofs_free_device_args(args, arg_count); + if (error != 0) + return (error); + vfs_mountedfrom(mp, fspec); + return (erofs_statfs(mp, &mp->mnt_stat)); +} + +static int +erofs_root(struct mount *mp, int flags, struct vnode **vpp) +{ + int error; + + error = erofs_vget_raw(mp, MTOE(mp)->root_nid, flags, vpp); + if (error != 0) + vfs_mount_error(mp, + "erofs: failed to load root nid %ju: error %d", + (uintmax_t)MTOE(mp)->root_nid, error); + return (error); +} + +static int +erofs_statfs(struct mount *mp, struct statfs *sbp) +{ + struct erofs_sb_info *sbi; + + sbi = MTOE(mp); + sbp->f_bsize = sbi->block_size; + sbp->f_iosize = sbi->block_size; + sbp->f_blocks = sbi->total_blocks; + sbp->f_bfree = 0; + sbp->f_bavail = 0; + sbp->f_files = INT64_MAX; + sbp->f_ffree = sbi->inos <= (uint64_t)INT64_MAX ? + INT64_MAX - (int64_t)sbi->inos : 0; + return (0); +} + +static int +erofs_unmount(struct mount *mp, int mntflags) +{ + struct erofs_sb_info *sbi; + int error, flags; + + sbi = MTOE(mp); + if (sbi == NULL) + return (EINVAL); + flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0; + error = vflush(mp, 0, flags, curthread); + if (error != 0) + return (error); + error = erofs_sb_free(sbi); + if (error != 0) { + return (error); + } + mp->mnt_data = NULL; + return (0); +} + +/* Persistent EROFS file handle to locked vnode. */ +static int +erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp) +{ + struct erofs_fid efid; + struct erofs_inode *vi; + struct vnode *vp; + erofs_nid_t nid; + int error; + + *vpp = NULL; + bzero(&efid, sizeof(efid)); + memcpy(&efid, fhp, sizeof(efid)); + if (efid.len != sizeof(efid) || efid.pad != 0) + return (EINVAL); + nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo; + if (!erofs_nid_is_valid(MTOE(mp), nid)) + return (ESTALE); + error = erofs_vget_raw(mp, nid, flags, &vp); + if (error != 0) + return (error); + vi = VTOE(vp); + if (vi->mode == 0 || vi->nlink == 0 || vi->nid != nid || + vi->generation != efid.gen) { + vput(vp); + return (ESTALE); + } + *vpp = vp; + return (0); +} + +static struct vfsops erofs_vfsops = { + .vfs_fhtovp = erofs_fhtovp, + .vfs_mount = erofs_mount, + .vfs_root = erofs_root, + .vfs_statfs = erofs_statfs, + .vfs_unmount = erofs_unmount, + .vfs_vget = erofs_vget, +}; +VFS_SET(erofs_vfsops, erofs, VFCF_READONLY); +#if WITH_EROFS_POSIX_ACL +MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1); +#endif +#if WITH_EROFS_ZIP_DEFLATE +MODULE_DEPEND(erofs, zlib, 1, 1, 1); +#endif +MODULE_VERSION(erofs, 1); +#ifdef KDTRACE_HOOKS +MODULE_DEPEND(erofs, sdt, 1, 1, 1); +#endif diff --git a/sysfs.c b/sysfs.c new file mode 100644 index 0000000..94b4e50 --- /dev/null +++ b/sysfs.c @@ -0,0 +1,571 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* FreeBSD per-mount management controls for EROFS. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" +#include "internal.h" + +#define EROFS_DIR_RA_DEFAULT_BYTES (16UL * 1024) +#define EROFS_DIR_RA_HARD_MAX_BYTES (1024UL * 1024) +#define EROFS_DIR_RA_SLOTS 256UL + +enum erofs_cache_metric { + EROFS_CACHE_HITS, + EROFS_CACHE_MISSES, + EROFS_CACHE_BYPASSES, + EROFS_CACHE_EVICTIONS, + EROFS_CACHE_RECLAIMS, + EROFS_CACHE_WAITER_MERGES, + EROFS_CACHE_ENTRIES, + EROFS_CACHE_BYTES, +}; + +static struct mtx erofs_control_id_lock; +static uint64_t erofs_control_next_id = 1; + +MTX_SYSINIT(erofs_control_id, &erofs_control_id_lock, "erofs control id", + MTX_DEF); + +SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "EROFS filesystem"); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, mounts, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Mounted EROFS filesystems"); + +void +erofs_management_init(struct erofs_sb_info *sbi) +{ + + mtx_init(&sbi->control_lock, "erofs control", NULL, MTX_DEF); + sbi->dir_ra_bytes = EROFS_DIR_RA_DEFAULT_BYTES; + sbi->dir_ra_max_bytes = EROFS_DIR_RA_HARD_MAX_BYTES; + sbi->control_initialized = true; +} + +void +erofs_management_fini(struct erofs_sb_info *sbi) +{ + + if (!sbi->control_initialized) + return; + KASSERT(!sbi->sysctl_registered, + ("erofs management sysctls remain registered")); + mtx_destroy(&sbi->control_lock); + sbi->control_initialized = false; +} + +uint64_t +erofs_dir_ra_bytes(struct erofs_sb_info *sbi) +{ + uint64_t value; + + mtx_lock(&sbi->control_lock); + value = sbi->dir_ra_bytes; + mtx_unlock(&sbi->control_lock); + return (value); +} + +/* The readahead is limited by the size of the per-directory run lists. */ +static uint64_t +erofs_dir_ra_limit(const struct erofs_sb_info *sbi) +{ + + return (MIN((uint64_t)EROFS_DIR_RA_HARD_MAX_BYTES, + (uint64_t)sbi->block_size * EROFS_DIR_RA_SLOTS)); +} + +int +erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value) +{ + + if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) || + (value != 0 && value % sbi->block_size != 0)) + return (EINVAL); + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + sbi->dir_ra_bytes = value; + mtx_unlock(&sbi->control_lock); + return (0); +} + +void +erofs_record_dir_readahead(struct erofs_sb_info *sbi, unsigned int blocks) +{ + + mtx_lock(&sbi->control_lock); + ++sbi->dir_ra_requests; + sbi->dir_ra_blocks += blocks; + mtx_unlock(&sbi->control_lock); +} + +static int +erofs_parse_uint64(const char *value, size_t len, uint64_t *result) +{ + size_t i; + uint64_t parsed; + + if (value == NULL || len <= 1 || value[len - 1] != '\0' || + strnlen(value, len) != len - 1) + return (EINVAL); + parsed = 0; + for (i = 0; i < len - 1; ++i) { + if (value[i] < '0' || value[i] > '9') + return (EINVAL); + if (parsed > (UINT64_MAX - (value[i] - '0')) / 10) + return (ERANGE); + parsed = parsed * 10 + (value[i] - '0'); + } + *result = parsed; + return (0); +} + +int +erofs_parse_dir_ra_value(struct erofs_sb_info *sbi, struct vfsoptlist *opts, + uint64_t *valuep, bool *presentp) +{ + struct vfsopt *opt; + uint64_t value; + int error; + + *presentp = false; + TAILQ_FOREACH(opt, opts, link) { + if (strcmp(opt->name, "dir_ra_bytes") != 0) + continue; + error = erofs_parse_uint64(opt->value, opt->len, &value); + if (error != 0) + return (error); + if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) || + (value != 0 && value % sbi->block_size != 0)) + return (EINVAL); + *valuep = value; + *presentp = true; + return (0); + } + return (0); +} + +int +erofs_parse_dir_ra_option(struct erofs_sb_info *sbi, struct vfsoptlist *opts) +{ + uint64_t value; + bool present; + int error; + + error = erofs_parse_dir_ra_value(sbi, opts, &value, &present); + if (error != 0 || !present) + return (error); + return (erofs_set_dir_ra_bytes(sbi, value)); +} + +static int +erofs_sysctl_dir_ra(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + int error; + + sbi = arg1; + value = erofs_dir_ra_bytes(sbi); + error = sysctl_handle_64(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + return (erofs_set_dir_ra_bytes(sbi, value)); +} + +static int +erofs_sysctl_dir_ra_metric(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = arg2 == 0 ? sbi->dir_ra_requests : sbi->dir_ra_blocks; + mtx_unlock(&sbi->control_lock); + return (sysctl_handle_64(oidp, &value, 0, req)); +} + +#if WITH_EROFS_ZIP +static int +erofs_validate_zip_value(const char *value, int len) +{ + + if (value == NULL || len <= 1 || value[len - 1] != '\0' || + strnlen(value, (size_t)len) != (size_t)len - 1) + return (EINVAL); + return (0); +} + +static int +erofs_parse_zip_uint(const char *value, int len, unsigned int *result) +{ + size_t i; + unsigned int parsed; + + if (erofs_validate_zip_value(value, len) != 0) + return (EINVAL); + parsed = 0; + for (i = 0; i < (size_t)len - 1; ++i) { + if (value[i] < '0' || value[i] > '9' || + parsed > (UINT_MAX - (value[i] - '0')) / 10) + return (EINVAL); + parsed = parsed * 10 + (value[i] - '0'); + } + *result = parsed; + return (0); +} + +int +erofs_parse_zip_options(struct mount *mp, struct erofs_sb_info_opts *opts, + unsigned int *syncp, bool *cache_presentp, bool *sync_presentp) +{ + struct vfsopt *opt; + unsigned int value; + bool queue_priority_present; + int error; + + *cache_presentp = false; + *sync_presentp = false; + queue_priority_present = false; + TAILQ_FOREACH(opt, mp->mnt_optnew, link) { + if (strcmp(opt->name, "cache_strategy") == 0) { + if (opt->value == NULL || opt->len <= 1 || + ((char *)opt->value)[opt->len - 1] != '\0' || + strnlen(opt->value, opt->len) != + (size_t)opt->len - 1) + return (EINVAL); + if (strcmp(opt->value, "disabled") == 0) + opts->cache_strategy = EROFS_ZIP_CACHE_DISABLED; + else if (strcmp(opt->value, "readahead") == 0) + opts->cache_strategy = + EROFS_ZIP_CACHE_READAHEAD; + else if (strcmp(opt->value, "readaround") == 0) + opts->cache_strategy = + EROFS_ZIP_CACHE_READAROUND; + else + return (EINVAL); + *cache_presentp = true; + continue; + } + if (strcmp(opt->name, "worker_queue_priority") == 0) { + error = erofs_parse_zip_uint(opt->value, opt->len, + &value); + if (error != 0 || value > UINT8_MAX) + return (EINVAL); + opts->worker_queue_priority = (uint8_t)value; + queue_priority_present = true; + continue; + } + if (strcmp(opt->name, "sync_decompress") != 0) + continue; + error = erofs_validate_zip_value(opt->value, opt->len); + if (error != 0) + return (error); + if (strcmp(opt->value, "auto") == 0) + value = EROFS_SYNC_DECOMPRESS_AUTO; + else if (strcmp(opt->value, "on") == 0) + value = EROFS_SYNC_DECOMPRESS_FORCE_ON; + else if (strcmp(opt->value, "off") == 0) + value = EROFS_SYNC_DECOMPRESS_FORCE_OFF; + else { + error = erofs_parse_zip_uint(opt->value, opt->len, + &value); + if (error != 0 || + value > EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (EINVAL); + } + *syncp = value; + *sync_presentp = true; + } + if (*cache_presentp) + vfs_deleteopt(mp->mnt_optnew, "cache_strategy"); + if (*sync_presentp) + vfs_deleteopt(mp->mnt_optnew, "sync_decompress"); + if (queue_priority_present) + vfs_deleteopt(mp->mnt_optnew, "worker_queue_priority"); + return (0); +} + +static int +erofs_sysctl_drop_caches(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + uint64_t value; + int error; + + if (req->newptr == NULL) + return (EPERM); + sbi = arg1; + value = 0; + error = sysctl_handle_64(oidp, &value, 0, req); + if (error != 0) + return (error); + if (value != 1) + return (EINVAL); + return (z_erofs_extent_cache_drop(sbi)); +} + +static int +erofs_sysctl_cache_metric(SYSCTL_HANDLER_ARGS) +{ + struct erofs_cache_stats stats; + struct erofs_sb_info *sbi; + uint64_t value; + + sbi = arg1; + z_erofs_extent_cache_stats(sbi, &stats); + switch (arg2) { + case EROFS_CACHE_HITS: + value = stats.hits; + break; + case EROFS_CACHE_MISSES: + value = stats.misses; + break; + case EROFS_CACHE_BYPASSES: + value = stats.bypasses; + break; + case EROFS_CACHE_EVICTIONS: + value = stats.evictions; + break; + case EROFS_CACHE_RECLAIMS: + value = stats.reclaims; + break; + case EROFS_CACHE_WAITER_MERGES: + value = stats.waiter_merges; + break; + case EROFS_CACHE_ENTRIES: + value = stats.entries; + break; + case EROFS_CACHE_BYTES: + value = stats.bytes; + break; + default: + return (EINVAL); + } + return (sysctl_handle_64(oidp, &value, 0, req)); +} + +static int +erofs_sysctl_zip_setting(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + int value; + int error; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = arg2 == 0 ? (int)sbi->sync_decompress : + (int)sbi->opt.cache_strategy; + mtx_unlock(&sbi->control_lock); + error = sysctl_handle_int(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (arg2 == 0) { + if (value < EROFS_SYNC_DECOMPRESS_AUTO || + value > EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (EINVAL); + } else { + if (value < EROFS_ZIP_CACHE_DISABLED || + value > EROFS_ZIP_CACHE_READAROUND) + return (EINVAL); + } + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + if (arg2 == 0) + sbi->sync_decompress = (unsigned int)value; + else + sbi->opt.cache_strategy = (uint8_t)value; + mtx_unlock(&sbi->control_lock); + return (0); +} + +static int +erofs_sysctl_worker_queue_priority(SYSCTL_HANDLER_ARGS) +{ + struct erofs_sb_info *sbi; + int error, value; + + sbi = arg1; + mtx_lock(&sbi->control_lock); + value = sbi->decompress_queue_priority; + mtx_unlock(&sbi->control_lock); + error = sysctl_handle_int(oidp, &value, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + if (value < 0 || value > UINT8_MAX) + return (EINVAL); + mtx_lock(&sbi->control_lock); + if (sbi->control_closing) { + mtx_unlock(&sbi->control_lock); + return (ENXIO); + } + sbi->decompress_queue_priority = (uint8_t)value; + sbi->opt.worker_queue_priority = (uint8_t)value; + mtx_unlock(&sbi->control_lock); + return (0); +} +#endif + +int +erofs_register_sysfs(struct erofs_sb_info *sbi) +{ + struct sysctl_oid *cache_node, *mount_node; + char name[24]; + int error; + + mtx_lock(&erofs_control_id_lock); + sbi->control_id = erofs_control_next_id++; + mtx_unlock(&erofs_control_id_lock); + + sbi->dir_ra_max_bytes = erofs_dir_ra_limit(sbi); + + snprintf(name, sizeof(name), "%ju", (uintmax_t)sbi->control_id); + sysctl_ctx_init(&sbi->sysctl_ctx); + sbi->sysctl_registered = true; + mount_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx, + SYSCTL_STATIC_CHILDREN(_vfs_erofs_mounts), OID_AUTO, name, + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "EROFS mount"); + if (mount_node == NULL) + goto fail; + if (SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "mount_id", CTLFLAG_RD, &sbi->control_id, 0, + "Unique mount control identifier") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_bytes", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_dir_ra, "QU", + "Directory readahead window in bytes") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_max_bytes", CTLFLAG_RD, &sbi->dir_ra_max_bytes, 0, + "Maximum directory readahead window") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_requests", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_dir_ra_metric, "QU", + "Directory readahead submissions") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "dir_ra_blocks", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, 1, erofs_sysctl_dir_ra_metric, "QU", + "Directory readahead blocks submitted") == NULL) + goto fail; +#if WITH_EROFS_ZIP + if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "drop_caches", CTLTYPE_U64 | CTLFLAG_WR | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_drop_caches, "QU", + "Write 1 to drop the FreeBSD decoded extent cache") == NULL) + goto fail; + if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "sync_decompress", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 0, erofs_sysctl_zip_setting, "I", + "0=auto, 1=force synchronous, 2=taskqueue fallback") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node), + OID_AUTO, "cache_strategy", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + sbi, 1, erofs_sysctl_zip_setting, "I", + "0=disabled, 1=readahead, 2=readaround") == NULL) + goto fail; + cache_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx, + SYSCTL_CHILDREN(mount_node), OID_AUTO, "decoded_cache", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, + "FreeBSD decoded extent cache, not Linux managed pclusters"); + if (cache_node == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "hits", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_HITS, erofs_sysctl_cache_metric, "QU", + "Cache hits") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "misses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_MISSES, erofs_sysctl_cache_metric, "QU", + "Cache misses") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "bypasses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_BYPASSES, erofs_sysctl_cache_metric, "QU", + "Cache admission bypasses") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "evictions", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_EVICTIONS, erofs_sysctl_cache_metric, "QU", + "Cache evictions") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "reclaims", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_RECLAIMS, erofs_sysctl_cache_metric, "QU", + "Cache pressure reclaims") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "waiter_merges", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, + sbi, EROFS_CACHE_WAITER_MERGES, erofs_sysctl_cache_metric, "QU", + "Inflight cache waiter merges") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "entries", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_ENTRIES, erofs_sysctl_cache_metric, "QU", + "Resident decoded extents") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "bytes", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi, + EROFS_CACHE_BYTES, erofs_sysctl_cache_metric, "QU", + "Resident decoded bytes") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_submissions", CTLFLAG_RD, + &sbi->decompress_submissions, + 0, "Taskqueue decompression submissions") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_runs", CTLFLAG_RD, &sbi->decompress_worker_runs, + 0, "Taskqueue decompression worker runs") == NULL || + SYSCTL_ADD_UINT(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_count", CTLFLAG_RD, &sbi->decompress_worker_count, + 0, "Bounded decompression worker count") == NULL || + SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "queue_fallbacks", CTLFLAG_RD, + &sbi->decompress_queue_fallbacks, 0, + "Direct decode after taskqueue fallback") == NULL || + SYSCTL_ADD_U8(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_cpu_bound", CTLFLAG_RD, + (uint8_t *)&sbi->decompress_worker_cpu_bound, 0, + "Workers restricted to the online CPU set") == NULL || + SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node), + OID_AUTO, "worker_queue_priority", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sbi, 0, + erofs_sysctl_worker_queue_priority, "I", + "Taskqueue order priority; 0 is highest and 255 is lowest") == NULL) + goto fail; +#else + (void)cache_node; +#endif + return (0); + +fail: + error = erofs_unregister_sysfs(sbi); + return (error != 0 ? error : ENOMEM); +} + +int +erofs_unregister_sysfs(struct erofs_sb_info *sbi) +{ + int error; + + if (!sbi->sysctl_registered) + return (0); + mtx_lock(&sbi->control_lock); + sbi->control_closing = true; + mtx_unlock(&sbi->control_lock); + error = sysctl_ctx_free(&sbi->sysctl_ctx); + if (error != 0) { + mtx_lock(&sbi->control_lock); + sbi->control_closing = false; + mtx_unlock(&sbi->control_lock); + return (error); + } + sbi->sysctl_registered = false; + return (0); +} diff --git a/trace.c b/trace.c new file mode 100644 index 0000000..25c8685 --- /dev/null +++ b/trace.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include "internal.h" +#include "trace.h" + +SDT_PROVIDER_DEFINE(erofs); + +SDT_PROBE_DEFINE4(erofs, data, map_blocks, enter, + "uint64_t", "uint64_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE6(erofs, data, map_blocks, exit, + "uint64_t", "uint64_t", "uint64_t", "uint64_t", "uint64_t", + "uint64_t"); +SDT_PROBE_DEFINE5(erofs, data, read_folio, entry, + "uint64_t", "uint64_t", "uint64_t", "int", "bool"); +SDT_PROBE_DEFINE4(erofs, data, readahead, entry, + "uint64_t", "uint64_t", "uint64_t", "bool"); +SDT_PROBE_DEFINE3(erofs, inode, fill_inode, entry, + "uint64_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE4(erofs, namei, lookup, entry, + "uint64_t", "uint32_t", "uint64_t", "uint32_t"); +SDT_PROBE_DEFINE3(erofs, , , corrupt, "uint64_t", "int", "int"); diff --git a/trace.h b/trace.h new file mode 100644 index 0000000..77bc6bd --- /dev/null +++ b/trace.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _FS_EROFS_TRACE_H_ +#define _FS_EROFS_TRACE_H_ + +#include + +#include "internal.h" + +SDT_PROVIDER_DECLARE(erofs); + +SDT_PROBE_DECLARE(erofs, data, map_blocks, enter); +SDT_PROBE_DECLARE(erofs, data, map_blocks, exit); +SDT_PROBE_DECLARE(erofs, data, read_folio, entry); +SDT_PROBE_DECLARE(erofs, data, readahead, entry); +SDT_PROBE_DECLARE(erofs, inode, fill_inode, entry); +SDT_PROBE_DECLARE(erofs, namei, lookup, entry); +SDT_PROBE_DECLARE(erofs, , , corrupt); + +#if WITH_EROFS_SDT +#define EROFS_TRACE_CORRUPT(nid, line, error) \ + SDT_PROBE3(erofs, , , corrupt, (nid), (line), (error)) +#else +#define EROFS_TRACE_CORRUPT(nid, line, error) do { } while (0) +#endif + +#define EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, flags) \ + SDT_PROBE4(erofs, data, map_blocks, enter, \ + (vi)->nid, (map)->m_la, (map)->m_llen, (flags)) + +#ifdef KDTRACE_HOOKS +#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \ + uint64_t erofs_trace_status; \ + if (SDT_PROBES_ENABLED()) { \ + erofs_trace_status = ((uint64_t)(uint32_t)(error) << 32) | \ + (uint64_t)(uint32_t)(map)->m_flags; \ + SDT_PROBE6(erofs, data, map_blocks, exit, (vi)->nid, \ + (map)->m_la, (map)->m_pa, (map)->m_llen, (map)->m_plen, \ + erofs_trace_status); \ + } \ +} while (0) +#else +#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \ + (void)(vi); \ + (void)(map); \ + (void)(error); \ +} while (0) +#endif + +#define EROFS_TRACE_READ_FOLIO(vi, offset, length, raw) \ + SDT_PROBE5(erofs, data, read_folio, entry, (vi)->nid, \ + (uint64_t)(offset), (uint64_t)(length), (vi)->vtype == VDIR, (raw)) + +#define EROFS_TRACE_READAHEAD(vi, start, nrpages, raw) \ + SDT_PROBE4(erofs, data, readahead, entry, (vi)->nid, \ + (uint64_t)(start), (uint64_t)(nrpages), (raw)) + +#define EROFS_TRACE_FILL_INODE(nid, blkaddr, offset) \ + SDT_PROBE3(erofs, inode, fill_inode, entry, (nid), \ + (uint64_t)(blkaddr), (uint32_t)(offset)) + +static __inline void +erofs_trace_lookup(const struct erofs_inode *dir, const char *name, + size_t length, uint32_t flags) +{ +#ifdef KDTRACE_HOOKS + uint32_t hash; + size_t i; + + if (!SDT_PROBES_ENABLED()) + return; + hash = UINT32_C(2166136261); + for (i = 0; i < length; ++i) { + hash ^= (uint8_t)name[i]; + hash *= UINT32_C(16777619); + } + SDT_PROBE4(erofs, namei, lookup, entry, dir->nid, hash, + (uint64_t)length, flags); +#else + (void)dir; + (void)name; + (void)length; + (void)flags; +#endif +} + +#endif diff --git a/xattr.c b/xattr.c new file mode 100644 index 0000000..07671e4 --- /dev/null +++ b/xattr.c @@ -0,0 +1,1151 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021-2022, Alibaba Cloud + */ + +#include +#include +#include +#include +#include +#include + +#if WITH_EROFS_POSIX_ACL +#include +#endif + +#include + +#include "xattr.h" + +#if WITH_EROFS_POSIX_ACL +struct posix_acl_xattr_entry { + uint16_t e_tag; + uint16_t e_perm; + uint32_t e_id; +}; + +struct posix_acl_xattr_header { + uint32_t a_version; +}; + +#define POSIX_ACL_XATTR_VERSION 0x0002 +#define EROFS_XATTR_FILTER_POSIX_ACL \ + ((1U << 21) | (1U << 30)) +#endif +#define EROFS_XATTR_CACHE_BODY_LIMIT (64UL * 1024) +#define EROFS_XATTR_CACHE_MOUNT_BUDGET (1024UL * 1024) + +struct erofs_xattr_iter { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + int attrnamespace; + const char *name; + size_t name_len; + struct uio *uio; + size_t *sizep; +}; + +static int erofs_xattr_load_body(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_buf *bodybuf, + struct erofs_xattr_ibody_header **ihp, size_t *header_sizep); + +static uint32_t +erofs_xxh32_rotl(uint32_t value, unsigned int count) +{ + return ((value << count) | (value >> (32 - count))); +} + +static uint32_t +erofs_xxh32_round(uint32_t seed, uint32_t input) +{ + seed += input * UINT32_C(2246822519); + seed = erofs_xxh32_rotl(seed, 13); + return (seed * UINT32_C(2654435761)); +} + +static uint32_t +erofs_xxh32(const void *input, size_t length, uint32_t seed) +{ + const uint8_t *cursor, *end; + uint32_t hash; + + cursor = input; + end = cursor + length; + if (length >= 16) { + const uint8_t *limit; + uint32_t v1, v2, v3, v4; + + limit = end - 16; + v1 = seed + UINT32_C(2654435761) + UINT32_C(2246822519); + v2 = seed + UINT32_C(2246822519); + v3 = seed; + v4 = seed - UINT32_C(2654435761); + do { + v1 = erofs_xxh32_round(v1, le32dec(cursor)); + cursor += 4; + v2 = erofs_xxh32_round(v2, le32dec(cursor)); + cursor += 4; + v3 = erofs_xxh32_round(v3, le32dec(cursor)); + cursor += 4; + v4 = erofs_xxh32_round(v4, le32dec(cursor)); + cursor += 4; + } while (cursor <= limit); + hash = erofs_xxh32_rotl(v1, 1) + erofs_xxh32_rotl(v2, 7) + + erofs_xxh32_rotl(v3, 12) + erofs_xxh32_rotl(v4, 18); + } else { + hash = seed + UINT32_C(374761393); + } + hash += (uint32_t)length; + while (cursor + 4 <= end) { + hash += le32dec(cursor) * UINT32_C(3266489917); + hash = erofs_xxh32_rotl(hash, 17) * UINT32_C(668265263); + cursor += 4; + } + while (cursor < end) { + hash += *cursor++ * UINT32_C(374761393); + hash = erofs_xxh32_rotl(hash, 11) * UINT32_C(2654435761); + } + hash ^= hash >> 15; + hash *= UINT32_C(2246822519); + hash ^= hash >> 13; + hash *= UINT32_C(3266489917); + hash ^= hash >> 16; + return (hash); +} + +static bool +erofs_xattr_filter_name(int attrnamespace, const char *name, size_t name_len, + uint8_t *indexp, const char **filter_namep, size_t *filter_name_lenp) +{ +#if WITH_EROFS_POSIX_ACL + static const char acl_access[] = "posix_acl_access"; + static const char acl_default[] = "posix_acl_default"; +#endif +#if WITH_EROFS_SECURITY + static const char security[] = "security."; +#endif + static const char trusted[] = "trusted."; + + if (attrnamespace == EXTATTR_NAMESPACE_USER) { + *indexp = EROFS_XATTR_INDEX_USER; + *filter_namep = name; + *filter_name_lenp = name_len; + return (true); + } + if (attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (false); +#if WITH_EROFS_POSIX_ACL + if (name_len == sizeof(acl_access) - 1 && + memcmp(name, acl_access, sizeof(acl_access) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } + if (name_len == sizeof(acl_default) - 1 && + memcmp(name, acl_default, sizeof(acl_default) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + *filter_namep = name + name_len; + *filter_name_lenp = 0; + return (true); + } +#endif + if (name_len >= sizeof(trusted) - 1 && + memcmp(name, trusted, sizeof(trusted) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_TRUSTED; + *filter_namep = name + sizeof(trusted) - 1; + *filter_name_lenp = name_len - (sizeof(trusted) - 1); + return (true); + } +#if WITH_EROFS_SECURITY + if (name_len >= sizeof(security) - 1 && + memcmp(name, security, sizeof(security) - 1) == 0) { + *indexp = EROFS_XATTR_INDEX_SECURITY; + *filter_namep = name + sizeof(security) - 1; + *filter_name_lenp = name_len - (sizeof(security) - 1); + return (true); + } +#endif + return (false); +} + +static int +erofs_xattr_read_backing(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, erofs_off_t off, size_t len, + struct erofs_buf *buf) +{ + void *data; + int error; + + /* Both reads check the range (and report it) themselves. */ + if (backing_en != NULL) + error = erofs_read_data(sbi, backing_en, off, len, &data); + else + error = erofs_bread(sbi, off, len, &data); + if (error != 0) + return (error); + buf->data = data; + buf->release = erofs_brelse; + return (0); +} + +static int +erofs_xattr_filter_negative(struct erofs_sb_info *sbi, struct erofs_inode *vi, + int attrnamespace, const char *name, size_t name_len, bool *negativep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + const char *filter_name; + size_t filter_name_len, header_size; + uint32_t hashbit, name_filter; + uint8_t index; + int error; + + *negativep = false; + if (!erofs_sb_has_xattr_filter_v1(sbi) || + !erofs_xattr_filter_name(attrnamespace, name, name_len, &index, + &filter_name, &filter_name_len)) + return (0); + error = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (error != 0) + return (error); + name_filter = le32toh(ih->h_name_filter); + hashbit = erofs_xxh32(filter_name, filter_name_len, + EROFS_XATTR_FILTER_SEED + index); + hashbit &= EROFS_XATTR_FILTER_BITS - 1; + *negativep = (name_filter & (1U << hashbit)) != 0; + error = 0; + erofs_put_metabuf(&body); + return (error); +} + +/* + * Read one prefix table metadata record. + * + * When backing_en == NULL the record is in the physical metadata area; + * otherwise it lives in the selected metadata carrier's logical data stream. + */ +static int +erofs_xattr_read_metadata(struct erofs_sb_info *sbi, + struct erofs_inode *backing_en, + erofs_off_t *offp, struct erofs_buf *buf, size_t *lenp) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + uint16_t raw_len; + erofs_off_t off; + size_t len; + int error; + + /* The offset is below 2^35: 255 records of at most 64 KiB each. */ + off = roundup2(*offp, sizeof(struct erofs_xattr_entry)); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(raw_len), + &hdrbuf); + if (error != 0) + return (error); + raw_len = le16dec(hdrbuf.data); + erofs_put_metabuf(&hdrbuf); + len = (raw_len == 0) ? (size_t)UINT16_MAX + 1 : raw_len; + if (len < sizeof(struct erofs_xattr_long_prefix) || + len > EROFS_NAME_LEN + sizeof(struct erofs_xattr_long_prefix)) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "xattr prefix record of %zu bytes at %ju has an invalid " + "size", + len, (uintmax_t)off)); + error = erofs_xattr_read_backing(sbi, backing_en, + off + sizeof(raw_len), len, buf); + if (error != 0) + return (error); + *offp = off + sizeof(raw_len) + len; + *lenp = len; + return (0); +} + +static int +erofs_xattr_move(void *value, size_t value_size, struct uio *uio, size_t *sizep) +{ + if (sizep != NULL) + *sizep = value_size; + if (uio == NULL || value_size == 0) + return (0); + return (uiomove(value, value_size, uio)); +} + +static int +erofs_xattr_load_body_uncached(struct erofs_sb_info *sbi, + struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + char *body; + uint64_t body_off; + size_t header_size; + int error; + + if (vi->xattr_isize < sizeof(*ih)) + return (erofs_corrupt(sbi, vi->nid, + "xattr area of %zu bytes is smaller than its header", + (size_t)vi->xattr_isize)); + body_off = vi->inode_off + vi->inode_isize; + error = erofs_xattr_read_backing(sbi, + erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL, body_off, + vi->xattr_isize, &buf); + if (error != 0) + return (error); + body = buf.data; + ih = (struct erofs_xattr_ibody_header *)body; + if (vi->xattr_isize == sizeof(*ih)) { + error = EOPNOTSUPP; + goto fail; + } + header_size = sizeof(*ih) + sizeof(uint32_t) * ih->h_shared_count; + if (header_size > vi->xattr_isize) { + error = erofs_corrupt(sbi, vi->nid, + "%u shared xattr ids do not fit the xattr area of %zu " + "bytes", + ih->h_shared_count, (size_t)vi->xattr_isize); + goto fail; + } + *bodybuf = buf; + *ihp = ih; + *header_sizep = header_size; + return (0); +fail: + erofs_put_metabuf(&buf); + return (error); +} + +static int +erofs_xattr_validate_entry(struct erofs_xattr_entry *entry, size_t remaining, + size_t *entry_sizep, size_t *value_sizep) +{ + size_t entry_size, value_size; + + if (remaining < sizeof(*entry)) + return (EINTEGRITY); + value_size = le16toh(entry->e_value_size); + /* The aligned size covers the header, the name and the value. */ + entry_size = erofs_xattr_entry_size(entry); + if (entry_size > remaining) + return (EINTEGRITY); + if (entry_sizep != NULL) + *entry_sizep = entry_size; + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static bool +erofs_xattr_prefix(uint8_t base_index, int *namespacep, + const char **prefixp, size_t *prefix_lenp) +{ + switch (base_index) { + case EROFS_XATTR_INDEX_USER: + *namespacep = EXTATTR_NAMESPACE_USER; + *prefixp = NULL; + *prefix_lenp = 0; + return (true); +#if WITH_EROFS_POSIX_ACL + case EROFS_XATTR_INDEX_POSIX_ACL_ACCESS: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_access"; + *prefix_lenp = sizeof("posix_acl_access") - 1; + return (true); + case EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "posix_acl_default"; + *prefix_lenp = sizeof("posix_acl_default") - 1; + return (true); +#endif + case EROFS_XATTR_INDEX_TRUSTED: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "trusted."; + *prefix_lenp = sizeof("trusted.") - 1; + return (true); +#if WITH_EROFS_SECURITY + case EROFS_XATTR_INDEX_SECURITY: + *namespacep = EXTATTR_NAMESPACE_SYSTEM; + *prefixp = "security."; + *prefix_lenp = sizeof("security.") - 1; + return (true); +#endif + case EROFS_XATTR_INDEX_LUSTRE: + default: + return (false); + } +} + +static int +erofs_xattr_namespace_prefix(int attrnamespace, uint8_t base_index, + const char **prefixp, size_t *prefix_lenp) +{ + int mapped_namespace; + + if (attrnamespace != EXTATTR_NAMESPACE_USER && + attrnamespace != EXTATTR_NAMESPACE_SYSTEM) + return (EOPNOTSUPP); + if (!erofs_xattr_prefix(base_index, &mapped_namespace, prefixp, + prefix_lenp)) + return (ENOATTR); + if (mapped_namespace != attrnamespace) + return (ENOATTR); + return (0); +} + +static int +erofs_xattr_list_move(const char *namespace_prefix, size_t namespace_prefix_len, + const char *infix, size_t infix_len, const char *name, uint8_t name_len, + struct uio *uio, size_t *sizep) +{ + uint8_t total_name_len; + int error; + + if (namespace_prefix_len + infix_len + name_len > EROFS_NAME_LEN) + return (EINTEGRITY); + total_name_len = namespace_prefix_len + infix_len + name_len; + if (sizep != NULL) { + *sizep += total_name_len + 1; + return (0); + } + if (uio == NULL) + return (0); + error = uiomove(__DECONST(void *, &total_name_len), 1, uio); + if (error != 0) + return (error); + if (namespace_prefix_len != 0) { + error = uiomove(__DECONST(void *, namespace_prefix), + namespace_prefix_len, uio); + if (error != 0) + return (error); + } + if (infix_len != 0) { + error = uiomove(__DECONST(void *, infix), infix_len, uio); + if (error != 0) + return (error); + } + return (uiomove(__DECONST(void *, name), name_len, uio)); +} + +static int +erofs_xattr_resolve_name(struct erofs_sb_info *sbi, + const struct erofs_xattr_entry *entry, uint8_t *base_indexp, + const char **infixp, size_t *infix_lenp) +{ + struct erofs_xattr_prefix_item *prefix; + uint8_t prefix_id; + + if ((entry->e_name_index & EROFS_XATTR_LONG_PREFIX) == 0) { + *base_indexp = entry->e_name_index; + *infixp = NULL; + *infix_lenp = 0; + return (0); + } + if (sbi->xattr_prefixes == NULL) + return (ENOATTR); + prefix_id = entry->e_name_index & EROFS_XATTR_LONG_PREFIX_MASK; + if (prefix_id >= sbi->xattr_prefix_count) + return (ENOATTR); + prefix = &sbi->xattr_prefixes[prefix_id]; + *base_indexp = prefix->base_index; + *infixp = prefix->infix; + *infix_lenp = prefix->infix_len; + return (0); +} + +static bool +erofs_xattr_name_match(const char *namespace_prefix, + size_t namespace_prefix_len, const char *infix, size_t infix_len, + const struct erofs_xattr_entry *entry, const char *name, size_t name_len) +{ + if (name_len != namespace_prefix_len + infix_len + entry->e_name_len) + return (false); + if (namespace_prefix_len != 0 && + memcmp(name, namespace_prefix, namespace_prefix_len) != 0) + return (false); + if (infix_len != 0 && + memcmp(name + namespace_prefix_len, infix, infix_len) != 0) + return (false); + return (memcmp(name + namespace_prefix_len + infix_len, entry->e_name, + entry->e_name_len) == 0); +} + +static erofs_off_t +erofs_xattr_shared_entry_offset(const struct erofs_sb_info *sbi, + uint32_t shared_id) +{ + /* A 32-bit block address and a 16-bit shift: below 2^48. */ + return (((erofs_off_t)sbi->xattr_blkaddr << sbi->blkszbits) + + (erofs_off_t)shared_id * sizeof(uint32_t)); +} + +static int +erofs_xattr_load_shared_entry(struct erofs_sb_info *sbi, uint32_t shared_id, + struct erofs_buf *entrybuf, size_t *value_sizep) +{ + struct erofs_buf hdrbuf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + struct erofs_inode *backing_en; + erofs_off_t off; + size_t entry_size, value_size; + int error; + + backing_en = erofs_sb_has_shared_ea_in_metabox(sbi) ? + sbi->metabox_en : NULL; + if (erofs_sb_has_shared_ea_in_metabox(sbi) && backing_en == NULL) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "shared xattrs are in the metabox, but the image has " + "none")); + + off = erofs_xattr_shared_entry_offset(sbi, shared_id); + error = erofs_xattr_read_backing(sbi, backing_en, off, sizeof(*entry), + &hdrbuf); + if (error != 0) + return (error); + entry = hdrbuf.data; + value_size = le16toh(entry->e_value_size); + entry_size = erofs_xattr_entry_size(entry); + + erofs_put_metabuf(&hdrbuf); + + /* + * The entry is read whole, so the name and the value are inside it by + * construction (entry_size covers both). + */ + error = erofs_xattr_read_backing(sbi, backing_en, off, entry_size, + entrybuf); + if (error != 0) + return (error); + if (value_sizep != NULL) + *value_sizep = value_size; + return (0); +} + +static int +erofs_listxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int err; + + err = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (err == ENOATTR) + return (0); + if (err != 0) + return (err); + err = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (err == ENOATTR) + return (0); + if (err != 0) + return (err); + err = erofs_xattr_list_move(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry->e_name, entry->e_name_len, it->uio, + it->sizep); + if (err == EINTEGRITY) + return (erofs_corrupt(it->sbi, it->vi->nid, + "xattr name is longer than %d bytes", EROFS_NAME_LEN)); + return (err); +} + +static int +erofs_getxattr_foreach(struct erofs_xattr_iter *it, + struct erofs_xattr_entry *entry, size_t value_size) +{ + const char *infix, *namespace_prefix; + size_t infix_len, namespace_prefix_len; + uint8_t base_index; + int error; + + error = erofs_xattr_resolve_name(it->sbi, entry, &base_index, &infix, + &infix_len); + if (error != 0) + return (error); + error = erofs_xattr_namespace_prefix(it->attrnamespace, base_index, + &namespace_prefix, &namespace_prefix_len); + if (error != 0) + return (error); + if (!erofs_xattr_name_match(namespace_prefix, namespace_prefix_len, + infix, infix_len, entry, it->name, it->name_len)) + return (ENOATTR); + return (erofs_xattr_move(entry->e_name + entry->e_name_len, value_size, + it->uio, it->sizep)); +} + +static int +erofs_xattr_iter_inline(struct erofs_xattr_iter *it, char *body, + size_t header_size, bool get) +{ + struct erofs_xattr_entry *entry; + char *cursor; + size_t entry_size, remaining, value_size; + int ret; + + remaining = it->vi->xattr_isize - header_size; + cursor = body + header_size; + while (remaining != 0) { + entry = (struct erofs_xattr_entry *)cursor; + ret = erofs_xattr_validate_entry(entry, remaining, + &entry_size, get ? &value_size : NULL); + if (ret != 0) + return (erofs_corrupt(it->sbi, it->vi->nid, + "invalid xattr entry at offset %zu of the xattr " + "area", + (size_t)(cursor - body))); + if (get) + ret = erofs_getxattr_foreach(it, entry, value_size); + else + ret = erofs_listxattr_foreach(it, entry); + if (get) { + if (ret != ENOATTR) + return (ret); + } else if (ret != 0) { + return (ret); + } + cursor += entry_size; + remaining -= entry_size; + } + return (get ? ENOATTR : 0); +} + +static int +erofs_xattr_iter_shared(struct erofs_xattr_iter *it, + struct erofs_xattr_ibody_header *ih, bool get) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_entry *entry; + uint32_t shared_id; + size_t value_size; + int ret; + + for (uint8_t i = 0; i < ih->h_shared_count; i++) { + shared_id = le32toh(ih->h_shared_xattrs[i]); + ret = erofs_xattr_load_shared_entry(it->sbi, shared_id, &buf, + get ? &value_size : NULL); + if (ret != 0) + return (ret); + entry = buf.data; + if (get) + ret = erofs_getxattr_foreach(it, entry, value_size); + else + ret = erofs_listxattr_foreach(it, entry); + erofs_put_metabuf(&buf); + if (get) { + if (ret != ENOATTR) + return (ret); + } else if (ret != 0) { + return (ret); + } + } + return (get ? ENOATTR : 0); +} + +/* + * Name exposure rules: + * - user namespace: bare name, no "user." prefix; + * - system namespace: the full "trusted.*" and "security.*" names, and + * "posix_acl_access" and "posix_acl_default" for the ACL attributes. + */ +int +erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size, name_len; + bool filter_negative; + int ret; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (name == NULL || name[0] == '\0') + return (EINVAL); + name_len = strlen(name); + if (name_len > EROFS_NAME_LEN) + return (EINVAL); + if (vi->xattr_isize == 0) + return (ENOATTR); + ret = erofs_xattr_filter_negative(sbi, vi, attrnamespace, name, + name_len, &filter_negative); + if (ret != 0) + return (ret); + if (filter_negative) + return (ENOATTR); + ret = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (ret != 0) + return (ret); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = name; + it.name_len = name_len; + it.uio = uio; + it.sizep = sizep; + ret = erofs_xattr_iter_inline(&it, body.data, header_size, true); + if (ret == ENOATTR) + ret = erofs_xattr_iter_shared(&it, ih, true); + erofs_put_metabuf(&body); + return (ret); +} + +/* + * Return format: 1-byte name length followed by non-NUL-terminated name bytes. + */ +int +erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep) +{ + struct erofs_buf body = EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_xattr_ibody_header *ih; + struct erofs_xattr_iter it; + size_t header_size; + int ret; + + sbi = MTOE(vp->v_mount); + vi = VTOE(vp); + if (sizep != NULL) + *sizep = 0; + if (vi->xattr_isize == 0) + return (0); + ret = erofs_xattr_load_body(sbi, vi, &body, &ih, &header_size); + if (ret != 0) + return (ret); + it.sbi = sbi; + it.vi = vi; + it.attrnamespace = attrnamespace; + it.name = NULL; + it.name_len = 0; + it.uio = uio; + it.sizep = sizep; + ret = erofs_xattr_iter_inline(&it, body.data, header_size, false); + if (ret == 0) + ret = erofs_xattr_iter_shared(&it, ih, false); + erofs_put_metabuf(&body); + return (ret); +} + +static bool +erofs_xattr_cache_reserve(struct erofs_sb_info *sbi, size_t size) +{ + u_long resident; + + for (;;) { + resident = atomic_load_acq_long(&sbi->xattr_cache_resident); + if (resident > EROFS_XATTR_CACHE_MOUNT_BUDGET || + size > EROFS_XATTR_CACHE_MOUNT_BUDGET - resident) + return (false); + if (atomic_cmpset_acq_long(&sbi->xattr_cache_resident, resident, + resident + size)) + return (true); + } +} + +static void +erofs_xattr_cache_release(struct erofs_sb_info *sbi, size_t size) +{ + KASSERT(size <= EROFS_XATTR_CACHE_MOUNT_BUDGET, + ("erofs xattr cache release exceeds mount budget")); + atomic_subtract_rel_long(&sbi->xattr_cache_resident, size); +} + +static void * +erofs_xattr_cache_invalidate_locked(struct erofs_sb_info *sbi, + struct erofs_xattr_cache *cache) +{ + void *data; + + data = cache->data; + if (cache->charged_bytes != 0) { + erofs_xattr_cache_release(sbi, cache->charged_bytes); + cache->charged_bytes = 0; + } + cache->data = NULL; + cache->error = 0; + cache->state = EROFS_XATTR_CACHE_EMPTY; + return (data); +} + +void +erofs_xattr_cache_init(struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + + cache = &vi->xattr_cache; + bzero(cache, sizeof(*cache)); + mtx_init(&cache->lock, "erofs xattr", NULL, MTX_DEF); + cv_init(&cache->cv, "erofs xattr"); + cache->state = EROFS_XATTR_CACHE_EMPTY; + cache->initialized = true; +} + +void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_xattr_cache *cache; + void *data; + + cache = &vi->xattr_cache; + if (!cache->initialized) + return; + mtx_lock(&cache->lock); + cache->closing = true; + while (cache->state == EROFS_XATTR_CACHE_INFLIGHT || + cache->waiters != 0) + cv_wait(&cache->cv, &cache->lock); + data = erofs_xattr_cache_invalidate_locked(sbi, cache); + cache->initialized = false; + mtx_unlock(&cache->lock); + free(data, M_EROFS); + cv_destroy(&cache->cv); + mtx_destroy(&cache->lock); +} + +static int +erofs_xattr_load_body(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_buf *bodybuf, struct erofs_xattr_ibody_header **ihp, + size_t *header_sizep) +{ + struct erofs_xattr_cache *cache; + bool reserved; + int error; + + cache = &vi->xattr_cache; + if (!cache->initialized || + vi->xattr_isize > EROFS_XATTR_CACHE_BODY_LIMIT) + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep)); + mtx_lock(&cache->lock); + if (cache->closing) { + mtx_unlock(&cache->lock); + return (ENXIO); + } + if (cache->state == EROFS_XATTR_CACHE_READY) { + KASSERT(cache->data != NULL, + ("erofs xattr ready cache has no body")); + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + mtx_unlock(&cache->lock); + return (0); + } + if (cache->state == EROFS_XATTR_CACHE_INFLIGHT) { + cache->waiters++; + do { + cv_wait(&cache->cv, &cache->lock); + } while (cache->state == EROFS_XATTR_CACHE_INFLIGHT); + if (cache->state == EROFS_XATTR_CACHE_READY) { + bodybuf->data = cache->data; + *ihp = bodybuf->data; + *header_sizep = sizeof(**ihp) + + sizeof(uint32_t) * (*ihp)->h_shared_count; + error = 0; + } else { + KASSERT(cache->state == EROFS_XATTR_CACHE_FAILED && + cache->error > 0, + ("erofs xattr failed cache has no typed error")); + error = cache->error; + } + cache->waiters--; + if (cache->state == EROFS_XATTR_CACHE_FAILED && + cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + if (cache->waiters == 0) + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (error); + } + if (cache->state == EROFS_XATTR_CACHE_FAILED) { + if (cache->waiters != 0) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, + ihp, header_sizep)); + } + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + } + reserved = erofs_xattr_cache_reserve(sbi, vi->xattr_isize); + if (!reserved) { + mtx_unlock(&cache->lock); + return (erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep)); + } + cache->state = EROFS_XATTR_CACHE_INFLIGHT; + cache->error = 0; + mtx_unlock(&cache->lock); + + error = erofs_xattr_load_body_uncached(sbi, vi, bodybuf, ihp, + header_sizep); + mtx_lock(&cache->lock); + if (error == 0 && !cache->closing) { + cache->data = bodybuf->data; + cache->charged_bytes = vi->xattr_isize; + cache->state = EROFS_XATTR_CACHE_READY; + bodybuf->release = NULL; + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + return (0); + } + if (error == 0) + error = ENXIO; + erofs_xattr_cache_release(sbi, vi->xattr_isize); + cache->error = error; + cache->state = EROFS_XATTR_CACHE_FAILED; + if (cache->waiters == 0) + (void)erofs_xattr_cache_invalidate_locked(sbi, cache); + cv_broadcast(&cache->cv); + mtx_unlock(&cache->lock); + erofs_put_metabuf(bodybuf); + return (error); +} + +void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi) +{ + if (sbi->xattr_prefixes == NULL) + return; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) + free(sbi->xattr_prefixes[i].infix, M_EROFS); + free(sbi->xattr_prefixes, M_EROFS); + sbi->xattr_prefixes = NULL; +} + +int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_long_prefix *prefix = NULL; + struct erofs_inode *prefix_en; + erofs_off_t off; + size_t infix_len, len; + int ret; + + if (!erofs_sb_has_xattr_prefixes(sbi) || sbi->xattr_prefix_count == 0) + return (0); + prefix_en = NULL; + if (!erofs_sb_has_plain_xattr_pfx(sbi)) { + if (erofs_sb_has_metabox(sbi)) { + if (sbi->metabox_en == NULL) + return (erofs_corrupt(sbi, EROFS_NID_NONE, + "xattr prefixes are in the metabox, but " + "the image has none")); + prefix_en = sbi->metabox_en; + } else if (sbi->packed_inode != NULL) { + prefix_en = sbi->packed_inode; + } + } + sbi->xattr_prefixes = malloc(sizeof(*sbi->xattr_prefixes) * + sbi->xattr_prefix_count, + M_EROFS, M_WAITOK | M_ZERO); + off = (uint64_t)sbi->xattr_prefix_start << 2; + for (uint8_t i = 0; i < sbi->xattr_prefix_count; i++) { + ret = erofs_xattr_read_metadata(sbi, prefix_en, &off, + &buf, &len); + if (ret != 0) + goto fail; + prefix = buf.data; + infix_len = len - sizeof(*prefix); + sbi->xattr_prefixes[i].base_index = prefix->base_index; + sbi->xattr_prefixes[i].infix_len = infix_len; + sbi->xattr_prefixes[i].infix = malloc(infix_len + 1, M_EROFS, + M_WAITOK); + memcpy(sbi->xattr_prefixes[i].infix, prefix->infix, infix_len); + sbi->xattr_prefixes[i].infix[infix_len] = '\0'; + erofs_put_metabuf(&buf); + } + return (0); +fail: + erofs_put_metabuf(&buf); + erofs_xattr_prefixes_cleanup(sbi); + return (ret); +} + +#if WITH_EROFS_POSIX_ACL +static int +erofs_inode_has_noacl(struct erofs_sb_info *sbi, struct erofs_inode *vi, + bool *noaclp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_xattr_ibody_header *ih; + struct erofs_inode *backing_en; + erofs_off_t body_off; + uint32_t name_filter; + int error; + + *noaclp = false; + if (vi->xattr_isize < sizeof(*ih)) { + *noaclp = true; + return (0); + } + if (!erofs_sb_has_xattr_filter(sbi) || + sbi->xattr_filter_reserved != 0) + return (0); + body_off = vi->inode_off + vi->inode_isize; + backing_en = erofs_nid_in_metabox(vi->nid) ? sbi->metabox_en : NULL; + error = erofs_xattr_read_backing(sbi, backing_en, body_off, sizeof(*ih), + &buf); + if (error != 0) + return (error); + ih = buf.data; + name_filter = le32toh(ih->h_name_filter); + erofs_put_metabuf(&buf); + *noaclp = (name_filter & EROFS_XATTR_FILTER_POSIX_ACL) == + EROFS_XATTR_FILTER_POSIX_ACL; + return (0); +} + +static void +erofs_acl_from_mode(mode_t mode, acl_type_t type, struct acl *aclp) +{ + if (type == ACL_TYPE_DEFAULT) { + aclp->acl_cnt = 0; + return; + } + aclp->acl_cnt = 3; + aclp->acl_entry[0].ae_tag = ACL_USER_OBJ; + aclp->acl_entry[0].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[0].ae_perm = acl_posix1e_mode_to_perm(ACL_USER_OBJ, + mode); + aclp->acl_entry[1].ae_tag = ACL_GROUP_OBJ; + aclp->acl_entry[1].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[1].ae_perm = acl_posix1e_mode_to_perm(ACL_GROUP_OBJ, + mode); + aclp->acl_entry[2].ae_tag = ACL_OTHER; + aclp->acl_entry[2].ae_id = ACL_UNDEFINED_ID; + aclp->acl_entry[2].ae_perm = acl_posix1e_mode_to_perm(ACL_OTHER, mode); +} + +static int +erofs_posix_acl_from_xattr(const void *value, size_t size, mode_t mode, + acl_type_t type, struct acl *aclp) +{ + const uint8_t *buf; + struct posix_acl_xattr_header hdr; + struct posix_acl_xattr_entry entry; + uint32_t id; + int count, i; + + buf = value; + if (size < sizeof(hdr) || (size - sizeof(hdr)) % sizeof(entry) != 0) + return (EINTEGRITY); + + memcpy(&hdr, buf, sizeof(hdr)); + if (le32toh(hdr.a_version) != POSIX_ACL_XATTR_VERSION) + return (EINTEGRITY); + + count = (size - sizeof(hdr)) / sizeof(entry); + if (count > ACL_MAX_ENTRIES) + return (EINTEGRITY); + if (count == 0) { + erofs_acl_from_mode(mode, type, aclp); + return (0); + } + + aclp->acl_cnt = count; + for (i = 0; i < count; i++) { + memcpy(&entry, buf + sizeof(hdr) + i * sizeof(entry), + sizeof(entry)); + id = le32toh(entry.e_id); + aclp->acl_entry[i].ae_tag = le16toh(entry.e_tag); + aclp->acl_entry[i].ae_perm = le16toh(entry.e_perm); + aclp->acl_entry[i].ae_id = (id == UINT32_MAX) ? + ACL_UNDEFINED_ID : id; + } + /* + * Tags, their counts, permission bits and unnamed ids are the + * kernel's ACL contract: acl_posix1e_check() enforces them. Entry + * order and duplicate ids are not checked: vaccess_acl_posix1e() + * does not depend on either, and Linux accepts both. + */ + if (acl_posix1e_check(aclp) != 0) + return (EINTEGRITY); + + return (0); +} + +int +erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp) +{ + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + const char *xattr_name; + struct uio auio; + struct iovec aiov; + uint8_t buf[sizeof(struct posix_acl_xattr_header) + + sizeof(struct posix_acl_xattr_entry) * ACL_MAX_ENTRIES]; + size_t size; + bool noacl; + int error; + + sbi = MTOE(vp->v_mount); + if (!test_opt(&sbi->opt, POSIX_ACL)) + return (EOPNOTSUPP); + + vi = VTOE(vp); + + switch (type) { + case ACL_TYPE_ACCESS: + xattr_name = "posix_acl_access"; + break; + case ACL_TYPE_DEFAULT: + if (vp->v_type != VDIR) + return (EINVAL); + xattr_name = "posix_acl_default"; + break; + default: + return (EINVAL); + } + error = erofs_inode_has_noacl(sbi, vi, &noacl); + if (error != 0) + return (error); + if (noacl) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, NULL, + &size); + if (error == ENOATTR) { + erofs_acl_from_mode(vi->mode, type, aclp); + return (0); + } + if (error != 0) + return (error); + if (size > sizeof(buf)) + return (erofs_corrupt(sbi, vi->nid, + "ACL xattr of %zu bytes is too large", size)); + + aiov.iov_base = buf; + aiov.iov_len = size; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_offset = 0; + auio.uio_resid = size; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_rw = UIO_READ; + auio.uio_td = curthread; + error = erofs_getxattr(vp, EXTATTR_NAMESPACE_SYSTEM, xattr_name, &auio, + NULL); + if (error != 0) + return (error); + if (auio.uio_resid != 0) + return (erofs_corrupt(sbi, vi->nid, + "short read of the ACL xattr")); + + error = erofs_posix_acl_from_xattr(buf, size, vi->mode, type, aclp); + if (error == EINTEGRITY) + return (erofs_corrupt(sbi, vi->nid, + "invalid ACL xattr of %zu bytes", size)); + return (error); +} +#endif diff --git a/xattr.h b/xattr.h new file mode 100644 index 0000000..d7d17a5 --- /dev/null +++ b/xattr.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef _FS_EROFS_XATTR_H_ +#define _FS_EROFS_XATTR_H_ + +#if WITH_EROFS_XATTR && WITH_EROFS_POSIX_ACL +#include +#endif + +#include "internal.h" + +#if WITH_EROFS_XATTR +int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi); +void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi); +int erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio, + size_t *sizep); +int erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name, + struct uio *uio, size_t *sizep); +#if WITH_EROFS_POSIX_ACL +int erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp); +#endif +void erofs_xattr_cache_init(struct erofs_inode *vi); +void erofs_xattr_cache_fini(struct erofs_sb_info *sbi, + struct erofs_inode *vi); +#else +static inline int +erofs_xattr_prefixes_init(struct erofs_sb_info *sbi __unused) +{ + return (0); +} + +static inline void +erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi __unused) +{ +} + +static inline void +erofs_xattr_cache_init(struct erofs_inode *vi __unused) +{ +} + +static inline void +erofs_xattr_cache_fini(struct erofs_sb_info *sbi __unused, + struct erofs_inode *vi __unused) +{ +} +#endif +#endif diff --git a/zdata.c b/zdata.c new file mode 100644 index 0000000..6e91482 --- /dev/null +++ b/zdata.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" + +enum z_erofs_cache_lookup { + Z_EROFS_CACHE_BYPASS, + Z_EROFS_CACHE_HIT, + Z_EROFS_CACHE_OWNER, + Z_EROFS_CACHE_ERROR, +}; + +#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024) +#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024) +#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024) +#define EROFS_ZCACHE_WORKER_HARD_LIMIT EROFS_ZCACHE_MAX_ENTRIES + +static int z_erofs_cache_enabled = 1; +static unsigned long z_erofs_cache_mount_budget = + EROFS_ZCACHE_MOUNT_HARD_BUDGET; +static unsigned long z_erofs_cache_global_budget = + EROFS_ZCACHE_GLOBAL_HARD_BUDGET; +static unsigned long z_erofs_cache_minimum_decode_work = + EROFS_ZCACHE_MIN_DECODE_WORK; +static unsigned long z_erofs_cache_worker_limit = + EROFS_ZCACHE_WORKER_HARD_LIMIT; + +TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled); +TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget", + &z_erofs_cache_mount_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget", + &z_erofs_cache_global_budget); +TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work", + &z_erofs_cache_minimum_decode_work); +TUNABLE_ULONG("vfs.erofs.decoded_cache.worker_limit", + &z_erofs_cache_worker_limit); + +static struct mtx z_erofs_cache_list_lock; +static struct mtx z_erofs_cache_budget_lock; +static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts = + LIST_HEAD_INITIALIZER(z_erofs_cache_mounts); +static size_t z_erofs_cache_global_charged; +static eventhandler_tag z_erofs_cache_lowmem_tag; + +MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock, + "erofs zcache list", MTX_DEF); +MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock, + "erofs zcache budget", MTX_DEF); + +static size_t +z_erofs_extent_cache_global_limit(void) +{ + + return (MIN(z_erofs_cache_global_budget, + EROFS_ZCACHE_GLOBAL_HARD_BUDGET)); +} + +static bool +z_erofs_extent_cache_reserve(size_t bytes) +{ + size_t limit; + bool reserved; + + reserved = false; + mtx_lock(&z_erofs_cache_budget_lock); + limit = z_erofs_extent_cache_global_limit(); + if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) { + z_erofs_cache_global_charged += bytes; + reserved = true; + } + mtx_unlock(&z_erofs_cache_budget_lock); + return (reserved); +} + +static void +z_erofs_extent_cache_release(size_t bytes) +{ + + mtx_lock(&z_erofs_cache_budget_lock); + KASSERT(bytes <= z_erofs_cache_global_charged, + ("erofs decoded cache budget underflow")); + z_erofs_cache_global_charged -= bytes; + mtx_unlock(&z_erofs_cache_budget_lock); +} + +static bool +z_erofs_extent_cache_match(const struct erofs_zextent_cache_entry *entry, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size) +{ + const struct erofs_zextent_key *key = &entry->key; + + return (entry->active && entry->state != EROFS_ZCACHE_EMPTY && + key->nid == vi->nid && key->decoded_size == decoded_size && + key->map.m_pa == map->m_pa && key->map.m_la == map->m_la && + key->map.m_plen == map->m_plen && key->map.m_llen == map->m_llen && + key->map.m_deviceid == map->m_deviceid && + key->map.m_algorithmformat == map->m_algorithmformat && + key->map.m_flags == map->m_flags); +} + +static void +z_erofs_extent_cache_reset_locked(struct erofs_zextent_cache *cache, + struct erofs_zextent_cache_entry *entry) +{ + + KASSERT(entry->state != EROFS_ZCACHE_INFLIGHT && entry->waiters == 0, + ("erofs cache entry reset while busy")); + if (entry->active) { + KASSERT(cache->entry_count != 0, + ("erofs cache entry underflow")); + --cache->entry_count; + } + entry->active = false; + entry->state = EROFS_ZCACHE_EMPTY; + entry->error = 0; + entry->lru = 0; + bzero(&entry->key, sizeof(entry->key)); +} + +static void * +z_erofs_extent_cache_drop_entry_locked(struct erofs_sb_info *sbi __unused, + struct erofs_zextent_cache *cache, struct erofs_zextent_cache_entry *entry, + bool evicted, bool reclaimed) +{ + struct erofs_zextent_cache_metrics *metrics; + void *data; + + KASSERT(entry->state == EROFS_ZCACHE_READY && entry->waiters == 0, + ("erofs cache drop of busy entry")); + data = entry->data; + metrics = &cache->metrics[entry->key.map.m_algorithmformat]; + KASSERT(metrics->resident_bytes >= entry->charged_bytes, + ("erofs decoded cache codec accounting mismatch")); + metrics->resident_bytes -= entry->charged_bytes; + if (evicted) + ++metrics->evictions; + if (reclaimed) + ++metrics->reclaims; + KASSERT(cache->charged_bytes >= entry->charged_bytes, + ("erofs decoded cache mount accounting underflow")); + cache->charged_bytes -= entry->charged_bytes; + z_erofs_extent_cache_release(entry->charged_bytes); + entry->data = NULL; + entry->charged_bytes = 0; + z_erofs_extent_cache_reset_locked(cache, entry); + cv_broadcast(&entry->cv); + return (data); +} + +static enum z_erofs_cache_lookup +z_erofs_extent_cache_claim(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_metrics *metrics; + struct erofs_zextent_cache_entry *entry, *victim; + void *old; + enum z_erofs_cache_lookup result; + unsigned int i; + + KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS")); + KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff, + ("erofs extent cache copy exceeds decoded extent")); + *errorp = 0; + if (!sbi->z_extent_cache_initialized) + return (Z_EROFS_CACHE_BYPASS); + cache = &sbi->z_extent_cache; + KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX, + ("erofs decoded cache claim has unknown codec")); + metrics = &cache->metrics[map->m_algorithmformat]; + entry = NULL; + victim = NULL; + old = NULL; + mtx_lock(&sbi->z_extent_cache_lock); + if (cache->closing) { + ++metrics->bypasses; + goto bypass; + } + for (i = 0; i < nitems(cache->entries); ++i) { + if (z_erofs_extent_cache_match(&cache->entries[i], vi, map, + decoded_size)) { + entry = &cache->entries[i]; + break; + } + } + if (entry != NULL) { + if (entry->state == EROFS_ZCACHE_READY) { + KASSERT(entry->data != NULL, + ("erofs ready extent cache has no data")); + memcpy(dst, (char *)entry->data + (size_t)mapoff, len); + entry->lru = ++cache->lru_clock; + ++metrics->hits; + mtx_unlock(&sbi->z_extent_cache_lock); + return (Z_EROFS_CACHE_HIT); + } + if (entry->state == EROFS_ZCACHE_INFLIGHT) { + ++metrics->waiter_merges; + ++entry->waiters; + do { + cv_wait(&entry->cv, &sbi->z_extent_cache_lock); + } while (entry->state == EROFS_ZCACHE_INFLIGHT); + if (entry->state == EROFS_ZCACHE_READY) { + memcpy(dst, + (char *)entry->data + (size_t)mapoff, len); + entry->lru = ++cache->lru_clock; + ++metrics->hits; + result = Z_EROFS_CACHE_HIT; + } else { + KASSERT(entry->state == EROFS_ZCACHE_FAILED && + entry->error > 0, + ("erofs inflight cache has no typed " + "result")); + *errorp = entry->error; + result = Z_EROFS_CACHE_ERROR; + } + --entry->waiters; + if (entry->state == EROFS_ZCACHE_FAILED && + entry->waiters == 0) + z_erofs_extent_cache_reset_locked(cache, entry); + if (entry->waiters == 0) + cv_broadcast(&entry->cv); + mtx_unlock(&sbi->z_extent_cache_lock); + return (result); + } + if (entry->waiters != 0) { + ++metrics->bypasses; + goto bypass; + } + z_erofs_extent_cache_reset_locked(cache, entry); + } + for (i = 0; i < nitems(cache->entries); ++i) { + if (!cache->entries[i].active) { + entry = &cache->entries[i]; + break; + } + if (cache->entries[i].state == EROFS_ZCACHE_READY && + cache->entries[i].waiters == 0 && + (victim == NULL || cache->entries[i].lru < victim->lru)) + victim = &cache->entries[i]; + } + if (entry == NULL && victim != NULL) { + entry = victim; + old = z_erofs_extent_cache_drop_entry_locked(sbi, cache, entry, + true, false); + } + if (entry == NULL || decoded_size > cache->budget_bytes || + cache->charged_bytes > cache->budget_bytes - decoded_size || + !z_erofs_extent_cache_reserve(decoded_size)) { + ++metrics->bypasses; + goto bypass; + } + entry->key.map = *map; + entry->key.nid = vi->nid; + entry->key.decoded_size = decoded_size; + entry->charged_bytes = decoded_size; + entry->error = 0; + entry->state = EROFS_ZCACHE_INFLIGHT; + entry->active = true; + entry->lru = ++cache->lru_clock; + ++cache->entry_count; + cache->charged_bytes += decoded_size; + ++metrics->misses; + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_OWNER); + +bypass: + mtx_unlock(&sbi->z_extent_cache_lock); + free(old, M_EROFS); + return (Z_EROFS_CACHE_BYPASS); +} + +static void +z_erofs_extent_cache_complete(struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t decoded_size, void *decoded, int error) +{ + struct erofs_zextent_cache *cache; + struct erofs_zextent_cache_entry *entry; + struct erofs_zextent_cache_metrics *metrics; + unsigned int i; + + KASSERT((error == 0) == (decoded != NULL), + ("erofs extent cache completion is untyped")); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + entry = NULL; + for (i = 0; i < nitems(cache->entries); ++i) { + if (z_erofs_extent_cache_match(&cache->entries[i], vi, map, + decoded_size)) { + entry = &cache->entries[i]; + break; + } + } + KASSERT(entry != NULL && entry->state == EROFS_ZCACHE_INFLIGHT, + ("erofs extent cache owner lost its key")); + metrics = &cache->metrics[map->m_algorithmformat]; + if (error == 0) { + KASSERT(entry->charged_bytes == decoded_size, + ("erofs decoded cache success accounting mismatch")); + entry->data = decoded; + metrics->resident_bytes += decoded_size; + entry->error = 0; + entry->state = EROFS_ZCACHE_READY; + entry->lru = ++cache->lru_clock; + } else { + KASSERT(error > 0, + ("erofs extent cache published negative errno")); + KASSERT(cache->charged_bytes >= entry->charged_bytes, + ("erofs decoded cache accounting underflow")); + cache->charged_bytes -= entry->charged_bytes; + z_erofs_extent_cache_release(entry->charged_bytes); + entry->charged_bytes = 0; + entry->error = error; + entry->state = EROFS_ZCACHE_FAILED; + if (entry->waiters == 0) + z_erofs_extent_cache_reset_locked(cache, entry); + } + cv_broadcast(&entry->cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_extent_cache_lowmem(void *arg __unused, int howto __unused) +{ + struct erofs_zextent_cache *cache; + struct erofs_sb_info *sbi; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + mtx_lock(&z_erofs_cache_list_lock); + LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) { + count = 0; + if (!mtx_trylock(&sbi->z_extent_cache_lock)) + continue; + cache = &sbi->z_extent_cache; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY && + cache->entries[i].waiters == 0) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], true, true); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + } + mtx_unlock(&z_erofs_cache_list_lock); +} + +static void +z_erofs_extent_cache_global_init(void *arg __unused) +{ + z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +z_erofs_extent_cache_global_fini(void *arg __unused) +{ + if (z_erofs_cache_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag); + KASSERT(LIST_EMPTY(&z_erofs_cache_mounts), + ("erofs decoded cache mounts remain at unload")); + KASSERT(z_erofs_cache_global_charged == 0, + ("erofs decoded cache bytes remain at unload")); +} + +SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST, + z_erofs_extent_cache_global_init, NULL); +SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST, + z_erofs_extent_cache_global_fini, NULL); + +#if WITH_EROFS_PCPU_KTHREAD +static unsigned int +z_erofs_decode_worker_count(void) +{ + unsigned long count; + + count = z_erofs_cache_worker_limit; + if (count == 0) + count = 1; + count = MIN(count, (unsigned long)EROFS_ZCACHE_WORKER_HARD_LIMIT); + count = MIN(count, (unsigned long)MAX(1, mp_ncpus)); + return ((unsigned int)count); +} +#endif + +static void +z_erofs_decode_taskqueue_init(struct erofs_sb_info *sbi __unused) +{ +#if WITH_EROFS_PCPU_KTHREAD + unsigned int worker_count; + + worker_count = z_erofs_decode_worker_count(); + sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK, + taskqueue_thread_enqueue, &sbi->decompress_taskqueue); + if (sbi->decompress_taskqueue == NULL) + return; + if (taskqueue_start_threads_cpuset(&sbi->decompress_taskqueue, + (int)worker_count, PWAIT, &all_cpus, "erofs decode") == 0) { + sbi->decompress_worker_count = worker_count; + sbi->decompress_worker_cpu_bound = true; + return; + } + taskqueue_free(sbi->decompress_taskqueue); + sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK, + taskqueue_thread_enqueue, &sbi->decompress_taskqueue); + if (sbi->decompress_taskqueue == NULL) + return; + if (taskqueue_start_threads(&sbi->decompress_taskqueue, + (int)worker_count, PWAIT, "erofs decode") != 0) { + taskqueue_free(sbi->decompress_taskqueue); + sbi->decompress_taskqueue = NULL; + return; + } + sbi->decompress_worker_count = worker_count; +#endif +} + +static struct taskqueue * +z_erofs_decode_taskqueue_acquire(struct erofs_sb_info *sbi) +{ + struct taskqueue *taskqueue; + + mtx_lock(&sbi->z_extent_cache_lock); + if (sbi->z_extent_cache.closing || + sbi->decompress_taskqueue == NULL) { + taskqueue = NULL; + } else { + taskqueue = sbi->decompress_taskqueue; + ++sbi->decompress_taskqueue_users; + } + mtx_unlock(&sbi->z_extent_cache_lock); + return (taskqueue); +} + +static void +z_erofs_decode_taskqueue_release(struct erofs_sb_info *sbi) +{ + mtx_lock(&sbi->z_extent_cache_lock); + KASSERT(sbi->decompress_taskqueue_users != 0, + ("erofs decode taskqueue user underflow")); + if (--sbi->decompress_taskqueue_users == 0) + cv_broadcast(&sbi->decompress_taskqueue_cv); + mtx_unlock(&sbi->z_extent_cache_lock); +} + +void +z_erofs_extent_cache_init(struct erofs_sb_info *sbi) +{ + + bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache)); + sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget, + EROFS_ZCACHE_MOUNT_HARD_BUDGET); + sbi->z_extent_cache.minimum_decode_work = + z_erofs_cache_minimum_decode_work; + mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF); + cv_init(&sbi->decompress_taskqueue_cv, "erofs decode tq"); + for (unsigned int i = 0; i < nitems(sbi->z_extent_cache.entries); ++i) + cv_init(&sbi->z_extent_cache.entries[i].cv, "erofs zextent"); + z_erofs_decode_taskqueue_init(sbi); + sbi->z_extent_cache_initialized = true; + mtx_lock(&z_erofs_cache_list_lock); + LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); +} + +void +z_erofs_extent_cache_fini(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + struct taskqueue *taskqueue; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&z_erofs_cache_list_lock); + LIST_REMOVE(sbi, z_extent_cache_link); + mtx_unlock(&z_erofs_cache_list_lock); + mtx_lock(&sbi->z_extent_cache_lock); + cache->closing = true; + sbi->z_extent_cache_initialized = false; + for (;;) { + bool busy = false; + unsigned int wait_index = 0; + for (i = 0; i < nitems(cache->entries); ++i) + if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT || + cache->entries[i].waiters != 0) { + busy = true; + wait_index = i; + break; + } + if (!busy) + break; + cv_wait(&cache->entries[wait_index].cv, + &sbi->z_extent_cache_lock); + } + /* + * The workers were created with &sbi->decompress_taskqueue as their + * context and read it when they first run, which can be after this + * point if the mount failed early. Keep it valid until + * taskqueue_free() has waited for every worker to exit; closing + * already keeps new dispatches away from the queue. + */ + taskqueue = sbi->decompress_taskqueue; + sbi->decompress_worker_count = 0; + sbi->decompress_worker_cpu_bound = false; + while (sbi->decompress_taskqueue_users != 0) + cv_wait(&sbi->decompress_taskqueue_cv, + &sbi->z_extent_cache_lock); + mtx_unlock(&sbi->z_extent_cache_lock); + if (taskqueue != NULL) { + taskqueue_drain_all(taskqueue); + taskqueue_free(taskqueue); + } + mtx_lock(&sbi->z_extent_cache_lock); + sbi->decompress_taskqueue = NULL; + count = 0; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], false, false); + else if (cache->entries[i].active) + z_erofs_extent_cache_reset_locked(cache, + &cache->entries[i]); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + for (i = 0; i < nitems(cache->entries); ++i) + cv_destroy(&cache->entries[i].cv); + cv_destroy(&sbi->decompress_taskqueue_cv); + mtx_destroy(&sbi->z_extent_cache_lock); +} + +int +z_erofs_extent_cache_drop(struct erofs_sb_info *sbi) +{ + struct erofs_zextent_cache *cache; + void *data[EROFS_ZCACHE_MAX_ENTRIES]; + unsigned int count, i; + + if (!sbi->z_extent_cache_initialized) + return (ENXIO); + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + if (cache->closing) { + mtx_unlock(&sbi->z_extent_cache_lock); + return (ENXIO); + } + for (;;) { + bool busy = false; + unsigned int wait_index = 0; + for (i = 0; i < nitems(cache->entries); ++i) + if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT || + cache->entries[i].waiters != 0) { + busy = true; + wait_index = i; + break; + } + if (!busy) + break; + cv_wait(&cache->entries[wait_index].cv, + &sbi->z_extent_cache_lock); + } + count = 0; + for (i = 0; i < nitems(cache->entries); ++i) { + if (cache->entries[i].state == EROFS_ZCACHE_READY) + data[count++] = + z_erofs_extent_cache_drop_entry_locked(sbi, + cache, &cache->entries[i], true, false); + else if (cache->entries[i].active) + z_erofs_extent_cache_reset_locked(cache, + &cache->entries[i]); + } + mtx_unlock(&sbi->z_extent_cache_lock); + while (count != 0) + free(data[--count], M_EROFS); + return (0); +} + +void +z_erofs_extent_cache_stats(struct erofs_sb_info *sbi, + struct erofs_cache_stats *stats) +{ + struct erofs_zextent_cache *cache; + unsigned int algorithm; + + bzero(stats, sizeof(*stats)); + if (!sbi->z_extent_cache_initialized) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + stats->hits += cache->metrics[algorithm].hits; + stats->misses += cache->metrics[algorithm].misses; + stats->bypasses += cache->metrics[algorithm].bypasses; + stats->evictions += cache->metrics[algorithm].evictions; + stats->reclaims += cache->metrics[algorithm].reclaims; + stats->waiter_merges += cache->metrics[algorithm].waiter_merges; + stats->bytes += cache->metrics[algorithm].resident_bytes; + } + stats->entries = cache->entry_count; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static bool +z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi, + const struct erofs_inode *vi, const struct erofs_map_blocks *map, + size_t len) +{ + const struct erofs_zextent_cache *cache; + uint64_t decode_work; + + cache = &sbi->z_extent_cache; + decode_work = map->m_plen + len; /* at most 1 MiB + 12 MiB */ + return (z_erofs_cache_enabled != 0 && + sbi->opt.cache_strategy != EROFS_ZIP_CACHE_DISABLED && + len <= MAXPHYS && + len <= cache->budget_bytes && + decode_work >= cache->minimum_decode_work && + (map->m_flags & (EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT)) == 0 && + map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX && + sbi->z_extent_cache_initialized && vi != sbi->packed_inode && + vi != sbi->metabox_en); +} + +static void +z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi, + const struct erofs_map_blocks *map) +{ + struct erofs_zextent_cache *cache; + + if (!sbi->z_extent_cache_initialized || + map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX) + return; + cache = &sbi->z_extent_cache; + mtx_lock(&sbi->z_extent_cache_lock); + ++cache->metrics[map->m_algorithmformat].bypasses; + mtx_unlock(&sbi->z_extent_cache_lock); +} + +static void +z_erofs_decode_length(const struct erofs_map_blocks *map, + erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial) +{ + size_t end; + + /* z_erofs_do_read() has checked that [mapoff, mapoff + want) fits. */ + EROFS_DEBUG_ASSERT(mapoff <= map->m_llen && + want <= map->m_llen - mapoff, ("erofs: bad decode range")); + end = (size_t)mapoff + want; + *partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0; + *decoded_len = *partial ? end : (size_t)map->m_llen; +} + +int +z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + const void *input; + void *compressed, *decoded; + int error; + + *bufp = NULL; + /* + * z_erofs_do_read() handles fragment and unmapped extents itself and + * gets decoded_len from z_erofs_decode_length(). + */ + EROFS_DEBUG_ASSERT((map->m_flags & + (EROFS_MAP_FRAGMENT | EROFS_MAP_MAPPED)) == EROFS_MAP_MAPPED, + ("erofs: decoding a map with flags %#x", + (unsigned int)map->m_flags)); + EROFS_DEBUG_ASSERT(decoded_len != 0 && decoded_len <= map->m_llen && + (partial || decoded_len == map->m_llen), + ("erofs: bad decoded length %zu", decoded_len)); + + if ((map->m_flags & EROFS_MAP_META) != 0) { + error = erofs_read_metadata_range(sbi, vi->nid, map->m_pa, + (size_t)map->m_plen, &buf); + } else { + error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa, + (size_t)map->m_plen, &compressed); + } + if (error != 0) + return (error); + input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed; + + decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO); + error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen, + decoded, decoded_len, partial); + if ((map->m_flags & EROFS_MAP_META) != 0) + erofs_put_metabuf(&buf); + else + erofs_brelse(compressed); + if (error != 0) { + free(decoded, M_EROFS); + return (error); + } + *bufp = decoded; + return (0); +} + +static void +z_erofs_decode_task_run(void *context, int pending __unused) +{ + struct erofs_decompress_task *task = context; + + task->error = z_erofs_decode_extent(task->sbi, task->vi, &task->map, + task->decoded_len, task->partial, &task->decoded); + mtx_lock(&task->lock); + atomic_add_64(&task->sbi->decompress_worker_runs, 1); + task->done = true; + cv_signal(&task->cv); + mtx_unlock(&task->lock); +} + +static uint8_t +z_erofs_decode_queue_priority(struct erofs_sb_info *sbi) +{ + uint8_t priority; + + mtx_lock(&sbi->control_lock); + priority = sbi->decompress_queue_priority; + mtx_unlock(&sbi->control_lock); + return (priority); +} + +int +z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi, + struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len, + bool partial, void **bufp) +{ + struct erofs_decompress_task *task; + struct taskqueue *taskqueue; + int error; + uint8_t queue_priority; + bool queued; + + if (sbi->sync_decompress != EROFS_SYNC_DECOMPRESS_FORCE_OFF) + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + taskqueue = z_erofs_decode_taskqueue_acquire(sbi); + if (taskqueue == NULL) { + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + } + /* + * A decode worker can get here again: reading the inline input of a + * metabox inode reads the (compressed) metabox. Waiting for another + * task from inside a worker could use up every worker, so decode in + * place. + */ + if (taskqueue_member(taskqueue, curthread)) { + z_erofs_decode_taskqueue_release(sbi); + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + return (z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp)); + } + task = malloc(sizeof(*task), M_EROFS, M_WAITOK | M_ZERO); + task->sbi = sbi; + task->vi = vi; + task->map = *map; + task->decoded_len = decoded_len; + task->partial = partial; + mtx_init(&task->lock, "erofs decomp task", NULL, MTX_DEF); + cv_init(&task->cv, "erofs decomp task"); + queue_priority = z_erofs_decode_queue_priority(sbi); + TASK_INIT(&task->task, queue_priority, z_erofs_decode_task_run, task); + error = taskqueue_enqueue(taskqueue, &task->task); + queued = error == 0; + if (queued) { + atomic_add_64(&sbi->decompress_submissions, 1); + mtx_lock(&task->lock); + while (!task->done) + cv_wait(&task->cv, &task->lock); + mtx_unlock(&task->lock); + taskqueue_drain(taskqueue, &task->task); + error = task->error; + *bufp = task->decoded; + } else { + atomic_add_64(&sbi->decompress_queue_fallbacks, 1); + error = z_erofs_decode_extent(sbi, vi, map, decoded_len, + partial, bufp); + } + cv_destroy(&task->cv); + mtx_destroy(&task->lock); + free(task, M_EROFS); + z_erofs_decode_taskqueue_release(sbi); + return (error); +} + +static int +z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, char *out) +{ + struct erofs_map_blocks map; + void *decoded, *fragment; + erofs_off_t mapoff; + size_t decoded_len, done, want; + bool cache_eligible, partial; + enum z_erofs_cache_lookup cache_lookup; + int error; + + done = 0; + while (done < len) { + map = (struct erofs_map_blocks) { .m_la = loff + done }; + error = erofs_map_blocks(sbi, vi, &map); + if (error != 0) + return (error); + if (map.m_llen == 0 || map.m_la > loff + done || + loff + done - map.m_la >= map.m_llen) + return (EINTEGRITY); + mapoff = loff + done - map.m_la; + want = MIN((size_t)(map.m_llen - mapoff), len - done); + + if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) { + if (sbi->packed_inode == NULL || + sbi->packed_inode->nid == vi->nid || + vi->z_fragmentoff > UINT64_MAX - mapoff) + return (EINTEGRITY); + error = erofs_read_data(sbi, sbi->packed_inode, + vi->z_fragmentoff + mapoff, want, &fragment); + if (error != 0) + return (error); + memcpy(out + done, fragment, want); + erofs_brelse(fragment); + } else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) { + bzero(out + done, want); + } else { + z_erofs_decode_length(&map, mapoff, want, &decoded_len, + &partial); + cache_eligible = !partial && + z_erofs_extent_cache_eligible(sbi, vi, &map, + decoded_len); + cache_lookup = Z_EROFS_CACHE_BYPASS; + if (cache_eligible) { + cache_lookup = z_erofs_extent_cache_claim(sbi, + vi, &map, decoded_len, mapoff, want, + out + done, &error); + if (cache_lookup == Z_EROFS_CACHE_HIT) { + done += want; + continue; + } + if (cache_lookup == Z_EROFS_CACHE_ERROR) + return (error); + } else + z_erofs_extent_cache_record_bypass(sbi, &map); + error = z_erofs_decode_extent_dispatch(sbi, vi, &map, + decoded_len, partial, &decoded); + if (error != 0) { + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, + &map, decoded_len, NULL, error); + return (error); + } + memcpy(out + done, (char *)decoded + (size_t)mapoff, + want); + if (cache_lookup == Z_EROFS_CACHE_OWNER) + z_erofs_extent_cache_complete(sbi, vi, &map, + decoded_len, decoded, 0); + else + free(decoded, M_EROFS); + } + done += want; + } + return (0); +} + +int +z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t loff, size_t len, void **bufp) +{ + char *out; + int error; + + /* erofs_read_data_impl() has checked the arguments and the range. */ + EROFS_DEBUG_ASSERT(len != 0 && loff <= vi->size && + len <= vi->size - loff, ("erofs: bad compressed read")); + *bufp = NULL; + out = malloc(len, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, loff, len, out); + if (error != 0) { + free(out, M_EROFS); + return (error); + } + *bufp = out; + return (0); +} + +int +z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct uio *uio) +{ + char *buf; + size_t want; + int error; + + /* erofs_read_uio() has rejected a negative offset. */ + while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) { + want = MIN((size_t)uio->uio_resid, + (size_t)MIN((uint64_t)MAXPHYS, + vi->size - (uint64_t)uio->uio_offset)); + buf = malloc(want, M_EROFS, M_WAITOK); + error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset, + want, buf); + if (error == 0) + error = uiomove(buf, want, uio); + free(buf, M_EROFS); + if (error != 0) + return (error); + } + return (0); +} diff --git a/zmap.c b/zmap.c new file mode 100644 index 0000000..3315261 --- /dev/null +++ b/zmap.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ + +#include +#include +#include +#include + +#include "internal.h" + +struct z_erofs_maprecorder { + struct erofs_sb_info *sbi; + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn; + uint8_t type, headtype; + unsigned int clusterofs; + uint16_t delta[2]; + erofs_blk_t pblk, compressedblks; + erofs_off_t nextpackoff; + bool partialref; +}; + +_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META | + EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF | + EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag"); + +/* + * The arithmetic in this file is plain on purpose. Every value is bounded + * before it gets here, and a position that did wrap would still be range + * checked where it is read (erofs_bread_device(), erofs_read_data_impl()): + * + * - blkszbits is in [9, PAGE_SHIFT] (super.c) and z_lclusterbits is + * blkszbits + 0..15, rejected from 31 up (z_erofs_fill_inode()), so a + * cluster is at most 1 GiB and a shift by z_lclusterbits is safe. + * - vi->size is at most OFF_MAX (erofs_read_inode()) and every request has + * m_la < vi->size (z_erofs_map_blocks()), so lcn << z_lclusterbits stays + * below 2^63 + 2^46 even after adding a 16-bit delta. + * - vi->inode_off lies inside the image (erofs_nid_is_valid()), inode_isize + * is 32 or 64 and xattr_isize is below 2^19. + * - An extent table has at most 2^48 records of at most 32 bytes each. + * - erofs_blk_t is 64 bits wide (internal.h). + * + * The list is repeated in docs/validation-policy.md. + */ + +/* The map header follows the inode and its xattrs, 8-byte aligned. */ +static inline erofs_off_t +z_erofs_header_pos(const struct erofs_inode *vi) +{ + return (roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize, 8)); +} + +static int +z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len, + struct erofs_buf *buf) +{ + return (erofs_read_metadata_range(m->sbi, m->vi->nid, pos, len, buf)); +} + +static int +z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + struct z_erofs_lcluster_index *di; + erofs_off_t base, pos; + unsigned int advise; + int error; + + vi = m->vi; + base = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header) + 8; + pos = base + lcn * sizeof(*di); + m->nextpackoff = pos + sizeof(*di); + error = z_erofs_read_index(m, pos, sizeof(*di), &buf); + if (error != 0) + return (error); + + di = buf.data; + m->lcn = lcn; + advise = le16toh(di->di_advise); + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + m->delta[0] = le16toh(di->di_u.delta[0]); + if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju has a compressed block count but " + "the inode has no big pclusters", + (uintmax_t)lcn)); + } + m->compressedblks = + m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16toh(di->di_u.delta[1]); + } else { + m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0; + m->clusterofs = le16toh(di->di_clusterofs); + m->pblk = le32toh(di->di_u.blkaddr); + } + erofs_put_metabuf(&buf); + return (0); +} + +static unsigned int +decode_compactedbits(unsigned int lobits, const uint8_t *in, + unsigned int pos, uint8_t *type) +{ + uint32_t v; + unsigned int lo; + + v = le32dec(in + pos / 8) >> (pos & 7); + lo = v & ((1U << lobits) - 1); + *type = (v >> lobits) & 3; + return (lo); +} + +static unsigned int +get_compacted_la_distance(unsigned int lobits, unsigned int encodebits, + unsigned int vcnt, const uint8_t *in, int i) +{ + unsigned int lo, d1; + uint8_t type; + + d1 = 0; + do { + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + return (d1); + ++d1; + } while (++i < (int)vcnt); + + /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ + if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0) + d1 += lo - 1; + return (d1); +} + +static int +z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct erofs_inode *vi; + erofs_off_t ebase, pos; + uint64_t compacted_2b, totalidx, original_lcn; + unsigned int compacted_4b_initial, amortizedshift; + unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize; + bool big_pcluster; + uint8_t *in, type; + int error, i; + + vi = m->vi; + /* + * A compact index entry is 16 bits (4-byte units) or 14 bits (2-byte + * units): a 2-bit type and the offset in the lcluster. Linux has the + * same limit. + */ + if (vi->z_lclusterbits > 14) + return (erofs_corrupt(m->sbi, vi->nid, + "invalid lcluster bits %u", vi->z_lclusterbits)); + ebase = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header); + totalidx = (vi->size + (1ULL << vi->z_lclusterbits) - 1) >> + vi->z_lclusterbits; + if (lcn >= totalidx) + return (EINVAL); + + original_lcn = lcn; + m->lcn = lcn; + compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; + compacted_2b = 0; + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 && + compacted_4b_initial < totalidx) + compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16); + + pos = ebase; + amortizedshift = 2; + if (lcn >= compacted_4b_initial) { + pos += compacted_4b_initial * 4; + lcn -= compacted_4b_initial; + if (lcn < compacted_2b) { + amortizedshift = 1; + } else { + pos += compacted_2b * 2; + lcn -= compacted_2b; + } + } + pos += lcn << amortizedshift; + + if (amortizedshift == 2 && vi->z_lclusterbits <= 14) + vcnt = 2; + else if (amortizedshift == 1 && vi->z_lclusterbits <= 12) + vcnt = 16; + else { + erofs_err(m->sbi, "unsupported lcluster bits %u @ nid %ju", + vi->z_lclusterbits, (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + + packsize = vcnt << amortizedshift; + bytes = pos & (packsize - 1); + pos -= bytes; + m->nextpackoff = pos + packsize; + error = z_erofs_read_index(m, pos, packsize, &buf); + if (error != 0) + return (error); + in = buf.data; + lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT)); + encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt; + i = bytes >> amortizedshift; + + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1U << vi->z_lclusterbits; + if (lookahead) { + m->delta[1] = get_compacted_la_distance(lobits, + encodebits, vcnt, in, i); + } + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + if (!big_pcluster) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju has a compressed block count but " + "the inode has no big pclusters", + (uintmax_t)original_lcn)); + } + m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + } else { + EROFS_DEBUG_ASSERT(i > 0, + ("erofs: pack of %u slots", vcnt)); + lo = decode_compactedbits(lobits, in, + encodebits * (i - 1), &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + lo = 0; + else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) + lo = 1; + m->delta[0] = lo + 1; + } + erofs_put_metabuf(&buf); + return (0); + } + + m->clusterofs = lo; + m->delta[0] = 0; + big_pcluster = + (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) + i -= lo; + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) { + --i; + nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; + continue; + } + if (lo <= 1) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(m->sbi, vi->nid, + "invalid delta %u before lcn %ju", + lo, (uintmax_t)original_lcn)); + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + m->pblk = (erofs_blk_t)le32dec(in + packsize - sizeof(uint32_t)) + nblk; + erofs_put_metabuf(&buf); + m->lcn = original_lcn; + return (0); +} + +static int +z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn, + bool lookahead) +{ + int err; + + EROFS_DEBUG_ASSERT(m->vi->datalayout == + EROFS_INODE_COMPRESSED_COMPACT || + m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL, + ("erofs: data layout %u is not compressed", m->vi->datalayout)); + if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) + err = z_erofs_load_compact_lcluster(m, lcn, lookahead); + else + err = z_erofs_load_full_lcluster(m, lcn); + if (err != 0) + return (err); + /* The type is a 2-bit field, so it is always below TYPE_MAX. */ + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->clusterofs >= (1U << m->vi->z_lclusterbits)) + return (erofs_corrupt(m->sbi, m->vi->nid, + "clusterofs %u of lcn %ju is out of range", m->clusterofs, + (uintmax_t)lcn)); + return (0); +} + +static int +z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + uint64_t lcn; + int err; + + while (lookback_distance != 0 && m->lcn >= lookback_distance) { + lcn = m->lcn - lookback_distance; + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + lookback_distance = m->delta[0]; + continue; + } + m->headtype = m->type; + m->map->m_la = (lcn << m->vi->z_lclusterbits) + m->clusterofs; + return (0); + } + return (erofs_corrupt(m->sbi, m->vi->nid, + "no head lcluster found looking back from lcn %ju", + (uintmax_t)m->lcn)); +} + +static int +z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + bool bigpcl1, bigpcl2; + uint64_t lcn, lcnpos; + int err; + + vi = m->vi; + bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0; + bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0; + lcn = m->lcn + 1; + lcnpos = lcn << vi->z_lclusterbits; + if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || + ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || + m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || + lcnpos >= vi->size) + m->compressedblks = 1; + if (m->compressedblks == 0) { + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->delta[0] != 1) + return (erofs_corrupt(m->sbi, vi->nid, + "lcn %ju: delta %u should be 1 for the compressed " + "block count", (uintmax_t)lcn, m->delta[0])); + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD || + m->compressedblks == 0) + m->compressedblks = 1; + } + m->map->m_plen = m->compressedblks * m->sbi->block_size; + return (0); +} + +static int +z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) +{ + struct erofs_inode *vi; + struct erofs_map_blocks *map; + uint64_t lcn, headlcn, lend, lcnpos; + int err; + + vi = m->vi; + map = m->map; + lcn = m->lcn; + headlcn = map->m_la >> vi->z_lclusterbits; + for (;;) { + lcnpos = lcn << vi->z_lclusterbits; + if (lcnpos >= vi->size) { + EROFS_DEBUG_ASSERT(map->m_la <= vi->size, + ("erofs: extent at %ju beyond the size %ju", + (uintmax_t)map->m_la, (uintmax_t)vi->size)); + map->m_llen = vi->size - map->m_la; + return (0); + } + err = z_erofs_load_lcluster_from_disk(m, lcn, true); + if (err != 0) + return (err); + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (m->delta[1] == 0) + m->delta[1] = 1; + } else { + if (lcn != headlcn) + break; + m->delta[1] = 1; + } + lcn += m->delta[1]; + } + lend = (lcn << vi->z_lclusterbits) + m->clusterofs; + EROFS_DEBUG_ASSERT(lend >= map->m_la, + ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)lend, + (uintmax_t)map->m_la)); + map->m_llen = lend - map->m_la; + return (0); +} + +static inline erofs_off_t +z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz) +{ + return (roundup2(z_erofs_header_pos(vi) + + sizeof(struct z_erofs_map_header), recsz)); +} + +static inline erofs_off_t +z_erofs_extent_record_pos(erofs_off_t table_pos, unsigned int recsz, + uint64_t index) +{ + return (table_pos + index * recsz); +} + +static int +z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi, + erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + int error; + + bzero(ext, sizeof(*ext)); + error = erofs_read_metadata_range(sbi, vi->nid, pos, recsz, &buf); + if (error != 0) + return (error); + memcpy(ext, buf.data, recsz); + erofs_put_metabuf(&buf); + return (0); +} + +static uint64_t +z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz) +{ + uint64_t lstart; + + lstart = le32toh(ext->lstart_lo); + if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) + lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32; + return (lstart); +} + +static int +z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map, int flags) +{ + bool fragment, ztailpacking; + struct z_erofs_maprecorder m; + uint64_t initial_lcn, ofs, end; + unsigned int endoff; + int err; + + fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0; + ztailpacking = vi->z_idata_size != 0; + bzero(&m, sizeof(m)); + m.sbi = sbi; + m.vi = vi; + m.map = map; + + if (vi->size == 0) { + map->m_la = 0; + map->m_llen = 0; + map->m_flags = 0; + return (0); + } + ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ? + vi->size - 1 : map->m_la; + if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 && + vi->z_tailextent_headlcn == 0) { + map->m_la = 0; + map->m_llen = vi->size; + map->m_flags = EROFS_MAP_FRAGMENT; + return (0); + } + initial_lcn = ofs >> vi->z_lclusterbits; + endoff = ofs & ((1U << vi->z_lclusterbits) - 1); + err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); + if (err != 0) + return (err); + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking) + vi->z_fragmentoff = m.nextpackoff; + + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; + end = (m.lcn + 1) << vi->z_lclusterbits; + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + endoff >= m.clusterofs) { + m.headtype = m.type; + map->m_la = (m.lcn << vi->z_lclusterbits) + m.clusterofs; + if (ztailpacking && end > vi->size) + end = vi->size; + } else { + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + end = (m.lcn << vi->z_lclusterbits) + m.clusterofs; + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + m.delta[0] = 1; + } + err = z_erofs_extent_lookback(&m, m.delta[0]); + if (err != 0) + return (err); + } + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + EROFS_DEBUG_ASSERT(end >= map->m_la, + ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)end, + (uintmax_t)map->m_la)); + map->m_llen = end - map->m_la; + + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) { + vi->z_tailextent_headlcn = m.lcn; + if (fragment && + vi->datalayout == EROFS_INODE_COMPRESSED_FULL) { + /* Both are 32-bit fields of the on-disk index. */ + vi->z_fragmentoff |= m.pblk << 32; + } + } + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_fragmentoff; + map->m_plen = vi->z_idata_size; + /* Same rule as Linux: inline data does not cross a block. */ + if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen > + sbi->block_size) + return (erofs_corrupt(sbi, vi->nid, + "inline pcluster of %ju bytes crosses its block", + (uintmax_t)map->m_plen)); + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags = EROFS_MAP_FRAGMENT; + } else { + map->m_pa = m.pblk * sbi->block_size; + err = z_erofs_get_extent_compressedlen(&m); + if (err != 0) + return (err); + } + + if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { + /* + * Interlaced pclusters are block-aligned on disk. An inline + * (ztailpacking) pcluster is not, so it can only be shifted, + * whatever the inode advises. + */ + map->m_algorithmformat = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 && + (map->m_flags & EROFS_MAP_META) == 0 ? + Z_EROFS_COMPRESSION_INTERLACED : + Z_EROFS_COMPRESSION_SHIFTED; + } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { + map->m_algorithmformat = vi->z_algorithmtype[1]; + } else { + map->m_algorithmformat = vi->z_algorithmtype[0]; + } + + if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0) { + err = z_erofs_get_extent_decompressedlen(&m); + if (err == 0) + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + return (err); + } + return (0); +} + +/* + * The extent table is not checked when the inode is loaded: that its + * records are sorted and inside the file is fsck.erofs's job, and scanning + * it would read a table that grows with the file on every load. Each + * record the search visits is range checked by the read, and a table that + * is out of order only makes it return a hole or the wrong extent. + */ +static int +z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_extent ext; + unsigned int recsz, bmask; + uint8_t fmt; + uint64_t cluster_size, extent_idx; + erofs_off_t extent_pos, pos, table_pos; + uint64_t lend, l, r, mid, pa, la, lstart; + bool interlaced, last; + int error; + + interlaced = + (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0; + recsz = z_erofs_extent_recsize(vi->z_advise); + table_pos = z_erofs_extent_table_pos(vi, recsz); + pos = table_pos; + bmask = sbi->block_size - 1; + lend = vi->size; + cluster_size = 1ULL << vi->z_lclusterbits; + map->m_flags = 0; + + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { + error = erofs_read_metadata_range(sbi, vi->nid, pos, + sizeof(uint64_t), &buf); + if (error != 0) + return (error); + pa = le64dec(buf.data); + erofs_put_metabuf(&buf); + pos += sizeof(uint64_t); + lstart = 0; + extent_idx = 0; + } else { + lstart = rounddown2(map->m_la, cluster_size); + extent_idx = lstart >> vi->z_lclusterbits; + pa = EROFS_NULL_ADDR; + } + for (;;) { + EROFS_DEBUG_ASSERT(extent_idx < vi->z_extents, + ("erofs: extent %ju of %ju", (uintmax_t)extent_idx, + (uintmax_t)vi->z_extents)); + extent_pos = z_erofs_extent_record_pos(pos, recsz, + extent_idx); + error = z_erofs_read_extent(sbi, vi, extent_pos, recsz, + &ext); + if (error != 0) + return (error); + map->m_plen = le32toh(ext.plen); + if (pa != EROFS_NULL_ADDR) { + map->m_pa = pa; + pa += map->m_plen & Z_EROFS_EXTENT_PLEN_MASK; + } else { + map->m_pa = le32toh(ext.pstart_lo); + } + extent_idx++; + lstart += cluster_size; + if (lstart > map->m_la) + break; + } + last = lstart >= roundup2(lend, cluster_size); + lend = MIN(lstart, lend); + lstart -= cluster_size; + } else { + lstart = lend; + for (l = 0, r = vi->z_extents; l < r;) { + mid = l + (r - l) / 2; + extent_pos = z_erofs_extent_record_pos(table_pos, recsz, + mid); + error = z_erofs_read_extent(sbi, vi, extent_pos, + recsz, &ext); + if (error != 0) + return (error); + la = z_erofs_extent_lstart(&ext, recsz); + pa = le32toh(ext.pstart_lo) | + ((uint64_t)le32toh(ext.pstart_hi) << 32); + if (la > map->m_la) { + r = mid; + /* + * The records are not compared with each + * other, but this one must not start past the + * next one or the size: m_llen stays within the + * file. + */ + if (la > lend) + return (erofs_corrupt(sbi, vi->nid, + "extent %ju starts at %ju, past " + "the following extent or the size " + "%ju", + (uintmax_t)mid, (uintmax_t)la, + (uintmax_t)lend)); + lend = la; + } else { + l = mid + 1; + if (map->m_la == la) + r = MIN(l + 1, r); + lstart = la; + map->m_plen = le32toh(ext.plen); + map->m_pa = pa; + } + } + last = l >= vi->z_extents; + } + + if (lstart < lend) { + map->m_la = lstart; + if (last && + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + map->m_flags = EROFS_MAP_FRAGMENT; + if (recsz > offsetof(struct z_erofs_extent, + pstart_lo)) { + vi->z_fragmentoff = map->m_plen | + (map->m_pa << 32); + } else { + vi->z_fragmentoff = map->m_plen; + } + } else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) { + map->m_flags = EROFS_MAP_MAPPED; + fmt = (uint8_t)(map->m_plen >> + Z_EROFS_EXTENT_PLEN_FMT_BIT); + if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; + if (fmt != 0) { + if (fmt - 1 >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sbi, + "unsupported compression " + "algorithm %u @ nid %ju", fmt - 1, + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + map->m_algorithmformat = (uint8_t)(fmt - 1); + } else if (interlaced && + ((map->m_pa | map->m_plen) & bmask) == 0) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } + } + map->m_llen = lend - map->m_la; + return (0); +} + +int +z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi) +{ + struct erofs_buf buf = EROFS_BUF_INITIALIZER; + struct z_erofs_map_header *h; + struct erofs_map_blocks map; + uint64_t cluster_size, raw; + erofs_off_t pos; + unsigned int recsz; + int err; + + if (vi->z_initialized) + return (0); + pos = z_erofs_header_pos(vi); + err = erofs_read_metadata_range(sbi, vi->nid, pos, sizeof(*h), &buf); + if (err != 0) + return (err); + h = buf.data; + if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) { + if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + } + raw = le64dec(h); + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = raw ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + erofs_put_metabuf(&buf); + vi->z_initialized = true; + return (0); + } + + vi->z_advise = le16toh(h->h_advise); + vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15); + /* The lcluster size is used as a 32-bit shift count and size below. */ + if (vi->z_lclusterbits >= 31) { + erofs_put_metabuf(&buf); + return (erofs_corrupt(sbi, vi->nid, + "lcluster bits %u are too large", vi->z_lclusterbits)); + } + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) { + recsz = z_erofs_extent_recsize(vi->z_advise); + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + cluster_size = 1ULL << vi->z_lclusterbits; + vi->z_extents = roundup2(vi->size, cluster_size) >> + vi->z_lclusterbits; + } else { + vi->z_extents = le32toh(h->h_extents_lo) | + ((uint64_t)le16toh(h->h_extents_hi) << 32); + } + erofs_put_metabuf(&buf); + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + vi->z_initialized = true; + return (0); + } + vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15); + vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4); + /* + * Both HEAD formats are on-disk algorithm IDs. IDs at or above + * Z_EROFS_COMPRESSION_MAX would alias the runtime-only SHIFTED and + * INTERLACED formats, so refuse the inode instead of decoding it as + * uncompressed data. + */ + if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || + vi->z_algorithmtype[1] >= Z_EROFS_COMPRESSION_MAX) { + erofs_put_metabuf(&buf); + erofs_err(sbi, "unsupported compression algorithm %u/%u " + "@ nid %ju", vi->z_algorithmtype[0], vi->z_algorithmtype[1], + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) + vi->z_fragmentoff = le32toh(h->h_fragmentoff); + else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0) + vi->z_idata_size = le16toh(h->h_idata_size); + erofs_put_metabuf(&buf); + + /* Same rule as Linux: big pcluster advise needs the feature flag. */ + if (!erofs_sb_has_big_pcluster(sbi) && + (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0) + return (erofs_corrupt(sbi, vi->nid, + "big pcluster advise %#x, but the image has no big " + "pclusters", + vi->z_advise)); + /* + * Same rule as Linux: compact indexes set both big-pcluster bits or + * neither. + */ + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && + (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) != + ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0))) + return (erofs_corrupt(sbi, vi->nid, + "compact inode with inconsistent big pcluster advise %#x", + vi->z_advise)); + if (vi->z_idata_size != 0 || + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) { + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 && + (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0)) + return (erofs_corrupt(sbi, vi->nid, + "fragment inode, but the image has no packed " + "inode")); + bzero(&map, sizeof(map)); + err = z_erofs_map_blocks_fo(sbi, vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (err != 0) + return (err); + } + vi->z_initialized = true; + return (0); +} + +static int +z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) { + /* + * The fragment is read from the packed inode, and that read + * checks the packed inode and the range (z_erofs_do_read(), + * erofs_read_data()). + */ + EROFS_DEBUG_ASSERT((map->m_flags & + (EROFS_MAP_MAPPED | EROFS_MAP_META)) == 0, + ("erofs: fragment map with flags %#x", + (unsigned int)map->m_flags)); + return (0); + } + if ((map->m_flags & EROFS_MAP_MAPPED) == 0) + return (0); + EROFS_DEBUG_ASSERT(map->m_la < vi->size, + ("erofs: extent at %ju, beyond the size %ju", + (uintmax_t)map->m_la, (uintmax_t)vi->size)); + /* + * On-disk algorithm IDs are validated where they are read, so only + * the runtime formats can reach this point. + */ + EROFS_DEBUG_ASSERT(map->m_algorithmformat < + Z_EROFS_COMPRESSION_RUNTIME_MAX, + ("erofs: unvalidated algorithm format %u", map->m_algorithmformat)); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if ((sbi->available_compr_algs & + (1U << map->m_algorithmformat)) == 0) + return (erofs_corrupt(sbi, vi->nid, + "algorithm %u is not in the image's algorithm list", + map->m_algorithmformat)); + /* + * A fully mapped compressed extent does not decode to fewer + * bytes than it occupies, unless the end of the file cuts it + * short. + */ + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && + map->m_llen < vi->size - map->m_la) + return (erofs_corrupt(sbi, vi->nid, + "extent at %ju decompresses to %ju bytes from %ju " + "compressed", (uintmax_t)map->m_la, + (uintmax_t)map->m_llen, (uintmax_t)map->m_plen)); + } else if (map->m_llen > map->m_plen) { + /* Stored data is not longer than the pcluster holding it. */ + return (erofs_corrupt(sbi, vi->nid, + "uncompressed extent at %ju has length %ju > stored %ju", + (uintmax_t)map->m_la, (uintmax_t)map->m_llen, + (uintmax_t)map->m_plen)); + } + if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || + map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE) { + erofs_err(sbi, "extent at %ju is too large (%ju bytes encoded, " + "%ju decoded) @ nid %ju", (uintmax_t)map->m_la, + (uintmax_t)map->m_plen, (uintmax_t)map->m_llen, + (uintmax_t)vi->nid); + return (EOPNOTSUPP); + } + return (0); +} + +int +z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi, + struct erofs_map_blocks *map) +{ + int err; + + if (map->m_la >= vi->size) { + map->m_llen = map->m_la - vi->size + 1; + map->m_la = vi->size; + map->m_flags = 0; + return (0); + } + err = z_erofs_fill_inode(sbi, vi); + if (err == 0) { + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) + err = z_erofs_map_blocks_ext(sbi, vi, map); + else + err = z_erofs_map_blocks_fo(sbi, vi, map, + EROFS_GET_BLOCKS_FIEMAP); + } + if (err == 0) + err = z_erofs_map_sanity_check(sbi, vi, map); + if (err != 0) + map->m_llen = 0; + return (err); +} diff --git a/zutil.c b/zutil.c new file mode 100644 index 0000000..f3e7510 --- /dev/null +++ b/zutil.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* FreeBSD stream context pool and compression runtime accounting. */ + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024) +#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024) +#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024) +#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U) +#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U) +#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U) +#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U) + +static unsigned long erofs_stream_mount_budget = + EROFS_STREAM_MOUNT_HARD_BUDGET; +static unsigned long erofs_stream_global_budget = + EROFS_STREAM_GLOBAL_HARD_BUDGET; +static int erofs_stream_mount_contexts = + EROFS_STREAM_MOUNT_HARD_CONTEXTS; +static int erofs_stream_global_contexts_limit = + EROFS_STREAM_GLOBAL_HARD_CONTEXTS; +static int erofs_stream_mount_cached = + EROFS_STREAM_MOUNT_CACHED_PER_CODEC; +static int erofs_stream_global_cached = + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC; + +TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget", + &erofs_stream_mount_budget); +TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget", + &erofs_stream_global_budget); +TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts", + &erofs_stream_mount_contexts); +TUNABLE_INT("vfs.erofs.stream_pool.global_contexts", + &erofs_stream_global_contexts_limit); +TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec", + &erofs_stream_mount_cached); +TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec", + &erofs_stream_global_cached); + +static struct mtx erofs_stream_lock; +static struct cv erofs_stream_global_cv; +static unsigned int erofs_stream_global_waiters; +static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts = + LIST_HEAD_INITIALIZER(erofs_stream_mounts); +static uma_zone_t erofs_stream_zone; +static eventhandler_tag erofs_stream_lowmem_tag; +static size_t erofs_stream_global_resident; +static unsigned int erofs_stream_global_contexts; +static unsigned int erofs_stream_global_cached_by_codec[ + Z_EROFS_COMPRESSION_MAX]; +static unsigned long erofs_stream_cached_contexts; +static unsigned long erofs_stream_borrowed_contexts; +static unsigned long erofs_stream_idle_contexts; +static unsigned long erofs_stream_creations; +static unsigned long erofs_stream_reuses; +static unsigned long erofs_stream_fallbacks; +static unsigned long erofs_stream_exhaustions; +static unsigned long erofs_stream_waits; +static unsigned long erofs_stream_destroys; +static unsigned long erofs_stream_reclaims; + +MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool", + MTX_DEF); + +SYSCTL_DECL(_vfs_erofs); +SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool, + CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD, + &erofs_stream_mount_budget, 0, "Effective per-mount byte budget"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD, + &erofs_stream_global_budget, 0, "Effective global byte budget"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD, + &erofs_stream_mount_contexts, 0, + "Effective per-mount, per-algorithm context limit"); +SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit, + CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0, + "Effective global context limit"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD, + &erofs_stream_global_resident, 0, "Currently charged global bytes"); +SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD, + &erofs_stream_global_contexts, 0, "Currently allocated contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD, + &erofs_stream_cached_contexts, 0, "Currently cacheable contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD, + &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD, + &erofs_stream_idle_contexts, 0, "Currently idle contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD, + &erofs_stream_creations, 0, "Successful context creations"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD, + &erofs_stream_reuses, 0, "Idle context acquisitions"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD, + &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD, + &erofs_stream_exhaustions, 0, "Unsatisfiable limit or budget rejections"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, waits, CTLFLAG_RD, + &erofs_stream_waits, 0, "Sleeps waiting for a context or budget release"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD, + &erofs_stream_destroys, 0, "Destroyed contexts"); +SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD, + &erofs_stream_reclaims, 0, "Low-memory idle context reclaims"); + +struct erofs_stream_alloc_header { + struct erofs_stream_ctx *ctx; + size_t bytes; +}; + +static bool +erofs_stream_mount_budget_available_locked(const struct erofs_sb_info *sbi, + size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + return (bytes <= erofs_stream_mount_budget && + sbi->stream_pool.resident_bytes <= erofs_stream_mount_budget - + bytes); +} + +static bool +erofs_stream_global_budget_available_locked(size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + return (bytes <= erofs_stream_global_budget && + erofs_stream_global_resident <= erofs_stream_global_budget - bytes); +} + +static bool +erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + if (!erofs_stream_mount_budget_available_locked(sbi, bytes) || + !erofs_stream_global_budget_available_locked(bytes)) + return (false); + pool->resident_bytes += bytes; + erofs_stream_global_resident += bytes; + return (true); +} + +/* + * Wake threads sleeping in z_erofs_stream_ctx_get() after a context or byte + * budget was released. Mount-level sleepers and the unmount drain share the + * per-mount cv; sleepers blocked on a global limit use the global cv. + */ +static void +erofs_stream_wake_locked(struct erofs_sb_info *sbi) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + if (sbi->stream_pool.waiters != 0 || sbi->stream_pool.closing) + cv_broadcast(&sbi->stream_pool.cv); + if (erofs_stream_global_waiters != 0) + cv_broadcast(&erofs_stream_global_cv); +} + +static void +erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes) +{ + + mtx_assert(&erofs_stream_lock, MA_OWNED); + KASSERT(bytes <= sbi->stream_pool.resident_bytes && + bytes <= erofs_stream_global_resident, + ("erofs stream pool byte accounting underflow")); + sbi->stream_pool.resident_bytes -= bytes; + erofs_stream_global_resident -= bytes; +} + +static struct erofs_stream_ctx * +erofs_stream_take_idle_locked(struct erofs_sb_info *sbi) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx == NULL) + continue; + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && + erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + return (ctx); + } + return (NULL); +} + +/* + * Detach one idle context so that the caller can destroy it after dropping + * the lock. Idle contexts of this mount are preferred; other mounts are only + * considered when the limit being hit is a global one. + */ +static struct erofs_stream_ctx * +erofs_stream_take_reclaimable_locked(struct erofs_sb_info *sbi, bool global) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *other; + + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL || !global) + return (ctx); + LIST_FOREACH(other, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(other); + if (ctx != NULL) + break; + } + return (ctx); +} + +static void erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx); + +/* + * Charging never sleeps for budget: the caller already owns a context, and + * waiting here would hold part of the budget while waiting for the rest. Idle + * contexts are the only budget that can be returned immediately, so reclaim + * them first and fail with ENOMEM only when the budget is truly exhausted. + */ +int +z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_ctx *idle; + bool reserved; + + if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) { + ctx->allocation_failed = true; + mtx_lock(&erofs_stream_lock); + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mtx_lock(&erofs_stream_lock); + for (;;) { + if (ctx->charged_bytes > SIZE_MAX - bytes || + bytes > erofs_stream_mount_budget || + bytes > erofs_stream_global_budget) { + reserved = false; + break; + } + reserved = erofs_stream_reserve_locked(ctx->sbi, bytes); + if (reserved) + break; + idle = erofs_stream_take_reclaimable_locked(ctx->sbi, + erofs_stream_mount_budget_available_locked(ctx->sbi, + bytes)); + if (idle == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(idle); + mtx_lock(&erofs_stream_lock); + } + if (reserved) + ctx->charged_bytes += bytes; + else + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + if (!reserved) { + ctx->allocation_failed = true; + return (ENOMEM); + } + return (0); +} + +void +z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes) +{ + + mtx_lock(&erofs_stream_lock); + KASSERT(bytes <= ctx->charged_bytes, + ("erofs stream context byte accounting underflow")); + ctx->charged_bytes -= bytes; + erofs_stream_release_locked(ctx->sbi, bytes); + erofs_stream_wake_locked(ctx->sbi); + mtx_unlock(&erofs_stream_lock); +} + +void * +z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes) +{ + struct erofs_stream_alloc_header *header; + size_t total; + + if (__builtin_add_overflow(bytes, sizeof(*header), &total) || + z_erofs_stream_ctx_charge(ctx, total) != 0) + return (NULL); + header = malloc(total, M_EROFS, M_WAITOK | M_ZERO); + header->ctx = ctx; + header->bytes = total; + return (header + 1); +} + +void +z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address) +{ + struct erofs_stream_alloc_header *header; + size_t bytes; + + if (address == NULL) + return; + header = (struct erofs_stream_alloc_header *)address - 1; + KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch")); + bytes = header->bytes; + free(header, M_EROFS); + z_erofs_stream_ctx_uncharge(ctx, bytes); +} + +static void +erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_sb_info *sbi; + bool cached; + uint8_t algorithm; + + sbi = ctx->sbi; + algorithm = ctx->algorithm; + cached = ctx->cached; + ctx->fini(ctx); + KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE, + ("erofs stream backend allocation leaked")); + uma_zfree(erofs_stream_zone, ctx); + + mtx_lock(&erofs_stream_lock); + codec = &sbi->stream_pool.codec[algorithm]; + KASSERT(codec->contexts != 0 && codec->borrowed != 0 && + erofs_stream_global_contexts != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream context accounting underflow")); + --codec->contexts; + --codec->borrowed; + --erofs_stream_global_contexts; + --erofs_stream_borrowed_contexts; + if (cached) { + KASSERT(codec->cached != 0 && + erofs_stream_global_cached_by_codec[algorithm] != 0 && + erofs_stream_cached_contexts != 0, + ("erofs stream cache accounting underflow")); + --codec->cached; + --erofs_stream_global_cached_by_codec[algorithm]; + --erofs_stream_cached_contexts; + } + erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE); + ++erofs_stream_destroys; + erofs_stream_wake_locked(sbi); + mtx_unlock(&erofs_stream_lock); +} + +/* + * Sleep until a context or budget is released. Limits that only this mount + * can release are waited for on the mount cv; anything involving a global + * limit is waited for on the global cv, which every release also signals. + */ +static void +erofs_stream_wait_locked(struct erofs_sb_info *sbi, bool global) +{ + struct erofs_stream_pool *pool; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + pool = &sbi->stream_pool; + ++pool->waiters; + ++erofs_stream_waits; + if (global) { + ++erofs_stream_global_waiters; + cv_wait(&erofs_stream_global_cv, &erofs_stream_lock); + --erofs_stream_global_waiters; + } else { + cv_wait(&pool->cv, &erofs_stream_lock); + } + KASSERT(pool->waiters != 0, ("erofs stream waiter underflow")); + --pool->waiters; + if (pool->closing) + cv_broadcast(&pool->cv); +} + +int +z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm, + size_t context_size, erofs_stream_ctx_init_t *init, + erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp) +{ + struct erofs_stream_pool_codec *codec; + struct erofs_stream_ctx *ctx; + bool cached, global_full, mount_full, reserved __diagused; + int error; + + *ctxp = NULL; + if (algorithm == Z_EROFS_COMPRESSION_LZ4 || + algorithm >= Z_EROFS_COMPRESSION_MAX || + context_size < sizeof(*ctx) || + context_size > EROFS_STREAM_CTX_WRAPPER_SIZE) + return (EINVAL); + mtx_lock(&erofs_stream_lock); + for (;;) { + if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) { + mtx_unlock(&erofs_stream_lock); + return (ENXIO); + } + codec = &sbi->stream_pool.codec[algorithm]; + ctx = STAILQ_FIRST(&codec->idle); + if (ctx != NULL) { + STAILQ_REMOVE_HEAD(&codec->idle, link); + KASSERT(codec->idle_count != 0 && + erofs_stream_idle_contexts != 0, + ("erofs stream idle accounting underflow")); + --codec->idle_count; + --erofs_stream_idle_contexts; + ++codec->borrowed; + ++erofs_stream_borrowed_contexts; + ++erofs_stream_reuses; + ctx->allocation_failed = false; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); + } + /* + * Limits that no release can ever satisfy are configuration + * errors; waiting for them would never end. + */ + if (erofs_stream_mount_contexts == 0 || + erofs_stream_global_contexts_limit == 0 || + EROFS_STREAM_CTX_WRAPPER_SIZE > erofs_stream_mount_budget || + EROFS_STREAM_CTX_WRAPPER_SIZE > + erofs_stream_global_budget) { + ++erofs_stream_exhaustions; + mtx_unlock(&erofs_stream_lock); + return (ENOMEM); + } + mount_full = codec->contexts >= + (unsigned int)erofs_stream_mount_contexts; + global_full = erofs_stream_global_contexts >= + (unsigned int)erofs_stream_global_contexts_limit || + !erofs_stream_global_budget_available_locked( + EROFS_STREAM_CTX_WRAPPER_SIZE); + if (!mount_full && !global_full && + erofs_stream_mount_budget_available_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE)) + break; + /* + * Contexts of this codec are all borrowed when mount_full, so + * only a release can help. Otherwise idle contexts of other + * codecs (or mounts, for a global limit) hold the resource and + * are never released unless reclaimed here. + */ + if (!mount_full) { + ctx = erofs_stream_take_reclaimable_locked(sbi, + global_full); + if (ctx != NULL) { + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + continue; + } + } + erofs_stream_wait_locked(sbi, !mount_full && global_full); + } + reserved = erofs_stream_reserve_locked(sbi, + EROFS_STREAM_CTX_WRAPPER_SIZE); + KASSERT(reserved, ("erofs stream wrapper reservation failed")); + cached = codec->cached < (unsigned int)erofs_stream_mount_cached && + erofs_stream_global_cached_by_codec[algorithm] < + (unsigned int)erofs_stream_global_cached; + ++codec->contexts; + ++codec->borrowed; + ++erofs_stream_global_contexts; + ++erofs_stream_borrowed_contexts; + if (cached) { + ++codec->cached; + ++erofs_stream_global_cached_by_codec[algorithm]; + ++erofs_stream_cached_contexts; + } else { + ++erofs_stream_fallbacks; + } + mtx_unlock(&erofs_stream_lock); + + ctx = uma_zalloc(erofs_stream_zone, M_WAITOK | M_ZERO); + ctx->sbi = sbi; + ctx->fini = fini; + ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE; + ctx->algorithm = algorithm; + ctx->cached = cached; + error = init(ctx); + if (error != 0) { + erofs_stream_ctx_destroy(ctx); + return (error > 0 ? error : EIO); + } + mtx_lock(&erofs_stream_lock); + ++erofs_stream_creations; + mtx_unlock(&erofs_stream_lock); + *ctxp = ctx; + return (0); +} + +void +z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable) +{ + struct erofs_stream_pool_codec *codec; + + mtx_lock(&erofs_stream_lock); + codec = &ctx->sbi->stream_pool.codec[ctx->algorithm]; + if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) { + KASSERT(codec->borrowed != 0 && + erofs_stream_borrowed_contexts != 0, + ("erofs stream borrowed accounting underflow")); + --codec->borrowed; + --erofs_stream_borrowed_contexts; + STAILQ_INSERT_HEAD(&codec->idle, ctx, link); + ++codec->idle_count; + ++erofs_stream_idle_contexts; + erofs_stream_wake_locked(ctx->sbi); + mtx_unlock(&erofs_stream_lock); + return; + } + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); +} + +static bool +erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool) +{ + unsigned int algorithm; + + mtx_assert(&erofs_stream_lock, MA_OWNED); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) { + if (pool->codec[algorithm].contexts != 0) + return (false); + } + return (true); +} + +void +z_erofs_stream_pool_init(struct erofs_sb_info *sbi) +{ + unsigned int algorithm; + + bzero(&sbi->stream_pool, sizeof(sbi->stream_pool)); + for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) + STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle); + cv_init(&sbi->stream_pool.cv, "erofs stream drain"); + mtx_lock(&erofs_stream_lock); + sbi->stream_pool_initialized = true; + LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link); + mtx_unlock(&erofs_stream_lock); +} + +void +z_erofs_stream_pool_fini(struct erofs_sb_info *sbi) +{ + struct erofs_stream_ctx *ctx; + + if (!sbi->stream_pool_initialized) + return; + mtx_lock(&erofs_stream_lock); + sbi->stream_pool.closing = true; + erofs_stream_wake_locked(sbi); + for (;;) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx == NULL) + break; + mtx_unlock(&erofs_stream_lock); + erofs_stream_ctx_destroy(ctx); + mtx_lock(&erofs_stream_lock); + } + while (!erofs_stream_pool_empty_locked(&sbi->stream_pool) || + sbi->stream_pool.waiters != 0) + cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock); + KASSERT(sbi->stream_pool.resident_bytes == 0, + ("erofs stream mount bytes remain at unmount")); + LIST_REMOVE(sbi, stream_pool_link); + sbi->stream_pool_initialized = false; + mtx_unlock(&erofs_stream_lock); + cv_destroy(&sbi->stream_pool.cv); +} + +static void +erofs_stream_pool_lowmem(void *arg __unused, int howto __unused) +{ + struct erofs_stream_ctx *ctx; + struct erofs_sb_info *sbi; + + for (;;) { + ctx = NULL; + mtx_lock(&erofs_stream_lock); + LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) { + ctx = erofs_stream_take_idle_locked(sbi); + if (ctx != NULL) { + ++erofs_stream_reclaims; + break; + } + } + mtx_unlock(&erofs_stream_lock); + if (ctx == NULL) + break; + erofs_stream_ctx_destroy(ctx); + } +} + +static void +erofs_stream_pool_global_init(void *arg __unused) +{ + erofs_stream_mount_budget = MIN(erofs_stream_mount_budget, + EROFS_STREAM_MOUNT_HARD_BUDGET); + erofs_stream_global_budget = MIN(erofs_stream_global_budget, + EROFS_STREAM_GLOBAL_HARD_BUDGET); + erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts, + (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS)); + erofs_stream_global_contexts_limit = MAX(0, MIN( + erofs_stream_global_contexts_limit, + (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS)); + erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached, + (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC)); + erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached, + (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC)); + cv_init(&erofs_stream_global_cv, "erofs stream wait"); + /* + * The context count is bounded by erofs_stream_global_contexts_limit; + * a UMA item limit would only add a second, unrelated place to sleep. + */ + erofs_stream_zone = uma_zcreate("erofs stream ctx", + EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + KASSERT(erofs_stream_zone != NULL, + ("cannot create erofs stream UMA zone")); + uma_zone_set_maxcache(erofs_stream_zone, + EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3); + erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem, + erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT); +} + +static void +erofs_stream_pool_global_fini(void *arg __unused) +{ + if (erofs_stream_lowmem_tag != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag); + KASSERT(LIST_EMPTY(&erofs_stream_mounts), + ("erofs stream mounts remain at unload")); + KASSERT(erofs_stream_global_contexts == 0 && + erofs_stream_global_waiters == 0 && + erofs_stream_global_resident == 0 && + erofs_stream_borrowed_contexts == 0 && + erofs_stream_idle_contexts == 0 && + erofs_stream_cached_contexts == 0, + ("erofs stream resources remain at unload")); + uma_zdestroy(erofs_stream_zone); + cv_destroy(&erofs_stream_global_cv); +} + +SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST, + erofs_stream_pool_global_init, NULL); +SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST, + erofs_stream_pool_global_fini, NULL);