update

.clang-format

@@ -0,0 +1,199 @@
+# Basic .clang-format
+---
+BasedOnStyle: WebKit
+AlignAfterOpenBracket: DontAlign
+AlignConsecutiveMacros: AcrossEmptyLines
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: false
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: InlineOnly
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: TopLevelDefinitions
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: WebKit
+BreakBeforeTernaryOperators: false
+# TODO: BreakStringLiterals can cause very strange formatting so turn it off?
+BreakStringLiterals: false
+# Prefer:
+# some_var = function(arg1,
+#    arg2)
+# over:
+# some_var =
+#     function(arg1, arg2)
+PenaltyBreakAssignment: 100
+# Prefer:
+# some_long_function(arg1, arg2
+#     arg3)
+# over:
+# some_long_function(
+#     arg1, arg2, arg3)
+PenaltyBreakBeforeFirstCallParameter: 100
+CompactNamespaces: true
+DerivePointerAlignment: false
+DisableFormat: false
+ForEachMacros:
+  - ARB_ARRFOREACH
+  - ARB_ARRFOREACH_REVWCOND
+  - ARB_ARRFOREACH_REVERSE
+  - ARB_FOREACH
+  - ARB_FOREACH_FROM
+  - ARB_FOREACH_SAFE
+  - ARB_FOREACH_REVERSE
+  - ARB_FOREACH_REVERSE_FROM
+  - ARB_FOREACH_REVERSE_SAFE
+  - BIT_FOREACH_ISCLR
+  - BIT_FOREACH_ISSET
+  - CPU_FOREACH
+  - CPU_FOREACH_ISCLR
+  - CPU_FOREACH_ISSET
+  - FOREACH_THREAD_IN_PROC
+  - FOREACH_PROC_IN_SYSTEM
+  - FOREACH_PRISON_CHILD
+  - FOREACH_PRISON_DESCENDANT
+  - FOREACH_PRISON_DESCENDANT_LOCKED
+  - FOREACH_PRISON_DESCENDANT_LOCKED_LEVEL
+  - MNT_VNODE_FOREACH_ALL
+  - MNT_VNODE_FOREACH_ACTIVE
+  - RB_FOREACH
+  - RB_FOREACH_FROM
+  - RB_FOREACH_SAFE
+  - RB_FOREACH_REVERSE
+  - RB_FOREACH_REVERSE_FROM
+  - RB_FOREACH_REVERSE_SAFE
+  - SLIST_FOREACH
+  - SLIST_FOREACH_FROM
+  - SLIST_FOREACH_FROM_SAFE
+  - SLIST_FOREACH_SAFE
+  - SLIST_FOREACH_PREVPTR
+  - SPLAY_FOREACH
+  - LIST_FOREACH
+  - LIST_FOREACH_FROM
+  - LIST_FOREACH_FROM_SAFE
+  - LIST_FOREACH_SAFE
+  - STAILQ_FOREACH
+  - STAILQ_FOREACH_FROM
+  - STAILQ_FOREACH_FROM_SAFE
+  - STAILQ_FOREACH_SAFE
+  - TAILQ_FOREACH
+  - TAILQ_FOREACH_FROM
+  - TAILQ_FOREACH_FROM_SAFE
+  - TAILQ_FOREACH_REVERSE
+  - TAILQ_FOREACH_REVERSE_FROM
+  - TAILQ_FOREACH_REVERSE_FROM_SAFE
+  - TAILQ_FOREACH_REVERSE_SAFE
+  - TAILQ_FOREACH_SAFE
+  - VM_MAP_ENTRY_FOREACH
+  - VM_PAGE_DUMP_FOREACH
+SpaceBeforeParens: ControlStatementsExceptForEachMacros
+IndentCaseLabels: false
+IndentPPDirectives: None
+Language: Cpp
+NamespaceIndentation: None
+PointerAlignment: Right
+ContinuationIndentWidth: 4
+IndentWidth: 8
+TabWidth: 8
+ColumnLimit: 80
+UseTab: Always
+SpaceAfterCStyleCast: false
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex: '^\"opt_.*\.h\"'
+    Priority: 1
+    SortPriority: 10
+  - Regex: '^<sys/cdefs\.h>'
+    Priority: 2
+    SortPriority: 20
+  - Regex: '^<sys/types\.h>'
+    Priority: 2
+    SortPriority: 21
+  - Regex: '^<sys/param\.h>'
+    Priority: 2
+    SortPriority: 22
+  - Regex: '^<sys/systm\.h>'
+    Priority: 2
+    SortPriority: 23
+  - Regex: '^<sys.*/'
+    Priority: 2
+    SortPriority: 24
+  - Regex: '^<vm/vm\.h>'
+    Priority: 3
+    SortPriority: 30
+  - Regex: '^<vm/'
+    Priority: 3
+    SortPriority: 31
+  - Regex: '^<machine/'
+    Priority: 4
+    SortPriority: 40
+  - Regex: '^<(x86|amd64|i386|xen)/'
+    Priority: 5
+    SortPriority: 50
+  - Regex: '^<dev/'
+    Priority: 6
+    SortPriority: 60
+  - Regex: '^<net.*/'
+    Priority: 7
+    SortPriority: 70
+  - Regex: '^<protocols/'
+    Priority: 7
+    SortPriority: 71
+  - Regex: '^<(fs|nfs(|client|server)|ufs)/'
+    Priority: 8
+    SortPriority: 80
+  - Regex: '^<[^/].*\.h'
+    Priority: 9
+    SortPriority: 90
+  - Regex: '^\".*\.h\"'
+    Priority: 10
+    SortPriority: 100
+# LLVM's header include ordering style is almost the exact opposite of ours.
+# Unfortunately, they have hard-coded their preferences into clang-format.
+# Clobbering this regular expression to avoid matching prevents non-system
+# headers from being forcibly moved to the top of the include list.
+# http://llvm.org/docs/CodingStandards.html#include-style
+IncludeIsMainRegex: 'BLAH_DONT_MATCH_ANYTHING'
+SortIncludes: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+TypenameMacros:
+  - ARB_ELMTYPE
+  - ARB_HEAD
+  - ARB8_HEAD
+  - ARB16_HEAD
+  - ARB32_HEAD
+  - ARB_ENTRY
+  - ARB8_ENTRY
+  - ARB16_ENTRY
+  - ARB32_ENTRY
+  - LIST_CLASS_ENTRY
+  - LIST_CLASS_HEAD
+  - LIST_ENTRY
+  - LIST_HEAD
+  - QUEUE_TYPEOF
+  - RB_ENTRY
+  - RB_HEAD
+  - SLIST_CLASS_HEAD
+  - SLIST_CLASS_ENTRY
+  - SLIST_HEAD
+  - SLIST_ENTRY
+  - SMR_POINTER
+  - SPLAY_ENTRY
+  - SPLAY_HEAD
+  - STAILQ_CLASS_ENTRY
+  - STAILQ_CLASS_HEAD
+  - STAILQ_ENTRY
+  - STAILQ_HEAD
+  - TAILQ_CLASS_ENTRY
+  - TAILQ_CLASS_HEAD
+  - TAILQ_ENTRY
+  - TAILQ_HEAD

.gitignore

@@ -0,0 +1,18 @@
+
+i386
+machine
+x86
+.cache
+
+export_syms
+
+*.o
+*.ko
+
+opt_global.h
+
+vnode_if.h
+vnode_if_newproto.h
+vnode_if_typedef.h
+
+compile_commands.json

Makefile

@@ -0,0 +1,146 @@
+KMOD= erofs
+
+.EROFS_SUPPORTED_ARCHES= amd64 aarch64 riscv64
+.if empty(.EROFS_SUPPORTED_ARCHES:M${MACHINE_ARCH})
+.error erofs requires a 64-bit little-endian FreeBSD kernel; supported MACHINE_ARCH values are amd64, aarch64, and riscv64 (got ${MACHINE_ARCH})
+.endif
+
+WITH_ZSTDIO?= 1
+.if empty(WITH_ZSTDIO:M0) && empty(WITH_ZSTDIO:M1)
+.error WITH_ZSTDIO must be 0 or 1
+.endif
+
+WITH_EROFS_DEBUG?= 1
+.if empty(WITH_EROFS_DEBUG:M0) && empty(WITH_EROFS_DEBUG:M1)
+.error WITH_EROFS_DEBUG must be 0 or 1
+.endif
+
+WITH_EROFS_SDT?= 1
+.if empty(WITH_EROFS_SDT:M0) && empty(WITH_EROFS_SDT:M1)
+.error WITH_EROFS_SDT must be 0 or 1
+.endif
+
+WITH_EROFS_XATTR?= 1
+.if empty(WITH_EROFS_XATTR:M0) && empty(WITH_EROFS_XATTR:M1)
+.error WITH_EROFS_XATTR must be 0 or 1
+.endif
+
+WITH_EROFS_POSIX_ACL?= ${WITH_EROFS_XATTR}
+.if empty(WITH_EROFS_POSIX_ACL:M0) && empty(WITH_EROFS_POSIX_ACL:M1)
+.error WITH_EROFS_POSIX_ACL must be 0 or 1
+.endif
+
+WITH_EROFS_SECURITY?= ${WITH_EROFS_XATTR}
+.if empty(WITH_EROFS_SECURITY:M0) && empty(WITH_EROFS_SECURITY:M1)
+.error WITH_EROFS_SECURITY must be 0 or 1
+.endif
+
+WITH_EROFS_BACKED_BY_FILE?= 1
+.if empty(WITH_EROFS_BACKED_BY_FILE:M0) && empty(WITH_EROFS_BACKED_BY_FILE:M1)
+.error WITH_EROFS_BACKED_BY_FILE must be 0 or 1
+.endif
+
+WITH_EROFS_ZIP?= 1
+.if empty(WITH_EROFS_ZIP:M0) && empty(WITH_EROFS_ZIP:M1)
+.error WITH_EROFS_ZIP must be 0 or 1
+.endif
+
+WITH_EROFS_ZIP_LZMA?= ${WITH_EROFS_ZIP}
+.if ${WITH_EROFS_ZIP_LZMA} != 0 && ${WITH_EROFS_ZIP_LZMA} != 1
+.error WITH_EROFS_ZIP_LZMA must be 0 or 1
+.endif
+
+WITH_EROFS_ZIP_DEFLATE?= ${WITH_EROFS_ZIP}
+.if empty(WITH_EROFS_ZIP_DEFLATE:M0) && empty(WITH_EROFS_ZIP_DEFLATE:M1)
+.error WITH_EROFS_ZIP_DEFLATE must be 0 or 1
+.endif
+
+WITH_EROFS_ZIP_ZSTD?= ${WITH_EROFS_ZIP}
+.if empty(WITH_EROFS_ZIP_ZSTD:M0) && empty(WITH_EROFS_ZIP_ZSTD:M1)
+.error WITH_EROFS_ZIP_ZSTD must be 0 or 1
+.endif
+
+WITH_EROFS_PCPU_KTHREAD?= 1
+.if empty(WITH_EROFS_PCPU_KTHREAD:M0) && empty(WITH_EROFS_PCPU_KTHREAD:M1)
+.error WITH_EROFS_PCPU_KTHREAD must be 0 or 1
+.endif
+
+.if ${WITH_EROFS_XATTR} == 0
+.if ${WITH_EROFS_POSIX_ACL} != 0 || ${WITH_EROFS_SECURITY} != 0
+.error WITH_EROFS_POSIX_ACL and WITH_EROFS_SECURITY require WITH_EROFS_XATTR=1
+.endif
+.endif
+
+.if ${WITH_EROFS_ZIP} == 0
+.if ${WITH_EROFS_ZIP_LZMA} != 0 || ${WITH_EROFS_ZIP_DEFLATE} != 0 || \
+    ${WITH_EROFS_ZIP_ZSTD} != 0
+.error WITH_EROFS_ZIP_LZMA, WITH_EROFS_ZIP_DEFLATE, and WITH_EROFS_ZIP_ZSTD require WITH_EROFS_ZIP=1
+.endif
+.endif
+
+CFLAGS+= -DWITH_EROFS_DEBUG=${WITH_EROFS_DEBUG} \
+	-DWITH_EROFS_SDT=${WITH_EROFS_SDT} \
+	-DWITH_EROFS_XATTR=${WITH_EROFS_XATTR} \
+	-DWITH_EROFS_POSIX_ACL=${WITH_EROFS_POSIX_ACL} \
+	-DWITH_EROFS_SECURITY=${WITH_EROFS_SECURITY} \
+	-DWITH_EROFS_BACKED_BY_FILE=${WITH_EROFS_BACKED_BY_FILE} \
+	-DWITH_EROFS_ZIP=${WITH_EROFS_ZIP} \
+	-DWITH_EROFS_ZIP_LZMA=${WITH_EROFS_ZIP_LZMA} \
+	-DWITH_EROFS_ZIP_DEFLATE=${WITH_EROFS_ZIP_DEFLATE} \
+	-DWITH_EROFS_ZIP_ZSTD=${WITH_EROFS_ZIP_ZSTD} \
+	-DWITH_EROFS_PCPU_KTHREAD=${WITH_EROFS_PCPU_KTHREAD}
+
+.if ${WITH_EROFS_SDT} == 1
+CFLAGS+= -DKDTRACE_HOOKS
+.endif
+
+# FreeBSD filesystem core and vnode adapter.
+SRCS=	super.c \
+	inode.c \
+	data.c \
+	namei.c \
+	dir.c \
+	sysfs.c \
+	erofs_vnops.c \
+	vnode_if.h
+
+.if ${WITH_EROFS_BACKED_BY_FILE} == 1
+SRCS+=	fileio.c
+.endif
+
+.if ${WITH_EROFS_SDT} == 1
+SRCS+=	trace.c
+.endif
+
+# FreeBSD xattr and ACL adapter.
+.if ${WITH_EROFS_XATTR} == 1
+SRCS+=	xattr.c
+.endif
+
+.if ${WITH_EROFS_ZIP} == 1
+# Compression core.
+SRCS+=	decompressor.c \
+	zmap.c \
+	zdata.c \
+	zutil.c
+
+# Algorithm backends.
+SRCS+=	decompressor_lz4.c
+.if ${WITH_EROFS_ZIP_LZMA} == 1
+SRCS+=	decompressor_lzma.c
+.endif
+.if ${WITH_EROFS_ZIP_DEFLATE} == 1
+SRCS+=	decompressor_deflate.c
+.endif
+.if ${WITH_EROFS_ZIP_ZSTD} == 1
+SRCS+=	decompressor_zstd.c
+
+# Optional Zstd control plane.
+CFLAGS.decompressor_zstd.c+= -I${SYSDIR}/contrib/zstd/lib/freebsd
+.if ${WITH_ZSTDIO} == 1
+CFLAGS.decompressor_zstd.c+= -DZSTDIO
+.endif
+.endif
+.endif
+
+.include <bsd.kmod.mk>

compress.h

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef _FS_EROFS_COMPRESS_H_
+#define _FS_EROFS_COMPRESS_H_
+
+#include "internal.h"
+
+struct z_erofs_decompress_req {
+	struct erofs_sb_info *sbi;
+	const struct erofs_map_blocks *map;
+	const void *in;
+	size_t inputsize;
+	void *out;
+	size_t outputsize;
+	bool partial_decoding;
+};
+
+struct z_erofs_decompressor {
+	/* Callbacks return zero or a positive FreeBSD errno. */
+	int (*config)(struct erofs_sb_info *, const struct erofs_super_block *,
+	    const void *, size_t);
+	int (*decompress)(const struct z_erofs_decompress_req *);
+};
+
+int z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq);
+extern const struct z_erofs_decompressor z_erofs_lz4_decomp;
+extern const struct z_erofs_decompressor z_erofs_lzma_decomp;
+extern const struct z_erofs_decompressor z_erofs_deflate_decomp;
+extern const struct z_erofs_decompressor z_erofs_zstd_decomp;
+
+#endif

data.c

@@ -0,0 +1,878 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/_maxphys.h>
+#include <sys/buf.h>
+#include <sys/libkern.h>
+#include <sys/limits.h>
+#include <sys/malloc.h>
+#include <sys/uio.h>
+#include <sys/vnode.h>
+
+#include <machine/vmparam.h>
+#include <vm/vm.h>
+
+#include "internal.h"
+#include "trace.h"
+
+#define EROFS_DIR_READAHEAD_SLOTS	256
+
+void
+erofs_put_metabuf(struct erofs_buf *buf)
+{
+	void (*release)(void *);
+	void *data;
+
+	if (buf == NULL)
+		return;
+	data = buf->data;
+	release = buf->release;
+	buf->data = NULL;
+	buf->release = NULL;
+	if (data != NULL && release != NULL)
+		release(data);
+}
+
+/*
+ * bread() on a device vnode needs no vnode lock, but breadn() reads ahead
+ * through breada(), whose inmem() asserts that the vnode is locked.  Take the
+ * lock shared (the mount's device vnode allows that) only around the request.
+ *
+ * A doomed vnode has become VBAD, which the buffer cache only accepts when it
+ * is locked, and it no longer has a device behind it, so fail the read.
+ */
+static int
+erofs_breadn_devvp(struct vnode *devvp, daddr_t blkno, int size,
+    daddr_t *rablkno, int *rabsize, int racnt, struct buf **bpp)
+{
+	int error;
+
+	vn_lock(devvp, LK_SHARED | LK_RETRY);
+	if (VN_IS_DOOMED(devvp)) {
+		VOP_UNLOCK(devvp);
+		*bpp = NULL;
+		return (ENXIO);
+	}
+	error = breadn(devvp, blkno, size, rablkno, rabsize, racnt, NOCRED,
+	    bpp);
+	VOP_UNLOCK(devvp);
+	return (error);
+}
+
+static int
+erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif,
+    erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno,
+    int *rabsize, int racnt, void **bufp)
+{
+	struct buf *bp;
+	erofs_off_t device_end, device_off, end, limit;
+	off_t blkoff, current;
+	size_t blklen, done, iosize;
+	char *out;
+	int error;
+
+	if (bufp == NULL)
+		return (EINVAL);
+	*bufp = NULL;
+	if (len == 0) {
+		return (0);
+	}
+	if (!erofs_source_is_open(dif))
+		return (ENODEV);
+	if (__builtin_add_overflow(off, (uint64_t)len, &end))
+		return (erofs_corrupt(sbi, EROFS_NID_NONE,
+		    "read of %zu bytes at %ju overflows", len, (uintmax_t)off));
+	if (blocks != 0) {
+		if (blocks > (UINT64_MAX >> sbi->blkszbits))
+			return (erofs_corrupt(sbi, EROFS_NID_NONE,
+			    "device size of %ju blocks overflows",
+			    (uintmax_t)blocks));
+		limit = blocks << sbi->blkszbits;
+		if (end > limit)
+			return (erofs_corrupt(sbi, EROFS_NID_NONE,
+			    "read of %zu bytes at %ju is beyond the device "
+			    "(%ju bytes)", len, (uintmax_t)off,
+			    (uintmax_t)limit));
+	}
+	if (__builtin_add_overflow(dif->fsoff, off, &device_off) ||
+	    __builtin_add_overflow(device_off, (uint64_t)len, &device_end))
+		return (EOVERFLOW);
+	if (device_end > dif->mediasize)
+		return (ENXIO);
+	if (device_off > INT64_MAX || device_end > (uint64_t)INT64_MAX + 1)
+		return (EOVERFLOW);
+
+	iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize;
+	if (iosize == 0 || (iosize & (iosize - 1)) != 0)
+		return (EINVAL);
+	out = malloc(len, M_EROFS, M_WAITOK);
+	if (erofs_source_is_file(dif)) {
+		error = erofs_read_file_source(sbi, dif, device_off, len, out);
+		if (error != 0) {
+			free(out, M_EROFS);
+			return (error);
+		}
+		*bufp = out;
+		return (0);
+	}
+	done = 0;
+	while (done < len) {
+		current = (off_t)(device_off + done);
+		blkoff = rounddown2(current, (off_t)iosize);
+		blklen = MIN(iosize - (size_t)(current - blkoff), len - done);
+		if (done == 0 && racnt != 0)
+			error = erofs_breadn_devvp(dif->devvp, btodb(blkoff),
+			    iosize, rablkno, rabsize, racnt, &bp);
+		else
+			error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED,
+			    &bp);
+		if (error != 0) {
+			free(out, M_EROFS);
+			return (error);
+		}
+		if (bp->b_data == NULL) {
+			brelse(bp);
+			free(out, M_EROFS);
+			return (EIO);
+		}
+		if (bp->b_resid < 0 || bp->b_bcount < 0 ||
+		    (uintmax_t)bp->b_resid > (uintmax_t)bp->b_bcount ||
+		    (uintmax_t)(current - blkoff) + (uintmax_t)blklen >
+		    (uintmax_t)bp->b_bcount - (uintmax_t)bp->b_resid) {
+			brelse(bp);
+			free(out, M_EROFS);
+			return (EIO);
+		}
+		memcpy(out + done, (char *)bp->b_data + (current - blkoff),
+		    blklen);
+		brelse(bp);
+		done += blklen;
+	}
+	*bufp = out;
+	return (0);
+}
+
+int
+erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp)
+{
+	return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len,
+	    NULL, NULL, 0, bufp));
+}
+
+void
+erofs_brelse(void *buf)
+{
+	free(buf, M_EROFS);
+}
+
+/* Read inode metadata from either the primary image or the metabox file. */
+int
+erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid,
+    erofs_off_t off, size_t len, struct erofs_buf *buf)
+{
+	void *data;
+	int error;
+
+	if (!erofs_nid_in_metabox(nid)) {
+		if (off > INT64_MAX)
+			return (EOVERFLOW);
+		error = erofs_bread(sbi, (off_t)off, len, &data);
+	} else {
+		if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL)
+			return (erofs_corrupt(sbi, nid,
+			    "metabox nid, but the image has no metabox"));
+		error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data);
+	}
+	if (error != 0)
+		return (error);
+	buf->data = data;
+	buf->release = erofs_brelse;
+	return (0);
+}
+
+static int
+erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	struct erofs_inode_chunk_index *idx;
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	uint64_t chunk_idx, chunk_size, entry_size;
+	erofs_off_t loff, chunk_start, idx_off, idx_base;
+	uint64_t image_size, addrmask;
+	uint64_t blkaddr;
+	uint16_t raw_device_id;
+	int error;
+
+	loff = map->m_la;
+	chunk_size = 1ULL << vi->chunkbits;
+	chunk_idx = loff >> vi->chunkbits;
+	chunk_start = chunk_idx << vi->chunkbits;
+
+	if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0)
+		entry_size = sizeof(struct erofs_inode_chunk_index);
+	else
+		entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE;
+	/*
+	 * inode_off is below 2^63 (the image fits its provider, whose size is
+	 * an off_t; a metabox is at most OFF_MAX), isize is 32 or 64 and
+	 * xattr_isize is below 2^19; chunk_idx is below 2^54 because
+	 * loff < size <= OFF_MAX and a chunk has at least 512 bytes.
+	 */
+	idx_base = roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize,
+	    entry_size);
+	idx_off = idx_base + chunk_idx * entry_size;
+	if (erofs_nid_in_metabox(vi->nid)) {
+		if (sbi->metabox_en == NULL ||
+		    idx_off > sbi->metabox_en->size ||
+		    entry_size > sbi->metabox_en->size - idx_off)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "chunk %ju: index entry at %ju is beyond the "
+			    "metabox",
+			    (uintmax_t)chunk_idx, (uintmax_t)idx_off));
+	} else {
+		image_size = sbi->blocks << sbi->blkszbits;
+		if (idx_off > image_size || entry_size > image_size - idx_off)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "chunk %ju: index entry at %ju is beyond the image",
+			    (uintmax_t)chunk_idx, (uintmax_t)idx_off));
+	}
+
+	error = erofs_read_metadata_range(sbi, vi->nid, idx_off, entry_size,
+	    &buf);
+	if (error != 0)
+		return (error);
+
+	idx = buf.data;
+	if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) {
+		blkaddr = le32toh(idx->startblk_lo);
+		if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0)
+			blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32;
+		raw_device_id = le16toh(idx->device_id);
+		addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ?
+		    ((1ULL << 48) - 1) : UINT32_MAX;
+	} else {
+		blkaddr = le32dec(idx);
+		raw_device_id = 0;
+		addrmask = UINT32_MAX;
+	}
+	erofs_put_metabuf(&buf);
+
+	if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) {
+		map->m_la = chunk_start;
+		map->m_pa = 0;
+		map->m_llen = MIN(chunk_size, vi->size - chunk_start);
+		map->m_plen = map->m_llen;
+		return (0);
+	}
+
+	map->m_la = chunk_start;
+	map->m_llen = MIN(chunk_size, vi->size - chunk_start);
+	map->m_plen = map->m_llen;
+	map->m_deviceid = raw_device_id & sbi->device_id_mask;
+	/* blkaddr has at most 48 bits and blkszbits at most 16. */
+	map->m_pa = blkaddr << sbi->blkszbits;
+	map->m_flags |= EROFS_MAP_MAPPED;
+	return (0);
+}
+
+static erofs_off_t
+erofs_inline_tail_start(const struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi)
+{
+	if (vi->size == 0)
+		return (0);
+	return (roundup2(vi->size, (uint64_t)sbi->block_size) -
+	    sbi->block_size);
+}
+
+static int
+erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	erofs_off_t loff, tail_start;
+	uint64_t remain, block_rem;
+
+	loff = map->m_la;
+	if (loff >= vi->size)
+		return (0);
+
+	remain = vi->size - loff;
+	switch (vi->datalayout) {
+	case EROFS_INODE_CHUNK_BASED:
+		return (erofs_map_blocks_chunk(sbi, vi, map));
+	case EROFS_INODE_FLAT_PLAIN:
+		map->m_llen = remain;
+		map->m_plen = map->m_llen;
+		if (vi->startblk == EROFS_NULL_ADDR)
+			return (0);
+		if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) ||
+		    __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff,
+		    &map->m_pa))
+			return (erofs_corrupt(sbi, vi->nid,
+			    "start block %ju overflows",
+			    (uintmax_t)vi->startblk));
+		map->m_flags |= EROFS_MAP_MAPPED;
+		return (0);
+	case EROFS_INODE_FLAT_INLINE:
+		tail_start = erofs_inline_tail_start(sbi, vi);
+		if (loff < tail_start) {
+			map->m_llen = MIN(remain, tail_start - loff);
+			map->m_plen = map->m_llen;
+			if (vi->startblk == EROFS_NULL_ADDR)
+				return (0);
+			if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) ||
+			    __builtin_add_overflow(
+			    vi->startblk << sbi->blkszbits, loff, &map->m_pa))
+				return (erofs_corrupt(sbi, vi->nid,
+				    "start block %ju overflows",
+				    (uintmax_t)vi->startblk));
+			map->m_flags |= EROFS_MAP_MAPPED;
+			return (0);
+		}
+		block_rem = sbi->block_size -
+		    ((loff - tail_start) & (sbi->block_size - 1));
+		map->m_llen = MIN(remain, block_rem);
+		map->m_plen = map->m_llen;
+		if (__builtin_add_overflow(vi->inode_off, vi->inode_isize,
+		    &map->m_pa) ||
+		    __builtin_add_overflow(map->m_pa, vi->xattr_isize,
+		    &map->m_pa) ||
+		    __builtin_add_overflow(map->m_pa, loff - tail_start,
+		    &map->m_pa))
+			return (erofs_corrupt(sbi, vi->nid,
+			    "inline tail position overflows"));
+		map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META;
+		return (0);
+	case EROFS_INODE_COMPRESSED_FULL:
+	case EROFS_INODE_COMPRESSED_COMPACT:
+		return (EOPNOTSUPP);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
+int
+erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	struct erofs_map_blocks next = { .m_la = map->m_la };
+	int err;
+
+	EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, 0);
+	if (erofs_inode_is_data_compressed(vi->datalayout)) {
+#if WITH_EROFS_ZIP
+		err = z_erofs_map_blocks(sbi, vi, &next);
+#else
+		err = EOPNOTSUPP;
+#endif
+	} else {
+		err = erofs_map_blocks_flatmode(sbi, vi, &next);
+	}
+	*map = next;
+	EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, err);
+	return (err);
+}
+static int
+erofs_check_device_range(const struct erofs_sb_info *sbi,
+    const struct erofs_device_info *dif, erofs_off_t off, uint64_t len)
+{
+	erofs_off_t end, limit;
+
+	if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) ||
+	    __builtin_add_overflow(off, len, &end))
+		return (EINTEGRITY);
+	limit = dif->blocks << sbi->blkszbits;
+	return (end > limit ? EINTEGRITY : 0);
+}
+
+static void
+erofs_fill_from_devinfo(struct erofs_map_dev *map,
+    struct erofs_device_info *dif, erofs_off_t pa)
+{
+	map->m_dif = dif;
+	map->m_pa = pa;
+}
+
+static int
+erofs_map_dev_locate(struct erofs_sb_info *sbi, struct erofs_map_dev *map)
+{
+	struct erofs_device_info *dif;
+	uint64_t start;
+	unsigned int id;
+	int error;
+
+	erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa);
+	if (map->m_deviceid != 0) {
+		if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL)
+			return (ENODEV);
+		dif = &sbi->devs[map->m_deviceid - 1];
+		error = erofs_check_device_range(sbi, dif, map->m_pa,
+		    map->m_plen);
+		if (error != 0)
+			return (error);
+		if (sbi->flatdev) {
+			if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
+				return (EINTEGRITY);
+			start = dif->uniaddr << sbi->blkszbits;
+			if (__builtin_add_overflow(map->m_pa, start,
+			    &map->m_pa))
+				return (EINTEGRITY);
+			return (0);
+		}
+		if (!erofs_source_is_open(dif))
+			return (ENODEV);
+		erofs_fill_from_devinfo(map, dif, map->m_pa);
+		return (0);
+	}
+
+	if (sbi->extra_devices == 0)
+		return (0);
+	if (sbi->flatdev) {
+		error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa,
+		    map->m_plen);
+		if (error == 0)
+			return (0);
+		for (id = 0; id < sbi->extra_devices; ++id) {
+			dif = &sbi->devs[id];
+			if (dif->uniaddr == 0 ||
+			    dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
+				continue;
+			start = dif->uniaddr << sbi->blkszbits;
+			if (map->m_pa < start)
+				continue;
+			error = erofs_check_device_range(sbi, dif,
+			    map->m_pa - start, map->m_plen);
+			if (error == 0)
+				return (0);
+			if (map->m_pa - start <
+			    (dif->blocks << sbi->blkszbits))
+				return (error);
+		}
+		return (EINTEGRITY);
+	}
+	for (id = 0; id < sbi->extra_devices; ++id) {
+		dif = &sbi->devs[id];
+		if (dif->uniaddr == 0)
+			continue;
+		if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
+			return (EINTEGRITY);
+		start = dif->uniaddr << sbi->blkszbits;
+		if (map->m_pa >= start &&
+		    map->m_pa - start < (dif->blocks << sbi->blkszbits)) {
+			error = erofs_check_device_range(sbi, dif,
+			    map->m_pa - start, map->m_plen);
+			if (error != 0)
+				return (error);
+			if (!erofs_source_is_open(dif))
+				return (ENODEV);
+			erofs_fill_from_devinfo(map, dif, map->m_pa - start);
+			break;
+		}
+	}
+	return (0);
+}
+
+/*
+ * Resolve a physical range to a device.  The lookup tries several devices
+ * and treats a miss on one as normal, so the corruption is reported here,
+ * once, when no device is able to serve the range.
+ */
+static int
+erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map)
+{
+	uint64_t pa, plen;
+	unsigned int deviceid;
+	int error;
+
+	pa = map->m_pa;
+	plen = map->m_plen;
+	deviceid = map->m_deviceid;
+	error = erofs_map_dev_locate(sbi, map);
+	if (error == EINTEGRITY)
+		return (erofs_corrupt(sbi, EROFS_NID_NONE,
+		    "range %ju+%ju of device %u is not backed by any device",
+		    (uintmax_t)pa, (uintmax_t)plen, deviceid));
+	return (error);
+}
+
+int
+erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id,
+    erofs_off_t off, size_t len, void **bufp)
+{
+	struct erofs_map_dev map;
+	erofs_blk_t blocks;
+	int error;
+
+	map = (struct erofs_map_dev) {
+		.m_pa = off,
+		.m_deviceid = device_id,
+		.m_plen = len,
+	};
+	error = erofs_map_dev(sbi, &map);
+	if (error != 0)
+		return (error);
+	blocks = map.m_dif->blocks;
+	if (map.m_dif == &sbi->dif0 && sbi->flatdev)
+		blocks = sbi->flatdev_blocks;
+	return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len,
+	    NULL, NULL, 0, bufp));
+}
+
+static int
+erofs_read_physical_readahead(struct erofs_sb_info *sbi,
+    unsigned int device_id, erofs_off_t off, size_t len,
+    unsigned int rablocks, void **bufp)
+{
+	struct erofs_map_dev current, future;
+	daddr_t *rablkno;
+	int *rabsize;
+	erofs_off_t device_pa, step;
+	erofs_blk_t blocks;
+	unsigned int count;
+	int error, ret;
+
+	current = (struct erofs_map_dev) {
+		.m_pa = off,
+		.m_deviceid = device_id,
+		.m_plen = len,
+	};
+	error = erofs_map_dev(sbi, &current);
+	if (error != 0)
+		return (error);
+	blocks = current.m_dif->blocks;
+	if (current.m_dif == &sbi->dif0 && sbi->flatdev)
+		blocks = sbi->flatdev_blocks;
+	if (rablocks == 0)
+		return (erofs_bread_device(sbi, current.m_dif, blocks,
+		    current.m_pa, len, NULL, NULL, 0, bufp));
+	/*
+	 * 256 slots are 2 KiB, which is a large part of a kernel stack inside
+	 * the recursive read path, so keep the run lists off the stack.
+	 */
+	rablocks = MIN(rablocks, (unsigned int)EROFS_DIR_READAHEAD_SLOTS);
+	rablkno = malloc(rablocks * sizeof(*rablkno), M_EROFS, M_WAITOK);
+	rabsize = malloc(rablocks * sizeof(*rabsize), M_EROFS, M_WAITOK);
+	for (count = 0; count < rablocks; count++) {
+		step = (erofs_off_t)(count + 1) * sbi->block_size;
+		if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step)
+			break;
+		future = (struct erofs_map_dev) {
+			.m_pa = off + step,
+			.m_deviceid = device_id,
+			.m_plen = sbi->block_size,
+		};
+		if (erofs_map_dev(sbi, &future) != 0 ||
+		    future.m_dif != current.m_dif ||
+		    future.m_pa != current.m_pa + step)
+			break;
+		if (__builtin_add_overflow(future.m_dif->fsoff, future.m_pa,
+		    &device_pa) || device_pa > INT64_MAX)
+			break;
+		rablkno[count] = btodb(device_pa);
+		rabsize[count] = sbi->block_size;
+	}
+	ret = erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len,
+	    rablkno, rabsize, count, bufp);
+	free(rabsize, M_EROFS);
+	free(rablkno, M_EROFS);
+	return (ret);
+}
+
+static int
+erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct erofs_map_blocks map;
+	char *out;
+	void *blk;
+	erofs_off_t mapoff, physical;
+	size_t done, want;
+	int error;
+
+	if (bufp == NULL)
+		return (EINVAL);
+	*bufp = NULL;
+	if (len == 0) {
+		return (0);
+	}
+	EROFS_TRACE_READ_FOLIO(vi, loff, len,
+	    !erofs_inode_is_data_compressed(vi->datalayout));
+	if (loff > vi->size || (uint64_t)len > vi->size - loff)
+		return (erofs_corrupt(sbi, vi->nid,
+		    "read of %zu bytes at %ju is beyond the size %ju", len,
+		    (uintmax_t)loff, (uintmax_t)vi->size));
+
+	/* Compressed file path */
+	if (erofs_inode_is_data_compressed(vi->datalayout))
+#if WITH_EROFS_ZIP
+		return (z_erofs_read_data(sbi, vi, loff, len, bufp));
+#else
+		return (EOPNOTSUPP);
+#endif
+
+	/* Uncompressed file path */
+	out = malloc(len, M_EROFS, M_WAITOK);
+	done = 0;
+	while (done < len) {
+		map = (struct erofs_map_blocks) { .m_la = loff + done };
+		error = erofs_map_blocks(sbi, vi, &map);
+		if (error != 0) {
+			free(out, M_EROFS);
+			return (error);
+		}
+		if (map.m_llen == 0 || map.m_la > loff + done ||
+		    loff + done - map.m_la >= map.m_llen) {
+			free(out, M_EROFS);
+			return (erofs_corrupt(sbi, vi->nid,
+			    "mapping of offset %ju is empty or does not cover "
+			    "it",
+			    (uintmax_t)(loff + done)));
+		}
+		mapoff = loff + done - map.m_la;
+		want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS),
+		    len - done);
+		if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
+			bzero(out + done, want);
+		} else {
+			if ((map.m_flags & EROFS_MAP_META) != 0) {
+				if (mapoff > UINT64_MAX - map.m_pa) {
+					free(out, M_EROFS);
+					return (EOVERFLOW);
+				}
+				physical = map.m_pa + mapoff;
+				error = erofs_read_metadata_range(sbi, vi->nid,
+				    physical, want, &buf);
+			} else if (done == 0 && mapoff == 0 && rablocks != 0 &&
+			    map.m_flags == EROFS_MAP_MAPPED) {
+				error = erofs_read_physical_readahead(sbi,
+				    map.m_deviceid, map.m_pa, want, rablocks,
+				    &blk);
+			} else {
+				if (mapoff > UINT64_MAX - map.m_pa) {
+					free(out, M_EROFS);
+					return (EOVERFLOW);
+				}
+				physical = map.m_pa + mapoff;
+				error = erofs_read_physical(sbi, map.m_deviceid,
+				    physical, want, &blk);
+			}
+			if (error != 0) {
+				free(out, M_EROFS);
+				return (error);
+			}
+			if ((map.m_flags & EROFS_MAP_META) != 0) {
+				memcpy(out + done, buf.data, want);
+				erofs_put_metabuf(&buf);
+			} else {
+				memcpy(out + done, blk, want);
+				erofs_brelse(blk);
+			}
+		}
+		done += want;
+	}
+	*bufp = out;
+	return (0);
+}
+
+int
+erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, void **bufp)
+{
+	return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp));
+}
+
+int
+erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, bool sequential, void **bufp)
+{
+	uint64_t dir_ra_bytes, total_blocks;
+	uint64_t remaining;
+	unsigned int rablocks;
+	int error;
+
+	rablocks = 0;
+	if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN &&
+	    sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 &&
+	    len <= sbi->block_size && loff <= vi->size &&
+	    len <= vi->size - loff) {
+		dir_ra_bytes = erofs_dir_ra_bytes(sbi);
+		total_blocks = dir_ra_bytes / sbi->block_size;
+		if (total_blocks > 1) {
+			remaining = vi->size - loff - len;
+			rablocks = MIN(total_blocks - 1,
+			    howmany(remaining, sbi->block_size));
+			rablocks = MIN(rablocks,
+			    (unsigned int)EROFS_DIR_READAHEAD_SLOTS);
+		}
+	}
+	EROFS_TRACE_READAHEAD(vi, loff >> PAGE_SHIFT,
+	    howmany((uint64_t)len +
+	    (uint64_t)rablocks * (uint64_t)sbi->block_size, PAGE_SIZE),
+	    !erofs_inode_is_data_compressed(vi->datalayout));
+	error = erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp);
+	if (error == 0 && rablocks != 0)
+		erofs_record_dir_readahead(sbi, rablocks);
+	return (error);
+}
+
+static int
+erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct uio *uio)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct erofs_map_blocks map;
+	void *blk;
+	erofs_off_t mapoff, physical;
+	size_t want, chunk;
+	int error;
+
+	if (uio->uio_offset < 0)
+		return (EINVAL);
+	if ((uint64_t)uio->uio_offset >= vi->size)
+		return (0);
+	EROFS_TRACE_READ_FOLIO(vi, (uint64_t)uio->uio_offset,
+	    (uint64_t)uio->uio_resid,
+	    !erofs_inode_is_data_compressed(vi->datalayout));
+
+	/* Compressed file path */
+	if (erofs_inode_is_data_compressed(vi->datalayout))
+#if WITH_EROFS_ZIP
+		return (z_erofs_read_uio(sbi, vi, uio));
+#else
+		return (EOPNOTSUPP);
+#endif
+
+	/* Uncompressed file path */
+	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) {
+		map = (struct erofs_map_blocks) { .m_la = uio->uio_offset };
+		error = erofs_map_blocks(sbi, vi, &map);
+		if (error != 0)
+			return (error);
+		if (map.m_llen == 0 || map.m_la > (uint64_t)uio->uio_offset ||
+		    (uint64_t)uio->uio_offset - map.m_la >= map.m_llen)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "mapping of offset %jd is empty or does not cover "
+			    "it",
+			    (intmax_t)uio->uio_offset));
+		mapoff = (uint64_t)uio->uio_offset - map.m_la;
+		want = MIN((size_t)MIN(map.m_llen - mapoff, (uint64_t)MAXPHYS),
+		    (size_t)uio->uio_resid);
+		if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
+			chunk = want;
+			while (chunk > 0) {
+				size_t zlen = MIN(chunk, ZERO_REGION_SIZE);
+
+				error = uiomove(__DECONST(void *, zero_region),
+				    zlen, uio);
+				if (error != 0)
+					return (error);
+				chunk -= zlen;
+			}
+			continue;
+		}
+		if ((map.m_flags & EROFS_MAP_META) != 0) {
+			if (mapoff > UINT64_MAX - map.m_pa)
+				return (EOVERFLOW);
+			physical = map.m_pa + mapoff;
+			error = erofs_read_metadata_range(sbi, vi->nid,
+			    physical, want, &buf);
+		} else {
+			if (mapoff > UINT64_MAX - map.m_pa)
+				return (EOVERFLOW);
+			physical = map.m_pa + mapoff;
+			error = erofs_read_physical(sbi, map.m_deviceid,
+			    physical, want, &blk);
+		}
+		if (error != 0)
+			return (error);
+		if ((map.m_flags & EROFS_MAP_META) != 0) {
+			error = uiomove(buf.data, want, uio);
+			erofs_put_metabuf(&buf);
+		} else {
+			error = uiomove(blk, want, uio);
+			erofs_brelse(blk);
+		}
+		if (error != 0)
+			return (error);
+	}
+	return (0);
+}
+
+/* Read regular file data. */
+int
+erofs_read_file(struct vnode *vp, struct uio *uio)
+{
+	return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio));
+}
+
+int
+erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data)
+{
+	struct erofs_sb_info *sbi;
+	struct erofs_inode *vi;
+	struct erofs_map_blocks map;
+	erofs_off_t pos, end;
+	bool data;
+	int error;
+
+	if (vp->v_type != VREG)
+		return (ENOTTY);
+	vi = VTOE(vp);
+	if (*offset < 0 || (uint64_t)*offset >= vi->size)
+		return (ENXIO);
+
+	sbi = MTOE(vp->v_mount);
+	pos = (erofs_off_t)*offset;
+	while (pos < vi->size) {
+		map = (struct erofs_map_blocks) { .m_la = pos };
+		error = erofs_map_blocks(sbi, vi, &map);
+		if (error != 0)
+			return (error);
+		if (map.m_la > pos || map.m_llen == 0 ||
+		    map.m_la > UINT64_MAX - map.m_llen) {
+			return (erofs_corrupt(sbi, vi->nid,
+			    "mapping of offset %ju is empty or does not cover "
+			    "it",
+			    (uintmax_t)pos));
+		}
+		end = map.m_la + map.m_llen;
+		if (pos >= end)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "mapping of offset %ju ends at %ju", (uintmax_t)pos,
+			    (uintmax_t)end));
+		data = (map.m_flags &
+		    (EROFS_MAP_MAPPED | EROFS_MAP_FRAGMENT)) != 0;
+		if (data == seek_data) {
+			*offset = (off_t)pos;
+			return (0);
+		}
+		pos = MIN(end, vi->size);
+	}
+
+	if (seek_data)
+		return (ENXIO);
+	if (vi->size > OFF_MAX)
+		return (EOVERFLOW);
+	*offset = (off_t)vi->size;
+	return (0);
+}
+
+/* Read symlink target string. */
+int
+erofs_readlink_target(struct vnode *vp, struct uio *uio)
+{
+	struct erofs_inode *vi;
+
+	vi = VTOE(vp);
+	if (vi->size == 0)
+		return (erofs_corrupt(MTOE(vp->v_mount), vi->nid,
+		    "empty symlink"));
+	return (erofs_read_uio(MTOE(vp->v_mount), vi, uio));
+}

decompressor.c

@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2024 Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/endian.h>
+#include <sys/libkern.h>
+
+#include "compress.h"
+
+static int
+z_erofs_load_lz4_config(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb, const void *data, size_t size)
+{
+	const struct z_erofs_lz4_cfgs *lz4;
+	uint32_t max_pclusterblks;
+	uint16_t distance;
+
+	if (data != NULL) {
+		if (size < sizeof(*lz4))
+			return (EINTEGRITY);
+		lz4 = data;
+		max_pclusterblks = le16toh(lz4->max_pclusterblks);
+		if (max_pclusterblks >
+		    (Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits))
+			return (EOPNOTSUPP);
+	} else {
+		distance = le16toh(dsb->u1.lz4_max_distance);
+		if (distance == 0 && !erofs_sb_has_lz4_0padding(sbi))
+			return (0);
+		sbi->available_compr_algs = 1U << Z_EROFS_COMPRESSION_LZ4;
+	}
+	return (0);
+}
+
+static int
+z_erofs_transform_plain(const struct z_erofs_decompress_req *rq)
+{
+	const uint8_t *src;
+	uint8_t *dst;
+	size_t first, offset, span;
+
+	if (rq->outputsize > rq->inputsize)
+		return (EINTEGRITY);
+	src = rq->in;
+	dst = rq->out;
+	if (rq->map->m_algorithmformat == Z_EROFS_COMPRESSION_SHIFTED) {
+		memmove(dst, src, rq->outputsize);
+		return (0);
+	}
+	span = rq->sbi->block_size -
+	    (rq->map->m_la & (rq->sbi->block_size - 1));
+	if (span > rq->inputsize)
+		return (EINTEGRITY);
+	first = MIN(span, rq->outputsize);
+	offset = rq->inputsize - span;
+	if (offset > rq->inputsize || first > rq->inputsize - offset)
+		return (EINTEGRITY);
+	memmove(dst, src + offset, first);
+	if (first < rq->outputsize)
+		memmove(dst + first, src, rq->outputsize - first);
+	return (0);
+}
+
+static const struct z_erofs_decompressor z_erofs_shifted_decomp = {
+	.decompress = z_erofs_transform_plain,
+};
+
+static const struct z_erofs_decompressor z_erofs_interlaced_decomp = {
+	.decompress = z_erofs_transform_plain,
+};
+
+const struct z_erofs_decompressor z_erofs_lz4_decomp = {
+	.config = z_erofs_load_lz4_config,
+	.decompress = z_erofs_lz4_buffer_decompress,
+};
+
+static const struct z_erofs_decompressor * const z_erofs_decomp[] = {
+	[Z_EROFS_COMPRESSION_SHIFTED] = &z_erofs_shifted_decomp,
+	[Z_EROFS_COMPRESSION_INTERLACED] = &z_erofs_interlaced_decomp,
+	[Z_EROFS_COMPRESSION_LZ4] = &z_erofs_lz4_decomp,
+#if WITH_EROFS_ZIP_LZMA
+	[Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp,
+#endif
+#if WITH_EROFS_ZIP_DEFLATE
+	[Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp,
+#endif
+#if WITH_EROFS_ZIP_ZSTD
+	[Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp,
+#endif
+};
+
+static int
+z_erofs_read_cfg(struct erofs_sb_info *sbi, uint64_t *offset,
+    struct erofs_buf *buf,
+    size_t *sizep)
+{
+	struct erofs_buf metabuf = EROFS_BUF_INITIALIZER;
+	uint8_t length_buf[2];
+	uint64_t aligned;
+	uint16_t length;
+	int error;
+
+	/*
+	 * At most one configuration per algorithm, each at most 64 KiB, follows
+	 * the superblock, so the offset stays far below the overflow limit; the
+	 * reads are range checked.
+	 */
+	aligned = roundup2(*offset, 4);
+	error = erofs_read_metadata_range(sbi, 0, aligned, sizeof(length_buf),
+	    &metabuf);
+	if (error != 0)
+		return (error);
+	memcpy(length_buf, metabuf.data, sizeof(length_buf));
+	erofs_put_metabuf(&metabuf);
+	length = le16dec(length_buf);
+	*sizep = length != 0 ? length : UINT16_MAX + 1U;
+	*offset = aligned + sizeof(length_buf);
+	error = erofs_read_metadata_range(sbi, 0, *offset, *sizep, buf);
+	if (error == 0)
+		*offset += *sizep;
+	return (error);
+}
+
+int
+z_erofs_parse_cfgs(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb)
+{
+	struct erofs_buf data = EROFS_BUF_INITIALIZER;
+	const struct z_erofs_decompressor *decompressor;
+	uint64_t offset;
+	uint16_t algorithms;
+	size_t size;
+	int algorithm, ret;
+
+	if (!erofs_sb_has_compr_cfgs(sbi))
+		return (z_erofs_lz4_decomp.config(sbi, dsb, NULL, 0));
+	algorithms = le16toh(dsb->u1.available_compr_algs);
+	sbi->available_compr_algs = algorithms;
+	if ((algorithms & ~Z_EROFS_ALL_COMPR_ALGS) != 0)
+		return (EOPNOTSUPP);
+	offset = EROFS_SUPER_OFFSET + sbi->sb_size;
+	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX;
+	    ++algorithm) {
+		if ((algorithms & (1U << algorithm)) == 0)
+			continue;
+		ret = z_erofs_read_cfg(sbi, &offset, &data, &size);
+		if (ret != 0)
+			return (ret);
+		decompressor = z_erofs_decomp[algorithm];
+		if (decompressor == NULL || decompressor->config == NULL)
+			ret = EOPNOTSUPP;
+		else
+			ret = decompressor->config(sbi, dsb, data.data, size);
+		erofs_put_metabuf(&data);
+		if (ret != 0)
+			return (ret);
+	}
+	return (0);
+}
+
+int
+z_erofs_decompress(struct erofs_sb_info *sbi,
+    const struct erofs_map_blocks *map, const void *src0, size_t srclen,
+    void *dst, size_t dstlen, bool partial)
+{
+	const struct z_erofs_decompressor *decompressor;
+	struct z_erofs_decompress_req rq;
+	const uint8_t *src;
+	uint8_t algorithm;
+	size_t padding, padding_limit;
+
+	algorithm = map->m_algorithmformat;
+	if (algorithm >= nitems(z_erofs_decomp) ||
+	    z_erofs_decomp[algorithm] == NULL ||
+	    z_erofs_decomp[algorithm]->decompress == NULL)
+		return (EOPNOTSUPP);
+	decompressor = z_erofs_decomp[algorithm];
+	rq = (struct z_erofs_decompress_req) {
+		.sbi = sbi,
+		.map = map,
+		.in = src0,
+		.inputsize = srclen,
+		.out = dst,
+		.outputsize = dstlen,
+		.partial_decoding = partial,
+	};
+	if (algorithm == Z_EROFS_COMPRESSION_SHIFTED ||
+	    algorithm == Z_EROFS_COMPRESSION_INTERLACED)
+		return (decompressor->decompress(&rq));
+
+	src = src0;
+	if (map->m_algorithmformat != Z_EROFS_COMPRESSION_LZ4 ||
+	    erofs_sb_has_lz4_0padding(sbi)) {
+		padding_limit = MIN(srclen, sbi->block_size -
+		    (map->m_pa & (sbi->block_size - 1)));
+		for (padding = 0; padding < padding_limit && src[padding] == 0;
+		    ++padding)
+			;
+		if (padding == padding_limit)
+			return (EINTEGRITY);
+		src += padding;
+		srclen -= padding;
+	}
+	rq.in = src;
+	rq.inputsize = srclen;
+	return (decompressor->decompress(&rq));
+}

decompressor_deflate.c

@@ -0,0 +1,174 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/* Minimal DEFLATE decompressor for EROFS FreeBSD */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/limits.h>
+
+#include <contrib/zlib/zlib.h>
+
+#include "compress.h"
+
+struct z_erofs_deflate_ctx {
+	struct erofs_stream_ctx pool;
+	z_stream stream;
+	bool initialized;
+};
+
+_Static_assert(sizeof(struct z_erofs_deflate_ctx) <=
+    EROFS_STREAM_CTX_WRAPPER_SIZE, "Deflate stream wrapper exceeds UMA item");
+
+static voidpf
+z_erofs_deflate_alloc(voidpf opaque, uInt items, uInt size)
+{
+	struct erofs_stream_ctx *pool;
+	pool = opaque;
+	/* Two 32-bit factors cannot overflow a 64-bit size_t. */
+	return (z_erofs_stream_ctx_alloc(pool, (size_t)items * size));
+}
+
+static void
+z_erofs_deflate_free(voidpf opaque, voidpf address)
+{
+
+	z_erofs_stream_ctx_free(opaque, address);
+}
+
+static int
+z_erofs_load_deflate_config(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb __unused, const void *data, size_t size)
+{
+	const struct z_erofs_deflate_cfgs *deflate;
+
+	if (size < sizeof(*deflate))
+		return (EINTEGRITY);
+	deflate = data;
+	if (deflate->windowbits > MAX_WBITS)
+		return (EOPNOTSUPP);
+	return (0);
+}
+
+static int
+z_erofs_deflate_error(int ret)
+{
+
+	switch (ret) {
+	case Z_MEM_ERROR:
+		return (ENOMEM);
+	case Z_VERSION_ERROR:
+		return (EOPNOTSUPP);
+	case Z_NEED_DICT:
+	case Z_DATA_ERROR:
+	case Z_BUF_ERROR:
+		return (EINTEGRITY);
+	default:
+		return (EIO);
+	}
+}
+
+static void
+z_erofs_deflate_ctx_fini(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_deflate_ctx *ctx;
+
+	ctx = (struct z_erofs_deflate_ctx *)pool;
+	if (ctx->initialized) {
+		(void)inflateEnd(&ctx->stream);
+		ctx->initialized = false;
+	}
+}
+
+static int
+z_erofs_deflate_ctx_init(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_deflate_ctx *ctx;
+	int ret;
+
+	ctx = (struct z_erofs_deflate_ctx *)pool;
+	bzero(&ctx->stream, sizeof(ctx->stream));
+	ctx->stream.zalloc = z_erofs_deflate_alloc;
+	ctx->stream.zfree = z_erofs_deflate_free;
+	ctx->stream.opaque = pool;
+	ret = inflateInit2(&ctx->stream, -MAX_WBITS);
+	if (ret != Z_OK)
+		return (z_erofs_deflate_error(ret));
+	ctx->initialized = true;
+	return (0);
+}
+
+static int
+z_erofs_deflate_finish(const struct z_erofs_decompress_req *rq, int ret,
+    uInt avail_in)
+{
+
+	if (rq->partial_decoding)
+		return (0);
+	if (ret != Z_STREAM_END || avail_in != 0)
+		return (EINTEGRITY);
+	return (0);
+}
+
+static int
+z_erofs_deflate_decompress(const struct z_erofs_decompress_req *rq)
+{
+	struct erofs_stream_ctx *pool;
+	struct z_erofs_deflate_ctx *ctx;
+	z_stream *strm;
+	uInt in_before, out_before;
+	int error, ret;
+
+	/* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */
+	EROFS_DEBUG_ASSERT(rq->inputsize <= UINT_MAX &&
+	    rq->outputsize <= UINT_MAX, ("erofs: deflate sizes"));
+
+	error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_DEFLATE,
+	    sizeof(*ctx), z_erofs_deflate_ctx_init, z_erofs_deflate_ctx_fini,
+	    &pool);
+	if (error != 0)
+		return (error);
+	ctx = (struct z_erofs_deflate_ctx *)pool;
+	strm = &ctx->stream;
+	pool->allocation_failed = false;
+	ret = inflateReset2(strm, -MAX_WBITS);
+	if (ret != Z_OK) {
+		error = z_erofs_deflate_error(ret);
+		z_erofs_stream_ctx_put(pool, false);
+		return (error);
+	}
+	strm->next_in = __DECONST(void *, rq->in);
+	strm->avail_in = rq->inputsize;
+	strm->next_out = rq->out;
+	strm->avail_out = rq->outputsize;
+
+	error = 0;
+	ret = Z_OK;
+	while (strm->avail_out != 0) {
+		in_before = strm->avail_in;
+		out_before = strm->avail_out;
+		ret = inflate(strm, Z_SYNC_FLUSH);
+		if (ret == Z_STREAM_END)
+			break;
+		if (ret != Z_OK) {
+			error = z_erofs_deflate_error(ret);
+			break;
+		}
+		if (strm->avail_in == in_before &&
+		    strm->avail_out == out_before) {
+			error = EINTEGRITY;
+			break;
+		}
+	}
+	if (error == 0 && strm->avail_out != 0)
+		error = EINTEGRITY;
+	else if (error == 0)
+		error = z_erofs_deflate_finish(rq, ret, strm->avail_in);
+	if (pool->allocation_failed)
+		error = ENOMEM;
+	z_erofs_stream_ctx_put(pool, error == 0);
+	return (error);
+}
+
+const struct z_erofs_decompressor z_erofs_deflate_decomp = {
+	.config = z_erofs_load_deflate_config,
+	.decompress = z_erofs_deflate_decompress,
+};

decompressor_lz4.c

@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/* Contiguous-buffer LZ4 backend for EROFS FreeBSD. */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/endian.h>
+
+#include "compress.h"
+
+#define EROFS_LZ4_TOKEN_LITERAL_SHIFT 4
+#define EROFS_LZ4_TOKEN_MATCH_MASK 0x0f
+#define EROFS_LZ4_MAX_RUN 15
+#define EROFS_LZ4_EXT_SENTINEL 255
+#define EROFS_LZ4_MIN_MATCH 4
+#define EROFS_LZ4_OFFSET_BYTES 2
+
+static int
+lz4_finish(const uint8_t *ip, const uint8_t *iend, int partial)
+{
+	if (partial)
+		return (0);
+	while (ip < iend) {
+		if (*ip++ != 0)
+			return (EINTEGRITY);
+	}
+	return (0);
+}
+
+int
+z_erofs_lz4_buffer_decompress(const struct z_erofs_decompress_req *rq)
+{
+	const uint8_t *input;
+	uint8_t *output;
+	unsigned int token;
+	size_t length, copylen;
+	size_t offset;
+	size_t input_pos, output_pos;
+
+	if (rq == NULL || (rq->inputsize != 0 && rq->in == NULL) ||
+	    (rq->outputsize != 0 && rq->out == NULL))
+		return (EINTEGRITY);
+	if (rq->inputsize == 0)
+		return (rq->outputsize == 0 ? 0 : EINTEGRITY);
+	/*
+	 * A length grows by at most 255 per input byte, so it cannot reach
+	 * SIZE_MAX however long the input is.
+	 */
+	input = rq->in;
+	output = rq->out;
+	input_pos = 0;
+	output_pos = 0;
+
+	while (input_pos < rq->inputsize) {
+		token = input[input_pos++];
+		length = token >> EROFS_LZ4_TOKEN_LITERAL_SHIFT;
+		if (length == EROFS_LZ4_MAX_RUN) {
+			unsigned int value;
+			do {
+				if (input_pos >= rq->inputsize)
+					return (EINTEGRITY);
+				value = input[input_pos++];
+				length += value;
+			} while (value == EROFS_LZ4_EXT_SENTINEL);
+		}
+		if (length > rq->inputsize - input_pos)
+			return (EINTEGRITY);
+		if (!rq->partial_decoding &&
+		    length > rq->outputsize - output_pos)
+			return (EINTEGRITY);
+		copylen = MIN(length, rq->outputsize - output_pos);
+		memcpy(output + output_pos, input + input_pos, copylen);
+		input_pos += length;
+		output_pos += copylen;
+		if (output_pos == rq->outputsize)
+			return (lz4_finish(input + input_pos,
+			    input + rq->inputsize, rq->partial_decoding));
+		if (input_pos >= rq->inputsize)
+			break;
+		if (EROFS_LZ4_OFFSET_BYTES > rq->inputsize - input_pos)
+			return (EINTEGRITY);
+		offset = le16dec(input + input_pos);
+		input_pos += EROFS_LZ4_OFFSET_BYTES;
+		if (offset == 0 || offset > output_pos)
+			return (EINTEGRITY);
+		length = token & EROFS_LZ4_TOKEN_MATCH_MASK;
+		if (length == EROFS_LZ4_MAX_RUN) {
+			unsigned int value;
+			do {
+				if (input_pos >= rq->inputsize)
+					return (EINTEGRITY);
+				value = input[input_pos++];
+				length += value;
+			} while (value == EROFS_LZ4_EXT_SENTINEL);
+		}
+		length += EROFS_LZ4_MIN_MATCH;
+		if (!rq->partial_decoding &&
+		    length > rq->outputsize - output_pos)
+			return (EINTEGRITY);
+		copylen = MIN(length, rq->outputsize - output_pos);
+		while (copylen-- != 0) {
+			output[output_pos] = output[output_pos - offset];
+			++output_pos;
+		}
+		if (output_pos == rq->outputsize)
+			return (lz4_finish(input + input_pos,
+			    input + rq->inputsize, rq->partial_decoding));
+	}
+	return (output_pos == rq->outputsize ?
+	    lz4_finish(input + input_pos, input + rq->inputsize,
+	    rq->partial_decoding) : EINTEGRITY);
+}

decompressor_lzma.c

@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * EROFS MicroLZMA wrapper around FreeBSD's bundled XZ Embedded decoder.
+ * The decoder source is compiled with private symbol names because the
+ * stock xz.ko does not enable its optional MicroLZMA entry points.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include "compress.h"
+
+#define XZ_DEC_MICROLZMA
+#define xz_dec_lzma2_create erofs_xz_dec_lzma2_create
+#define xz_dec_lzma2_reset erofs_xz_dec_lzma2_reset
+#define xz_dec_lzma2_run erofs_xz_dec_lzma2_run
+#define xz_dec_lzma2_end erofs_xz_dec_lzma2_end
+#define xz_dec_microlzma_alloc erofs_xz_dec_microlzma_alloc
+#define xz_dec_microlzma_reset erofs_xz_dec_microlzma_reset
+#define xz_dec_microlzma_run erofs_xz_dec_microlzma_run
+#define xz_dec_microlzma_end erofs_xz_dec_microlzma_end
+#define xz_malloc erofs_xz_malloc
+#define xz_free erofs_xz_free
+
+static void *
+erofs_xz_malloc(unsigned long size)
+{
+	return (malloc(size, M_EROFS, M_WAITOK));
+}
+
+static void
+erofs_xz_free(void *ptr)
+{
+	free(ptr, M_EROFS);
+}
+
+#include <contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c>
+
+
+#undef bool
+#undef false
+#undef true
+#undef min
+
+struct z_erofs_lzma_ctx {
+	struct erofs_stream_ctx pool;
+	struct xz_dec_microlzma *state;
+	size_t state_bytes;
+};
+
+_Static_assert(sizeof(struct z_erofs_lzma_ctx) <=
+    EROFS_STREAM_CTX_WRAPPER_SIZE, "LZMA stream wrapper exceeds UMA item");
+
+static void
+z_erofs_lzma_ctx_fini(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_lzma_ctx *ctx;
+
+	ctx = (struct z_erofs_lzma_ctx *)pool;
+	if (ctx->state != NULL) {
+		xz_dec_microlzma_end(ctx->state);
+		ctx->state = NULL;
+	}
+	if (ctx->state_bytes != 0) {
+		z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes);
+		ctx->state_bytes = 0;
+	}
+}
+
+static int
+z_erofs_lzma_ctx_init(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_lzma_ctx *ctx;
+	int error;
+
+	ctx = (struct z_erofs_lzma_ctx *)pool;
+	ctx->state_bytes = sizeof(*ctx->state);
+	error = z_erofs_stream_ctx_charge(pool, ctx->state_bytes);
+	if (error != 0) {
+		ctx->state_bytes = 0;
+		return (error);
+	}
+	ctx->state = xz_dec_microlzma_alloc(XZ_SINGLE,
+	    pool->sbi->lzma_dict_size);
+	if (ctx->state == NULL) {
+		z_erofs_stream_ctx_uncharge(pool, ctx->state_bytes);
+		ctx->state_bytes = 0;
+		pool->allocation_failed = 1;
+		return (ENOMEM);
+	}
+	return (0);
+}
+
+static int
+z_erofs_load_lzma_config(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb __unused, const void *data, size_t size)
+{
+	const struct z_erofs_lzma_cfgs *lzma;
+	uint32_t dict_size;
+
+	if (size < sizeof(*lzma))
+		return (EINTEGRITY);
+	lzma = data;
+	if (le16toh(lzma->format) != 0)
+		return (EOPNOTSUPP);
+	dict_size = le32toh(lzma->dict_size);
+	if (dict_size < 4096)
+		return (EINTEGRITY);
+	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
+		return (EOPNOTSUPP);
+	sbi->lzma_dict_size = dict_size;
+	return (0);
+}
+
+static int
+z_erofs_lzma_error(enum xz_ret ret)
+{
+
+	switch (ret) {
+	case XZ_MEM_ERROR:
+		return (ENOMEM);
+	case XZ_MEMLIMIT_ERROR:
+	case XZ_OPTIONS_ERROR:
+	case XZ_UNSUPPORTED_CHECK:
+		return (EOPNOTSUPP);
+	default:
+		return (EINTEGRITY);
+	}
+}
+
+static int
+z_erofs_lzma_finish(const struct z_erofs_decompress_req *rq, enum xz_ret ret,
+    size_t input_pos)
+{
+
+	if (rq->partial_decoding &&
+	    (ret == XZ_OK || ret == XZ_STREAM_END))
+		return (0);
+	if (!rq->partial_decoding && ret == XZ_STREAM_END &&
+	    input_pos == rq->inputsize)
+		return (0);
+	return (z_erofs_lzma_error(ret));
+}
+
+static int
+z_erofs_lzma_decompress(const struct z_erofs_decompress_req *rq)
+{
+	struct erofs_stream_ctx *pool;
+	struct z_erofs_lzma_ctx *ctx;
+	struct xz_buf buffer;
+	enum xz_ret ret;
+	int error;
+
+	/* At most 1 MiB and 12 MiB (z_erofs_map_sanity_check()). */
+	EROFS_DEBUG_ASSERT(rq->inputsize <= UINT32_MAX &&
+	    rq->outputsize <= UINT32_MAX, ("erofs: lzma sizes"));
+	error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_LZMA,
+	    sizeof(*ctx), z_erofs_lzma_ctx_init, z_erofs_lzma_ctx_fini, &pool);
+	if (error != 0)
+		return (error);
+	ctx = (struct z_erofs_lzma_ctx *)pool;
+	bzero(&buffer, sizeof(buffer));
+	buffer.in = rq->in;
+	buffer.in_size = rq->inputsize;
+	buffer.out = rq->out;
+	buffer.out_size = rq->outputsize;
+	xz_dec_microlzma_reset(ctx->state, (uint32_t)rq->inputsize,
+	    (uint32_t)rq->outputsize, !rq->partial_decoding);
+	ret = xz_dec_microlzma_run(ctx->state, &buffer);
+	if (buffer.out_pos != rq->outputsize)
+		error = EINTEGRITY;
+	else
+		error = z_erofs_lzma_finish(rq, ret, buffer.in_pos);
+	z_erofs_stream_ctx_put(pool, error == 0);
+	return (error);
+}
+
+const struct z_erofs_decompressor z_erofs_lzma_decomp = {
+	.config = z_erofs_load_lzma_config,
+	.decompress = z_erofs_lzma_decompress,
+};

decompressor_zstd.c

@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/* Minimal zstd decompressor for EROFS FreeBSD */
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include "compress.h"
+
+#ifdef ZSTDIO
+#define ZSTD_STATIC_LINKING_ONLY
+#include <contrib/zstd/lib/zstd.h>
+
+
+struct z_erofs_zstd_ctx {
+	struct erofs_stream_ctx pool;
+	ZSTD_DCtx *dctx;
+};
+
+_Static_assert(sizeof(struct z_erofs_zstd_ctx) <=
+    EROFS_STREAM_CTX_WRAPPER_SIZE, "Zstd stream wrapper exceeds UMA item");
+#endif
+
+static bool
+erofs_zstd_available(void)
+{
+#ifdef ZSTDIO
+	return (true);
+#else
+	return (false);
+#endif
+}
+
+static int
+z_erofs_load_zstd_config(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb __unused, const void *data, size_t size)
+{
+	const struct z_erofs_zstd_cfgs *zstd;
+
+	if (!erofs_zstd_available()) {
+		vfs_mount_error(sbi->mnt,
+		    "erofs: ZSTD compression requires ZSTDIO support");
+		return (EOPNOTSUPP);
+	}
+	if (data == NULL || size < sizeof(*zstd))
+		return (EINTEGRITY);
+	zstd = data;
+	if (zstd->format != 0 || zstd->windowlog > 10)
+		return (EOPNOTSUPP);
+	sbi->zstd_windowlog = zstd->windowlog;
+	return (0);
+}
+
+#ifdef ZSTDIO
+static void *
+zstd_alloc(void *opaque, size_t size)
+{
+	return (z_erofs_stream_ctx_alloc(opaque, size));
+}
+
+static void
+zstd_free(void *opaque, void *address)
+{
+	z_erofs_stream_ctx_free(opaque, address);
+}
+
+static void
+z_erofs_zstd_ctx_fini(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_zstd_ctx *ctx;
+
+	ctx = (struct z_erofs_zstd_ctx *)pool;
+	if (ctx->dctx != NULL) {
+		(void)ZSTD_freeDCtx(ctx->dctx);
+		ctx->dctx = NULL;
+	}
+}
+
+static int
+z_erofs_zstd_ctx_init(struct erofs_stream_ctx *pool)
+{
+	struct z_erofs_zstd_ctx *ctx;
+	ZSTD_customMem alloc;
+	size_t ret;
+
+	ctx = (struct z_erofs_zstd_ctx *)pool;
+	alloc = (ZSTD_customMem) {
+		.customAlloc = zstd_alloc,
+		.customFree = zstd_free,
+		.opaque = pool,
+	};
+	ctx->dctx = ZSTD_createDCtx_advanced(alloc);
+	if (ctx->dctx == NULL)
+		return (ENOMEM);
+	ret = ZSTD_DCtx_setParameter(ctx->dctx, ZSTD_d_windowLogMax,
+	    pool->sbi->zstd_windowlog + 10);
+	if (ZSTD_isError(ret))
+		return (pool->allocation_failed ? ENOMEM : EOPNOTSUPP);
+	return (0);
+}
+
+static int
+z_erofs_zstd_finish(const struct z_erofs_decompress_req *rq, size_t ret,
+    size_t input_pos, size_t input_size)
+{
+
+	if (rq->partial_decoding)
+		return (0);
+	if (ret != 0 || input_pos != input_size)
+		return (EINTEGRITY);
+	return (0);
+}
+
+static int
+z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq)
+{
+	struct erofs_stream_ctx *pool;
+	struct z_erofs_zstd_ctx *ctx;
+	ZSTD_inBuffer input;
+	ZSTD_outBuffer output;
+	size_t in_before, out_before, ret;
+	int error;
+
+	error = z_erofs_stream_ctx_get(rq->sbi, Z_EROFS_COMPRESSION_ZSTD,
+	    sizeof(*ctx), z_erofs_zstd_ctx_init, z_erofs_zstd_ctx_fini, &pool);
+	if (error != 0)
+		return (error);
+	ctx = (struct z_erofs_zstd_ctx *)pool;
+	pool->allocation_failed = false;
+
+	input = (ZSTD_inBuffer) {
+		.src = rq->in,
+		.size = rq->inputsize,
+	};
+	output = (ZSTD_outBuffer) {
+		.dst = rq->out,
+		.size = rq->outputsize,
+	};
+	error = 0;
+	ret = 1;
+	while (output.pos != output.size) {
+		in_before = input.pos;
+		out_before = output.pos;
+		ret = ZSTD_decompressStream(ctx->dctx, &output, &input);
+		if (ZSTD_isError(ret)) {
+			error = pool->allocation_failed ? ENOMEM : EINTEGRITY;
+			break;
+		}
+		if (input.pos == in_before && output.pos == out_before) {
+			error = EINTEGRITY;
+			break;
+		}
+		if (ret == 0)
+			break;
+	}
+	if (error == 0 && output.pos != output.size)
+		error = EINTEGRITY;
+	else if (error == 0)
+		error = z_erofs_zstd_finish(rq, ret, input.pos, input.size);
+	z_erofs_stream_ctx_put(pool, error == 0 && !rq->partial_decoding);
+	return (error);
+}
+#else
+static int
+z_erofs_zstd_decompress(const struct z_erofs_decompress_req *rq __unused)
+{
+	return (EOPNOTSUPP);
+}
+#endif
+
+const struct z_erofs_decompressor z_erofs_zstd_decomp = {
+	.config = z_erofs_load_zstd_config,
+	.decompress = z_erofs_zstd_decompress,
+};

dir.c

@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2022, Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/dirent.h>
+#include <sys/libkern.h>
+#include <sys/limits.h>
+#include <sys/malloc.h>
+#include <sys/vnode.h>
+
+#include "internal.h"
+
+static unsigned char
+erofs_ftype_to_dtype(uint8_t ftype)
+{
+	switch (ftype) {
+	case EROFS_FT_REG_FILE:
+		return (DT_REG);
+	case EROFS_FT_DIR:
+		return (DT_DIR);
+	case EROFS_FT_CHRDEV:
+		return (DT_CHR);
+	case EROFS_FT_BLKDEV:
+		return (DT_BLK);
+	case EROFS_FT_FIFO:
+		return (DT_FIFO);
+	case EROFS_FT_SOCK:
+		return (DT_SOCK);
+	case EROFS_FT_SYMLINK:
+		return (DT_LNK);
+	default:
+		return (DT_UNKNOWN);
+	}
+}
+
+/*
+ * Directory block checks.
+ *
+ * A block is trusted only as far as it is used: its header is checked once
+ * per block read, and each entry is bounds-checked when it is looked at.
+ * Nothing here requires names to be sorted, unique, or ordered across
+ * blocks.  That is fsck.erofs's job, and a directory that violates it can
+ * only make a lookup miss; the binary search stays within the block either
+ * way.
+ */
+
+/*
+ * The first name offset is also the size of the dirent array, so it must
+ * leave room for at least one dirent and lie inside the block.
+ */
+int
+erofs_dirblock_ndirents(const char *blk, uint32_t maxsize, uint32_t *ndirentsp)
+{
+	uint32_t first_nameoff;
+
+	if (maxsize < sizeof(struct erofs_dirent))
+		return (EINTEGRITY);
+	first_nameoff = le16toh(((const struct erofs_dirent *)blk)->nameoff);
+	if (first_nameoff < sizeof(struct erofs_dirent) ||
+	    first_nameoff >= maxsize ||
+	    (first_nameoff % sizeof(struct erofs_dirent)) != 0)
+		return (EINTEGRITY);
+	*ndirentsp = first_nameoff / sizeof(struct erofs_dirent);
+	return (0);
+}
+
+/*
+ * Bounds of the name of the idx'th entry: [*nameoffp, *endoffp) is non-empty
+ * and inside the block.  A name ends where the next one starts; the last
+ * one ends at the block end and may be NUL terminated before that.
+ */
+int
+erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents,
+    uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp)
+{
+	const struct erofs_dirent *de;
+	uint32_t endoff, nameoff;
+
+	de = (const struct erofs_dirent *)blk;
+	nameoff = le16toh(de[idx].nameoff);
+	endoff = idx + 1 < ndirents ? le16toh(de[idx + 1].nameoff) : maxsize;
+	if (endoff > maxsize || endoff <= nameoff)
+		return (EINTEGRITY);
+	*nameoffp = nameoff;
+	*endoffp = endoff;
+	return (0);
+}
+
+/*
+ * The name of an entry that readdir is about to hand to userland must be a
+ * plain file name that fits d_name: no NUL inside it (so d_namlen matches
+ * strlen(d_name)), no '/', and at most EROFS_NAME_LEN bytes.  Lookup does
+ * not need any of this.
+ */
+static int
+erofs_dirent_namelen(const char *blk, uint32_t nameoff, uint32_t endoff,
+    bool trailing, size_t *namelenp)
+{
+	size_t namelen, span;
+
+	span = endoff - nameoff;
+	if (trailing) {
+		namelen = strnlen(blk + nameoff, span);
+	} else {
+		namelen = span;
+		if (memchr(blk + nameoff, '\0', span) != NULL)
+			return (EINTEGRITY);
+	}
+	if (namelen == 0 || namelen > EROFS_NAME_LEN)
+		return (EINTEGRITY);
+	for (size_t i = 0; i < namelen; i++) {
+		if (blk[nameoff + i] == '/')
+			return (EINTEGRITY);
+	}
+	*namelenp = namelen;
+	return (0);
+}
+
+/* Extract name, nid, and type of the idx'th directory entry from a block. */
+static int
+erofs_dirent_name(const char *blk, uint32_t maxsize,
+    uint32_t idx, uint32_t ndirents, char *name, size_t namesz __unused,
+    erofs_nid_t *nid, uint8_t *ftype, size_t *namelenp)
+{
+	const struct erofs_dirent *de;
+	uint32_t nameoff, endoff;
+	size_t namelen;
+	int error;
+
+	error = erofs_dirent_span(blk, maxsize, ndirents, idx, &nameoff,
+	    &endoff);
+	if (error != 0)
+		return (error);
+	error = erofs_dirent_namelen(blk, nameoff, endoff,
+	    idx + 1 == ndirents, &namelen);
+	if (error != 0)
+		return (error);
+	/* erofs_dirent_namelen() caps the name at EROFS_NAME_LEN. */
+	EROFS_DEBUG_ASSERT(namelen < namesz,
+	    ("erofs: dirent name of %zu bytes exceeds buffer", namelen));
+	de = (const struct erofs_dirent *)blk;
+	memcpy(name, blk + nameoff, namelen);
+	name[namelen] = '\0';
+	*nid = le64toh(de[idx].nid);
+	*ftype = de[idx].file_type;
+	*namelenp = namelen;
+	return (0);
+}
+
+/* Per-call state for readdir dirent/cookie output. */
+struct erofs_uiodir {
+	struct dirent *dirent;
+	uint64_t *cookies;
+	uint64_t last_cookie;
+	int ncookies;
+	int acookies;
+	int eofflag;
+};
+
+enum erofs_uiodir_result {
+	EROFS_UIODIR_BUFFER_FULL = -1,
+	EROFS_UIODIR_OK = 0,
+};
+
+/* Push a dirent and its cookie to the caller, modelled after UDF. */
+static int
+erofs_uiodir(struct erofs_uiodir *uiodir, int de_size, struct uio *uio,
+    uint64_t cookie)
+{
+	int error;
+
+	/*
+	 * Cookies are block offsets we computed, and grow with the entry
+	 * index whatever the block contains.
+	 */
+	EROFS_DEBUG_ASSERT(cookie > uiodir->last_cookie,
+	    ("erofs: readdir cookie %ju not above %ju", (uintmax_t)cookie,
+	    (uintmax_t)uiodir->last_cookie));
+	if (uio->uio_resid < de_size ||
+	    (uiodir->cookies != NULL &&
+	    uiodir->acookies >= uiodir->ncookies)) {
+		return (EROFS_UIODIR_BUFFER_FULL);
+	}
+	error = uiomove(uiodir->dirent, de_size, uio);
+	if (error != 0)
+		return (error);
+	uiodir->last_cookie = cookie;
+	if (uiodir->cookies != NULL)
+		uiodir->cookies[uiodir->acookies++] = cookie;
+	return (EROFS_UIODIR_OK);
+}
+
+/*
+ * Process directory entries within a single block and output them to uio.
+ * Returns 0 on success (all entries consumed), -1 if uio is full, or a
+ * positive error code on corruption.
+ */
+static int
+erofs_fill_dentries(struct erofs_sb_info *sbi, erofs_nid_t dirnid,
+    struct erofs_uiodir *uiodir, struct uio *uio, struct dirent *d,
+    const char *blk, uint32_t maxsize,
+    uint32_t start_idx, uint32_t ndirents, erofs_off_t block_off,
+    uint64_t *logical_offp)
+{
+	char name[EROFS_NAME_LEN + 1];
+	uint32_t idx;
+	uint64_t curpos, nextoff;
+	erofs_nid_t nid;
+	size_t namelen;
+	uint8_t ftype;
+	int error;
+
+	for (idx = start_idx; idx < ndirents; idx++) {
+		curpos = block_off + idx * sizeof(struct erofs_dirent);
+		nextoff = (idx + 1 < ndirents) ?
+		    (curpos + sizeof(struct erofs_dirent)) :
+		    (block_off + maxsize);
+		error = erofs_dirent_name(blk, maxsize, idx, ndirents, name,
+		    sizeof(name), &nid, &ftype, &namelen);
+		if (error != 0)
+			return (erofs_corrupt(sbi, dirnid,
+			    "invalid name of dirent %u in the block at offset "
+			    "%ju",
+			    idx, (uintmax_t)block_off));
+		/*
+		 * The nid is only reported as d_fileno; lookup checks it
+		 * (erofs_read_inode()) before it is used for anything else.
+		 */
+		bzero(d, sizeof(*d));
+		d->d_fileno = erofs_nid_to_ino64(sbi, nid);
+		d->d_type = erofs_ftype_to_dtype(ftype);
+		d->d_namlen = namelen;
+		d->d_reclen = GENERIC_DIRSIZ(d);
+		d->d_off = nextoff;
+		strlcpy(d->d_name, name, sizeof(d->d_name));
+		error = erofs_uiodir(uiodir, d->d_reclen, uio, d->d_off);
+		if (error != 0)
+			return (error);
+		*logical_offp = nextoff;
+		uio->uio_offset = *logical_offp;
+	}
+	return (0);
+}
+
+/*
+ * Read directory contents and output a FreeBSD dirent stream to uio.
+ *
+ * Key points:
+ * - On-disk entries use their logical file offsets as cookies;
+ * - A dot_omitted directory appends a synthetic "." at i_size, matching
+ *   Linux, so existing on-disk cookies are not shifted;
+ * - The dirent array occupies only the front portion of a block, so after
+ *   scanning all entries offset must jump to maxsize (the block end),
+ *   otherwise the loop would get stuck on the same block;
+ * - Supports a_ncookies / a_cookies for NFS and other callers that need
+ *   resumable iteration.
+ */
+int
+erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag,
+    int *ncookies, uint64_t **cookies)
+{
+	struct erofs_inode *dir;
+	struct erofs_sb_info *sbi;
+	struct erofs_uiodir uiodir;
+	struct dirent d;
+	uint64_t *cookiebuf;
+	size_t cookie_count;
+	char *blk;
+	erofs_off_t block_off;
+	uint64_t logical_off;
+	uint32_t block_pos, blksz, ndirents, start_idx, maxsize;
+	bool sequential;
+	int error;
+
+	dir = VTOE(vp);
+	sbi = MTOE(vp->v_mount);
+	blksz = sbi->block_size;
+	error = 0;
+	cookiebuf = NULL;
+	cookie_count = 0;
+	uiodir.eofflag = 0;
+	uiodir.acookies = 0;
+	uiodir.dirent = &d;
+	uiodir.cookies = NULL;
+	uiodir.ncookies = 0;
+	if (cookies != NULL && ncookies != NULL) {
+		*cookies = NULL;
+		*ncookies = 0;
+		if (uio->uio_resid > 0) {
+			cookie_count = (size_t)uio->uio_resid /
+			    GENERIC_MINDIRSIZ;
+			cookie_count = MIN(cookie_count, (size_t)INT_MAX);
+		}
+		uiodir.ncookies = (int)cookie_count;
+		if (cookie_count != 0)
+			cookiebuf = malloc(sizeof(*cookiebuf) * cookie_count,
+			    M_TEMP, M_WAITOK);
+		uiodir.cookies = cookiebuf;
+	}
+
+	if (uio->uio_offset < 0) {
+		error = EINVAL;
+		goto out;
+	}
+	if (dir->dot_omitted && dir->size == (uint64_t)OFF_MAX) {
+		error = erofs_corrupt(sbi, dir->nid,
+		    "directory size leaves no room for '.'");
+		goto out;
+	}
+
+	logical_off = uio->uio_offset;
+	sequential = logical_off == 0;
+	uiodir.last_cookie = logical_off;
+	uio->uio_offset = logical_off;
+
+	while (logical_off < dir->size) {
+		block_off = rounddown2(logical_off, (uint64_t)blksz);
+		maxsize = MIN((uint64_t)blksz, dir->size - block_off);
+		block_pos = logical_off - block_off;
+		if ((block_pos % sizeof(struct erofs_dirent)) != 0) {
+			block_pos = roundup(block_pos,
+			    sizeof(struct erofs_dirent));
+			logical_off = block_off + block_pos;
+			uio->uio_offset = logical_off;
+		}
+		error = erofs_read_data_readahead(sbi, dir, block_off,
+		    maxsize, sequential, (void **)&blk);
+		if (error != 0)
+			goto out;
+		error = erofs_dirblock_ndirents(blk, maxsize, &ndirents);
+		if (error != 0) {
+			erofs_brelse(blk);
+			error = erofs_corrupt(sbi, dir->nid,
+			    "invalid dirent array in the block at offset %ju",
+			    (uintmax_t)block_off);
+			goto out;
+		}
+		start_idx = block_pos / sizeof(struct erofs_dirent);
+		if (start_idx >= ndirents) {
+			logical_off = block_off + maxsize;
+			uio->uio_offset = logical_off;
+			erofs_brelse(blk);
+			continue;
+		}
+		error = erofs_fill_dentries(sbi, dir->nid, &uiodir, uio, &d,
+		    blk, maxsize, start_idx, ndirents, block_off, &logical_off);
+		erofs_brelse(blk);
+		if (error != 0)
+			goto out;
+	}
+	if (dir->dot_omitted && logical_off == dir->size) {
+		bzero(&d, sizeof(d));
+		d.d_fileno = erofs_nid_to_ino64(sbi, dir->nid);
+		d.d_type = DT_DIR;
+		d.d_namlen = 1;
+		d.d_reclen = GENERIC_DIRSIZ(&d);
+		d.d_off = dir->size + 1;
+		d.d_name[0] = '.';
+		d.d_name[1] = '\0';
+		error = erofs_uiodir(&uiodir, d.d_reclen, uio, d.d_off);
+		if (error != 0)
+			goto out;
+		logical_off++;
+		uio->uio_offset = logical_off;
+	}
+	uiodir.eofflag = 1;
+out:
+	if (error == EROFS_UIODIR_BUFFER_FULL)
+		error = 0;
+	if (eofflag != NULL && error == 0)
+		*eofflag = uiodir.eofflag;
+	if (cookies != NULL && ncookies != NULL) {
+		if (error != 0) {
+			if (cookiebuf != NULL)
+				free(cookiebuf, M_TEMP);
+		} else {
+			*ncookies = uiodir.acookies;
+			*cookies = cookiebuf;
+		}
+	}
+	return (error);
+}

erofs_compat.h

@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/*
+ * What erofs_fs.h, which is kept as it is in Linux, takes from Linux headers:
+ * the integer and little-endian types, a few macros, and the file type
+ * and magic definitions that Linux keeps elsewhere.
+ */
+#ifndef _FS_EROFS_COMPAT_H_
+#define _FS_EROFS_COMPAT_H_
+
+#include <sys/param.h>
+#include <sys/endian.h>
+
+typedef uint8_t		u8;
+typedef uint8_t		__u8;
+typedef uint32_t	__u32;
+typedef uint16_t	__le16;
+typedef uint32_t	__le32;
+typedef uint64_t	__le64;
+
+#define le16_to_cpu(x)		le16toh(x)
+#define cpu_to_le64(x)		htole64(x)
+
+#define BIT(n)			(1UL << (n))
+#define BIT_ULL(n)		(1ULL << (n))
+#define round_up(x, y)		roundup2((x), (y))
+
+/* Fails to compile when the condition is true, as in Linux. */
+#define BUILD_BUG_ON(cond)	((void)sizeof(char[1 - 2 * !!(cond)]))
+
+/* include/uapi/linux/magic.h */
+#define EROFS_SUPER_MAGIC_V1	0xE0F5E1E2
+
+/*
+ * File types in directory entries; Linux uses the generic FT_* values from
+ * include/linux/fs_types.h.
+ */
+#define EROFS_FT_UNKNOWN	0
+#define EROFS_FT_REG_FILE	1
+#define EROFS_FT_DIR		2
+#define EROFS_FT_CHRDEV		3
+#define EROFS_FT_BLKDEV		4
+#define EROFS_FT_FIFO		5
+#define EROFS_FT_SOCK		6
+#define EROFS_FT_SYMLINK	7
+
+#endif

erofs_fs.h

@@ -0,0 +1,467 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * EROFS (Enhanced ROM File System) on-disk format definition
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+#ifndef __EROFS_FS_H
+#define __EROFS_FS_H
+
+/* to allow for x86 boot sectors and other oddities. */
+#define EROFS_SUPER_OFFSET      1024
+
+#define EROFS_FEATURE_COMPAT_SB_CHKSUM			0x00000001
+#define EROFS_FEATURE_COMPAT_MTIME			0x00000002
+#define EROFS_FEATURE_COMPAT_XATTR_FILTER		0x00000004
+#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX	0x00000008
+#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX		0x00000010
+#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS		0x00000020
+
+/*
+ * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should
+ * be incompatible with this kernel version.
+ */
+#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
+#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS	0x00000002
+#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER	0x00000002
+#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE	0x00000004
+#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE	0x00000008
+#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2	0x00000008
+#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING	0x00000010
+#define EROFS_FEATURE_INCOMPAT_FRAGMENTS	0x00000020
+#define EROFS_FEATURE_INCOMPAT_DEDUPE		0x00000020
+#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES	0x00000040
+#define EROFS_FEATURE_INCOMPAT_48BIT		0x00000080
+#define EROFS_FEATURE_INCOMPAT_METABOX		0x00000100
+#define EROFS_ALL_FEATURE_INCOMPAT		\
+	((EROFS_FEATURE_INCOMPAT_METABOX << 1) - 1)
+
+#define EROFS_SB_EXTSLOT_SIZE	16
+
+struct erofs_deviceslot {
+	u8 tag[64];		/* digest(sha256), etc. */
+	__le32 blocks_lo;	/* total blocks count of this device */
+	__le32 uniaddr_lo;	/* unified starting block of this device */
+	__le16 blocks_hi;	/* total blocks count MSB */
+	__le16 uniaddr_hi;	/* unified starting block MSB */
+	u8 reserved[52];
+};
+#define EROFS_DEVT_SLOT_SIZE	sizeof(struct erofs_deviceslot)
+
+/* erofs on-disk super block (currently 144 bytes at maximum) */
+struct erofs_super_block {
+	__le32 magic;           /* file system magic number */
+	__le32 checksum;        /* crc32c to avoid unexpected on-disk overlap */
+	__le32 feature_compat;
+	__u8 blkszbits;         /* filesystem block size in bit shift */
+	__u8 sb_extslots;	/* superblock size = 128 + sb_extslots * 16 */
+	union {
+		__le16 rootnid_2b;	/* nid of root directory */
+		__le16 blocks_hi;	/* (48BIT on) blocks count MSB */
+	} __packed rb;
+	__le64 inos;            /* total valid ino # (== f_files - f_favail) */
+	__le64 epoch;		/* base seconds used for compact inodes */
+	__le32 fixed_nsec;	/* fixed nanoseconds for compact inodes */
+	__le32 blocks_lo;	/* blocks count LSB */
+	__le32 meta_blkaddr;	/* start block address of metadata area */
+	__le32 xattr_blkaddr;	/* start block address of shared xattr area */
+	__u8 uuid[16];          /* 128-bit uuid for volume */
+	__u8 volume_name[16];   /* volume name */
+	__le32 feature_incompat;
+	union {
+		/* bitmap for available compression algorithms */
+		__le16 available_compr_algs;
+		/* customized sliding window size instead of 64k by default */
+		__le16 lz4_max_distance;
+	} __packed u1;
+	__le16 extra_devices;	/* # of devices besides the primary device */
+	__le16 devt_slotoff;	/* startoff = devt_slotoff * devt_slotsize */
+	__u8 dirblkbits;	/* directory block size in bit shift */
+	__u8 xattr_prefix_count;	/* # of long xattr name prefixes */
+	__le32 xattr_prefix_start;	/* start of long xattr prefixes */
+	__le64 packed_nid;	/* nid of the special packed inode */
+	__u8 xattr_filter_reserved; /* reserved for xattr name filter */
+	__u8 ishare_xattr_prefix_id;
+	__u8 reserved[2];
+	__le32 build_time;	/* seconds added to epoch for mkfs time */
+	__le64 rootnid_8b;	/* (48BIT on) nid of root directory */
+	__le64 reserved2;
+	__le64 metabox_nid;     /* (METABOX on) nid of the metabox inode */
+	__le64 reserved3;	/* [align to extslot 1] */
+};
+
+/*
+ * EROFS inode datalayout (i_format in on-disk inode):
+ * 0 - uncompressed flat inode without tail-packing inline data:
+ * 1 - compressed inode with non-compact indexes:
+ * 2 - uncompressed flat inode with tail-packing inline data:
+ * 3 - compressed inode with compact indexes:
+ * 4 - chunk-based inode with (optional) multi-device support:
+ * 5~7 - reserved
+ */
+enum {
+	EROFS_INODE_FLAT_PLAIN			= 0,
+	EROFS_INODE_COMPRESSED_FULL		= 1,
+	EROFS_INODE_FLAT_INLINE			= 2,
+	EROFS_INODE_COMPRESSED_COMPACT		= 3,
+	EROFS_INODE_CHUNK_BASED			= 4,
+	EROFS_INODE_DATALAYOUT_MAX
+};
+
+static inline bool erofs_inode_is_data_compressed(unsigned int datamode)
+{
+	return datamode == EROFS_INODE_COMPRESSED_COMPACT ||
+		datamode == EROFS_INODE_COMPRESSED_FULL;
+}
+
+/* bit definitions of inode i_format */
+#define EROFS_I_VERSION_MASK            0x01
+#define EROFS_I_DATALAYOUT_MASK         0x07
+
+#define EROFS_I_VERSION_BIT	0
+#define EROFS_I_DATALAYOUT_BIT	1
+#define EROFS_I_NLINK_1_BIT	4	/* non-directory compact inodes only */
+#define EROFS_I_DOT_OMITTED_BIT	4	/* (directories) omit the `.` dirent */
+#define EROFS_I_ALL		((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1)
+
+/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */
+#define EROFS_CHUNK_FORMAT_BLKBITS_MASK		0x001F
+/* with chunk indexes or just a 4-byte block array */
+#define EROFS_CHUNK_FORMAT_INDEXES		0x0020
+#define EROFS_CHUNK_FORMAT_48BIT		0x0040
+
+#define EROFS_CHUNK_FORMAT_ALL	((EROFS_CHUNK_FORMAT_48BIT << 1) - 1)
+
+/* 32-byte on-disk inode */
+#define EROFS_INODE_LAYOUT_COMPACT	0
+/* 64-byte on-disk inode */
+#define EROFS_INODE_LAYOUT_EXTENDED	1
+
+struct erofs_inode_chunk_info {
+	__le16 format;		/* chunk blkbits, etc. */
+	__le16 reserved;
+};
+
+union erofs_inode_i_u {
+	__le32 blocks_lo;	/* total blocks count (if compressed inodes) */
+	__le32 startblk_lo;	/* starting block number (if flat inodes) */
+	__le32 rdev;		/* device ID (if special inodes) */
+	struct erofs_inode_chunk_info c;
+};
+
+union erofs_inode_i_nb {
+	__le16 nlink;		/* if EROFS_I_NLINK_1_BIT is unset */
+	__le16 blocks_hi;	/* total blocks count MSB */
+	__le16 startblk_hi;	/* starting block number MSB */
+} __packed;
+
+/* 32-byte reduced form of an ondisk inode */
+struct erofs_inode_compact {
+	__le16 i_format;	/* inode format hints */
+	__le16 i_xattr_icount;
+	__le16 i_mode;
+	union erofs_inode_i_nb i_nb;
+	__le32 i_size;
+	__le32 i_mtime;
+	union erofs_inode_i_u i_u;
+
+	__le32 i_ino;		/* only used for 32-bit stat compatibility */
+	__le16 i_uid;
+	__le16 i_gid;
+	__le32 i_reserved;
+};
+
+/* 64-byte complete form of an ondisk inode */
+struct erofs_inode_extended {
+	__le16 i_format;	/* inode format hints */
+	__le16 i_xattr_icount;
+	__le16 i_mode;
+	union erofs_inode_i_nb i_nb;
+	__le64 i_size;
+	union erofs_inode_i_u i_u;
+
+	__le32 i_ino;		/* only used for 32-bit stat compatibility */
+	__le32 i_uid;
+	__le32 i_gid;
+	__le64 i_mtime;
+	__le32 i_mtime_nsec;
+	__le32 i_nlink;
+	__u8   i_reserved2[16];
+};
+
+/*
+ * inline xattrs (n == i_xattr_icount):
+ * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes
+ *          12 bytes           /                   \
+ *                            /                     \
+ *                           /-----------------------\
+ *                           |  erofs_xattr_entries+ |
+ *                           +-----------------------+
+ * inline xattrs must starts in erofs_xattr_ibody_header,
+ * for read-only fs, no need to introduce h_refcount
+ */
+struct erofs_xattr_ibody_header {
+	__le32 h_name_filter;		/* bit value 1 indicates not-present */
+	__u8   h_shared_count;
+	__u8   h_reserved2[7];
+	__le32 h_shared_xattrs[];       /* shared xattr id array */
+};
+
+/* Name indexes */
+#define EROFS_XATTR_INDEX_USER              1
+#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS  2
+#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3
+#define EROFS_XATTR_INDEX_TRUSTED           4
+#define EROFS_XATTR_INDEX_LUSTRE            5
+#define EROFS_XATTR_INDEX_SECURITY          6
+
+/*
+ * bit 7 of e_name_index is set when it refers to a long xattr name prefix,
+ * while the remained lower bits represent the index of the prefix.
+ */
+#define EROFS_XATTR_LONG_PREFIX		0x80
+#define EROFS_XATTR_LONG_PREFIX_MASK	0x7f
+
+#define EROFS_XATTR_FILTER_BITS		32
+#define EROFS_XATTR_FILTER_DEFAULT	UINT32_MAX
+#define EROFS_XATTR_FILTER_SEED		0x25BBE08F
+
+/* xattr entry (for both inline & shared xattrs) */
+struct erofs_xattr_entry {
+	__u8   e_name_len;      /* length of name */
+	__u8   e_name_index;    /* attribute name index */
+	__le16 e_value_size;    /* size of attribute value */
+	/* followed by e_name and e_value */
+	char   e_name[];        /* attribute name */
+};
+
+/* long xattr name prefix */
+struct erofs_xattr_long_prefix {
+	__u8   base_index;	/* short xattr name prefix index */
+	char   infix[];		/* infix apart from short prefix */
+};
+
+static inline unsigned int erofs_xattr_ibody_size(__le16 i_xattr_icount)
+{
+	if (!i_xattr_icount)
+		return 0;
+
+	/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+	return sizeof(struct erofs_xattr_ibody_header) +
+		sizeof(__u32) * (le16_to_cpu(i_xattr_icount) - 1);
+}
+
+#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry))
+
+static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
+{
+	return EROFS_XATTR_ALIGN(sizeof(struct erofs_xattr_entry) +
+				 e->e_name_len + le16_to_cpu(e->e_value_size));
+}
+
+/* represent a zeroed chunk (hole) */
+#define EROFS_NULL_ADDR			-1
+
+/* 4-byte block address array */
+#define EROFS_BLOCK_MAP_ENTRY_SIZE	sizeof(__le32)
+
+/* 8-byte inode chunk index */
+struct erofs_inode_chunk_index {
+	__le16 startblk_hi;	/* starting block number MSB */
+	__le16 device_id;	/* back-end storage id (with bits masked) */
+	__le32 startblk_lo;	/* starting block number of this chunk */
+};
+
+#define EROFS_DIRENT_NID_METABOX_BIT	63
+#define EROFS_DIRENT_NID_MASK	(BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT) - 1)
+
+/* dirent sorts in alphabet order, thus we can do binary search */
+struct erofs_dirent {
+	__le64 nid;     /* node number */
+	__le16 nameoff; /* start offset of file name */
+	__u8 file_type; /* file type */
+	__u8 reserved;  /* reserved */
+} __packed;
+
+/*
+ * EROFS file types should match generic FT_* types and
+ * it seems no need to add BUILD_BUG_ONs since potential
+ * unmatchness will break other fses as well...
+ */
+
+#define EROFS_NAME_LEN      255
+
+/* maximum supported encoded size of a physical compressed cluster */
+#define Z_EROFS_PCLUSTER_MAX_SIZE	(1024 * 1024)
+
+/* maximum supported decoded size of a physical compressed cluster */
+#define Z_EROFS_PCLUSTER_MAX_DSIZE	(12 * 1024 * 1024)
+
+/* available compression algorithm types (for h_algorithmtype) */
+enum {
+	Z_EROFS_COMPRESSION_LZ4		= 0,
+	Z_EROFS_COMPRESSION_LZMA	= 1,
+	Z_EROFS_COMPRESSION_DEFLATE	= 2,
+	Z_EROFS_COMPRESSION_ZSTD	= 3,
+	Z_EROFS_COMPRESSION_MAX
+};
+#define Z_EROFS_ALL_COMPR_ALGS		((1 << Z_EROFS_COMPRESSION_MAX) - 1)
+
+/* 14 bytes (+ length field = 16 bytes) */
+struct z_erofs_lz4_cfgs {
+	__le16 max_distance;
+	__le16 max_pclusterblks;
+	u8 reserved[10];
+} __packed;
+
+/* 14 bytes (+ length field = 16 bytes) */
+struct z_erofs_lzma_cfgs {
+	__le32 dict_size;
+	__le16 format;
+	u8 reserved[8];
+} __packed;
+
+#define Z_EROFS_LZMA_MAX_DICT_SIZE	(8 * Z_EROFS_PCLUSTER_MAX_SIZE)
+
+/* 6 bytes (+ length field = 8 bytes) */
+struct z_erofs_deflate_cfgs {
+	u8 windowbits;			/* 8..15 for DEFLATE */
+	u8 reserved[5];
+} __packed;
+
+/* 6 bytes (+ length field = 8 bytes) */
+struct z_erofs_zstd_cfgs {
+	u8 format;
+	u8 windowlog;           /* windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN(10) */
+	u8 reserved[4];
+} __packed;
+
+#define Z_EROFS_ZSTD_MAX_DICT_SIZE      Z_EROFS_PCLUSTER_MAX_SIZE
+
+/*
+ * Enable COMPACTED_2B for EROFS_INODE_COMPRESSED_COMPACT inodes:
+ *   4B (disabled) vs 4B+2B+4B (enabled)
+ */
+#define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
+/* Enable extent metadata for EROFS_INODE_COMPRESSED_FULL inodes */
+#define Z_EROFS_ADVISE_EXTENTS			0x0001
+#define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
+#define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
+#define Z_EROFS_ADVISE_INLINE_PCLUSTER		0x0008
+#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER	0x0010
+#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER	0x0020
+/* Indicate the record size for each extent if extent metadata is used */
+#define Z_EROFS_ADVISE_EXTRECSZ_BIT		1
+#define Z_EROFS_ADVISE_EXTRECSZ_MASK		0x3
+
+#define Z_EROFS_FRAGMENT_INODE_BIT              7
+struct z_erofs_map_header {
+	union {
+		/* fragment data offset in the packed inode */
+		__le32  h_fragmentoff;
+		struct {
+			__le16  h_reserved1;
+			/* indicates the encoded size of tailpacking data */
+			__le16  h_idata_size;
+		};
+		__le32 h_extents_lo;	/* extent count LSB */
+	};
+	__le16	h_advise;
+	union {
+		struct {
+			/* algorithm type (bit 0-3: HEAD1; bit 4-7: HEAD2) */
+			__u8	h_algorithmtype;
+			/*
+			 * bit 0-3 : logical cluster bits - blkszbits
+			 * bit 4-6 : reserved
+			 * bit 7   : pack the whole file into packed inode
+			 */
+			__u8	h_clusterbits;
+		} __packed;
+		__le16 h_extents_hi;	/* extent count MSB */
+	} __packed;
+};
+
+enum {
+	Z_EROFS_LCLUSTER_TYPE_PLAIN	= 0,
+	Z_EROFS_LCLUSTER_TYPE_HEAD1	= 1,
+	Z_EROFS_LCLUSTER_TYPE_NONHEAD	= 2,
+	Z_EROFS_LCLUSTER_TYPE_HEAD2	= 3,
+	Z_EROFS_LCLUSTER_TYPE_MAX
+};
+
+#define Z_EROFS_LI_LCLUSTER_TYPE_MASK	(Z_EROFS_LCLUSTER_TYPE_MAX - 1)
+
+/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */
+#define Z_EROFS_LI_PARTIAL_REF		(1 << 15)
+
+/* Set on 1st non-head lcluster to store compressed block counti (in blocks) */
+#define Z_EROFS_LI_D0_CBLKCNT		(1 << 11)
+
+struct z_erofs_lcluster_index {
+	__le16 di_advise;
+	/* where to decompress in the head lcluster */
+	__le16 di_clusterofs;
+
+	union {
+		__le32 blkaddr;		/* for the HEAD lclusters */
+		/*
+		 * [0] - distance to its HEAD lcluster
+		 * [1] - distance to the next HEAD lcluster
+		 */
+		__le16 delta[2];	/* for the NONHEAD lclusters */
+	} di_u;
+};
+
+#define Z_EROFS_MAP_HEADER_END(end)	\
+	(ALIGN(end, 8) + sizeof(struct z_erofs_map_header))
+#define Z_EROFS_FULL_INDEX_START(end)	(Z_EROFS_MAP_HEADER_END(end) + 8)
+
+#define Z_EROFS_EXTENT_PLEN_PARTIAL	BIT(27)
+#define Z_EROFS_EXTENT_PLEN_FMT_BIT	28
+#define Z_EROFS_EXTENT_PLEN_MASK	((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1)
+struct z_erofs_extent {
+	__le32 plen;		/* encoded length */
+	__le32 pstart_lo;	/* physical offset */
+	__le32 pstart_hi;	/* physical offset MSB */
+	__le32 lstart_lo;	/* logical offset */
+	__le32 lstart_hi;	/* logical offset MSB (>= 4GiB inodes) */
+	__u8 reserved[12];	/* for future use */
+};
+
+static inline int z_erofs_extent_recsize(unsigned int advise)
+{
+	return 4 << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) &
+		Z_EROFS_ADVISE_EXTRECSZ_MASK);
+}
+
+/* check the EROFS on-disk layout strictly at compile time */
+static inline void erofs_check_ondisk_layout_definitions(void)
+{
+	const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) {
+		.h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT
+	};
+
+	BUILD_BUG_ON(sizeof(struct erofs_super_block) != 144);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8);
+	BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8);
+	BUILD_BUG_ON(sizeof(struct z_erofs_lcluster_index) != 8);
+	BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+	/* keep in sync between 2 index structures for better extendibility */
+	BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) !=
+		     sizeof(struct z_erofs_lcluster_index));
+	BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128);
+
+	/* exclude old compiler versions like gcc 7.5.0 */
+	BUILD_BUG_ON(__builtin_constant_p(fmh) ?
+		     fmh != cpu_to_le64(1ULL << 63) : 0);
+}
+
+#endif

erofs_vnops.c

@@ -0,0 +1,681 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/extattr.h>
+#include <sys/filio.h>
+#include <sys/fnv_hash.h>
+#include <sys/libkern.h>
+#include <sys/limits.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/stat.h>
+#include <sys/unistd.h>
+#include <sys/vnode.h>
+
+#if WITH_EROFS_POSIX_ACL
+#include <sys/acl.h>
+#endif
+
+#include <vm/vm.h>
+#include <vm/vnode_pager.h>
+
+#include "internal.h"
+#include "xattr.h"
+
+static vop_inactive_t erofs_inactive;
+static vop_reclaim_t erofs_reclaim;
+
+static vop_readdir_t erofs_readdir;
+static vop_readlink_t erofs_readlink;
+
+static vop_open_t erofs_open;
+static vop_read_t erofs_read;
+static vop_bmap_t erofs_bmap;
+static vop_ioctl_t erofs_ioctl;
+
+static vop_getattr_t erofs_getattr;
+static vop_setattr_t erofs_setattr;
+static vop_access_t erofs_access;
+static vop_pathconf_t erofs_pathconf;
+
+static vop_getextattr_t erofs_getextattr;
+static vop_listextattr_t erofs_listextattr;
+static vop_deleteextattr_t erofs_deleteextattr;
+static vop_setextattr_t erofs_setextattr;
+#if WITH_EROFS_POSIX_ACL
+static vop_getacl_t erofs_vop_getacl;
+static vop_aclcheck_t erofs_aclcheck;
+static vop_setacl_t erofs_setacl;
+#endif
+
+/* vop_fhtovp removed in FreeBSD 15.0 */
+static vop_vptofh_t erofs_vptofh;
+
+/*
+ * Access check: data nodes are read-only, but device/FIFO nodes are not
+ * denied writes.
+ */
+static int
+erofs_access(struct vop_access_args *ap)
+{
+	struct vnode *vp;
+	struct erofs_inode *vi;
+#if WITH_EROFS_POSIX_ACL
+	struct acl *acl;
+#endif
+	accmode_t accmode;
+	int error;
+
+	vp = ap->a_vp;
+	vi = VTOE(vp);
+	accmode = ap->a_accmode;
+	if ((accmode & VMODIFY_PERMS) != 0) {
+		switch (vp->v_type) {
+		case VDIR:
+		case VLNK:
+		case VREG:
+			return (EROFS);
+		default:
+			break;
+		}
+	}
+	error = vfs_unixify_accmode(&accmode);
+	if (error != 0)
+		return (error);
+#if WITH_EROFS_POSIX_ACL
+	if ((vp->v_mount->mnt_flag & MNT_ACLS) == 0)
+		return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid,
+		    vi->gid, accmode, ap->a_cred));
+
+	acl = acl_alloc(M_WAITOK);
+	error = erofs_get_acl(vp, ACL_TYPE_ACCESS, acl);
+	if (error == 0)
+		error = vaccess_acl_posix1e(vp->v_type, vi->uid, vi->gid, acl,
+		    accmode, ap->a_cred);
+	acl_free(acl);
+	return (error);
+#else
+	return (vaccess(vp->v_type, vi->mode & ALLPERMS, vi->uid,
+	    vi->gid, accmode, ap->a_cred));
+#endif
+}
+
+/*
+ * Tell the generic pager that EROFS does not provide block-level bmap.
+ *
+ * Returning EOPNOTSUPP prevents the pager from assuming a bufobj/strategy is
+ * available, avoiding "No strategy for buffer" errors.  VM will correctly
+ * fall back to the VOP_READ-based page-in path.  The default async VOP wrapper
+ * must remain in use so synchronous fallback completion still calls iodone.
+ */
+static int
+erofs_bmap(struct vop_bmap_args *ap)
+{
+	if (ap->a_bop != NULL)
+		*ap->a_bop = NULL;
+	if (ap->a_bnp != NULL)
+		*ap->a_bnp = -1;
+	if (ap->a_runp != NULL)
+		*ap->a_runp = 0;
+	if (ap->a_runb != NULL)
+		*ap->a_runb = 0;
+	return (EOPNOTSUPP);
+}
+
+/* No dirty writeback on last ref release, so inactive is a no-op. */
+static int
+erofs_inactive(struct vop_inactive_args *ap __unused)
+{
+	return (0);
+}
+
+static int
+erofs_open(struct vop_open_args *ap)
+{
+	struct vnode *vp;
+	struct erofs_inode *vi;
+	int error;
+
+	vp = ap->a_vp;
+	vi = VTOE(vp);
+	if (VN_ISDEV(vp))
+		return (EOPNOTSUPP);
+	if (vp->v_type == VREG) {
+		error = vnode_create_vobject(vp, vi->size, ap->a_td);
+		return (error);
+	}
+	return (0);
+}
+
+static int
+erofs_getattr(struct vop_getattr_args *ap)
+{
+	struct vnode *vp;
+	struct erofs_inode *vi;
+	struct erofs_sb_info *sbi;
+	struct vattr *vap;
+
+	vp = ap->a_vp;
+	vi = VTOE(vp);
+	sbi = MTOE(vp->v_mount);
+	vap = ap->a_vap;
+	VATTR_NULL(vap);
+	/* The image has no creation time; keep the vop_stat() default. */
+	vap->va_birthtime.tv_sec = -1;
+	vap->va_birthtime.tv_nsec = 0;
+	vap->va_type = vp->v_type;
+	vap->va_mode = vi->mode & ALLPERMS;
+	vap->va_nlink = vi->nlink;
+	vap->va_uid = vi->uid;
+	vap->va_gid = vi->gid;
+	vap->va_fsid = vp->v_mount->mnt_stat.f_fsid.val[0];
+	vap->va_fileid = erofs_nid_to_ino64(sbi, vi->nid);
+	vap->va_size = vi->size;
+	vap->va_blocksize = sbi->block_size;
+	vap->va_atime.tv_sec = vi->mtime;
+	vap->va_mtime.tv_sec = vi->mtime;
+	vap->va_ctime.tv_sec = vi->mtime;
+	vap->va_atime.tv_nsec = vi->mtime_nsec;
+	vap->va_mtime.tv_nsec = vi->mtime_nsec;
+	vap->va_ctime.tv_nsec = vi->mtime_nsec;
+	vap->va_gen = vi->generation;
+	vap->va_flags = 0;
+	vap->va_rdev = VN_ISDEV(vp) ? vi->rdev : NODEV;
+	/* A 48-bit block count of 64 KiB blocks, or about size bytes. */
+	vap->va_bytes = vi->data_blocks << sbi->blkszbits;
+	vap->va_filerev = 0;
+	return (0);
+}
+
+/*
+ * Read-only xattr get entry point.
+ *
+ * Delegates to erofs_getxattr() for two namespaces:
+ * - EXTATTR_NAMESPACE_USER
+ * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes)
+ */
+static int
+erofs_getextattr(struct vop_getextattr_args *ap __unused)
+{
+#if WITH_EROFS_XATTR
+	int error;
+
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER &&
+	    !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER))
+		return (EOPNOTSUPP);
+	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred,
+	    ap->a_td, VREAD);
+	if (error != 0)
+		return (error);
+	if (ap->a_name == NULL || ap->a_name[0] == '\0')
+		return (EINVAL);
+	if (strlen(ap->a_name) > EXTATTR_MAXNAMELEN)
+		return (EINVAL);
+
+	switch (ap->a_attrnamespace) {
+	case EXTATTR_NAMESPACE_USER:
+	case EXTATTR_NAMESPACE_SYSTEM:
+		break;
+	default:
+		return (EOPNOTSUPP);
+	}
+
+	return (erofs_getxattr(ap->a_vp, ap->a_attrnamespace, ap->a_name,
+	    ap->a_uio, ap->a_size));
+#else
+	return (EOPNOTSUPP);
+#endif
+}
+
+/*
+ * Read-only xattr list entry point.
+ *
+ * Delegates to erofs_listxattr() for two namespaces:
+ * - EXTATTR_NAMESPACE_USER
+ * - EXTATTR_NAMESPACE_SYSTEM (trusted.*, security.* and the ACL attributes)
+ */
+static int
+erofs_listextattr(struct vop_listextattr_args *ap __unused)
+{
+#if WITH_EROFS_XATTR
+	int error;
+
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	if (ap->a_attrnamespace == EXTATTR_NAMESPACE_USER &&
+	    !test_opt(&MTOE(ap->a_vp->v_mount)->opt, XATTR_USER))
+		return (EOPNOTSUPP);
+	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred,
+	    ap->a_td, VREAD);
+	if (error != 0)
+		return (error);
+
+	switch (ap->a_attrnamespace) {
+	case EXTATTR_NAMESPACE_USER:
+	case EXTATTR_NAMESPACE_SYSTEM:
+		break;
+	default:
+		return (EOPNOTSUPP);
+	}
+
+	return (erofs_listxattr(ap->a_vp, ap->a_attrnamespace, ap->a_uio,
+	    ap->a_size));
+#else
+	return (EOPNOTSUPP);
+#endif
+}
+
+static int
+erofs_deleteextattr(struct vop_deleteextattr_args *ap __unused)
+{
+#if !WITH_EROFS_XATTR
+	return (EOPNOTSUPP);
+#else
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	return (EROFS);
+#endif
+}
+
+static int
+erofs_setextattr(struct vop_setextattr_args *ap __unused)
+{
+#if !WITH_EROFS_XATTR
+	return (EOPNOTSUPP);
+#else
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	return (EROFS);
+#endif
+}
+
+/*
+ * EROFS is read-only; mutations on regular files/dirs/symlinks are denied, size
+ * changes on special vnodes are treated as no-ops per read-only convention.
+ */
+static int
+erofs_setattr(struct vop_setattr_args *ap)
+{
+	struct vnode *vp;
+	struct vattr *vap;
+
+	vp = ap->a_vp;
+	vap = ap->a_vap;
+	if (vap->va_mode != (mode_t)VNOVAL || vap->va_uid != (uid_t)VNOVAL ||
+	    vap->va_gid != (gid_t)VNOVAL || vap->va_atime.tv_sec != VNOVAL ||
+	    vap->va_atime.tv_nsec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL ||
+	    vap->va_mtime.tv_nsec != VNOVAL || vap->va_flags != VNOVAL)
+		return (EROFS);
+	if (vap->va_size != VNOVAL) {
+		switch (vp->v_type) {
+		case VDIR:
+			return (EISDIR);
+		case VLNK:
+		case VREG:
+			return (EROFS);
+		case VCHR:
+		case VBLK:
+		case VSOCK:
+		case VFIFO:
+		case VNON:
+		case VBAD:
+		case VMARKER:
+			return (0);
+		}
+	}
+	return (0);
+}
+
+static int
+erofs_read(struct vop_read_args *ap)
+{
+	switch (ap->a_vp->v_type) {
+	case VREG:
+		return (erofs_read_file(ap->a_vp, ap->a_uio));
+	case VDIR:
+		return (EISDIR);
+	default:
+		return (EINVAL);
+	}
+}
+
+static int
+erofs_ioctl(struct vop_ioctl_args *ap)
+{
+	struct vnode *vp;
+	bool seek_data;
+	int error;
+
+	switch (ap->a_command) {
+	case FIOSEEKDATA:
+		seek_data = true;
+		break;
+	case FIOSEEKHOLE:
+		seek_data = false;
+		break;
+	default:
+		return (ENOTTY);
+	}
+
+	vp = ap->a_vp;
+	error = vn_lock(vp, LK_SHARED);
+	if (error != 0)
+		return (EBADF);
+	error = erofs_file_llseek(vp, (off_t *)ap->a_data, seek_data);
+	VOP_UNLOCK(vp);
+	return (error);
+}
+
+static int
+erofs_readdir(struct vop_readdir_args *ap)
+{
+	if (ap->a_vp->v_type != VDIR)
+		return (ENOTDIR);
+	return (erofs_readdir_block(ap->a_vp, ap->a_uio, ap->a_eofflag,
+	    ap->a_ncookies, ap->a_cookies));
+}
+
+static int
+erofs_readlink(struct vop_readlink_args *ap)
+{
+	if (ap->a_vp->v_type != VLNK)
+		return (EINVAL);
+	return (erofs_readlink_target(ap->a_vp, ap->a_uio));
+}
+
+static int
+erofs_pathconf(struct vop_pathconf_args *ap)
+{
+	switch (ap->a_name) {
+	case _PC_NAME_MAX:
+		*ap->a_retval = EROFS_NAME_LEN;
+		return (0);
+	case _PC_PATH_MAX:
+		*ap->a_retval = PATH_MAX;
+		return (0);
+	case _PC_FILESIZEBITS:
+		*ap->a_retval = 64;
+		return (0);
+	case _PC_LINK_MAX:
+		*ap->a_retval = INT_MAX;
+		return (0);
+	case _PC_CHOWN_RESTRICTED:
+	case _PC_NO_TRUNC:
+		*ap->a_retval = 1;
+		return (0);
+#if WITH_EROFS_POSIX_ACL
+	case _PC_ACL_EXTENDED:
+		*ap->a_retval =
+		    ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ? 1 : 0;
+		return (0);
+	case _PC_ACL_PATH_MAX:
+		*ap->a_retval =
+		    ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) != 0) ?
+		    ACL_MAX_ENTRIES : 3;
+		return (0);
+	case _PC_ACL_NFS4:
+		*ap->a_retval = 0;
+		return (0);
+#else
+	case _PC_ACL_EXTENDED:
+	case _PC_ACL_NFS4:
+		*ap->a_retval = 0;
+		return (0);
+	case _PC_ACL_PATH_MAX:
+		*ap->a_retval = 3;
+		return (0);
+#endif
+	default:
+		return (vop_stdpathconf(ap));
+	}
+}
+
+#if WITH_EROFS_POSIX_ACL
+static int
+erofs_vop_getacl(struct vop_getacl_args *ap)
+{
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0)
+		return (EOPNOTSUPP);
+	return (erofs_get_acl(ap->a_vp, ap->a_type, ap->a_aclp));
+}
+
+static int
+erofs_aclcheck(struct vop_aclcheck_args *ap)
+{
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	if ((ap->a_vp->v_mount->mnt_flag & MNT_ACLS) == 0)
+		return (EOPNOTSUPP);
+	if (ap->a_aclp == NULL)
+		return (EINVAL);
+	switch (ap->a_type) {
+	case ACL_TYPE_ACCESS:
+		break;
+	case ACL_TYPE_DEFAULT:
+		if (ap->a_vp->v_type != VDIR)
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (acl_posix1e_check(ap->a_aclp));
+}
+
+static int
+erofs_setacl(struct vop_setacl_args *ap)
+{
+	if (VN_ISDEV(ap->a_vp))
+		return (EOPNOTSUPP);
+	return (EROFS);
+}
+#endif
+
+static u_int
+erofs_vfs_hash(erofs_nid_t nid)
+{
+
+	return (fnv_32_buf(&nid, sizeof(nid), FNV1_32_INIT));
+}
+
+static int
+erofs_vfs_hash_cmp(struct vnode *vp, void *pnid)
+{
+	struct erofs_inode *vi;
+
+	vi = VTOE(vp);
+	return (vi == NULL || vi->nid != *(erofs_nid_t *)pnid);
+}
+
+static void
+erofs_fill_vnode(struct erofs_sb_info *sbi, struct vnode *vp,
+    const struct erofs_inode *vi)
+{
+	vp->v_type = vi->vtype;
+	if (vp->v_type == VFIFO)
+		vp->v_op = &erofs_fifoops;
+	else
+		VN_LOCK_ASHARE(vp);
+	if (vi->nid == sbi->root_nid)
+		vp->v_vflag |= VV_ROOT;
+}
+
+int
+erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags,
+    struct vnode **vpp)
+{
+	struct erofs_sb_info *sbi;
+	struct erofs_inode *vi;
+	struct thread *td;
+	struct vnode *vp;
+	u_int hash;
+	bool shared;
+	int error;
+
+	*vpp = NULL;
+	td = curthread;
+	shared = (flags & LK_TYPE_MASK) == LK_SHARED;
+	hash = erofs_vfs_hash(nid);
+	error = vfs_hash_get(mp, hash, flags, td, vpp, erofs_vfs_hash_cmp,
+	    &nid);
+	if (error != 0 || *vpp != NULL)
+		return (error);
+
+	sbi = MTOE(mp);
+	vi = malloc(sizeof(*vi), M_EROFS, M_WAITOK | M_ZERO);
+	error = getnewvnode("erofs", mp, &erofs_vnodeops, &vp);
+	if (error != 0) {
+		free(vi, M_EROFS);
+		*vpp = NULL;
+		return (error);
+	}
+	vp->v_data = vi;
+	vi->nid = nid;
+	lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL);
+	error = insmntque(vp, mp);
+	if (error != 0) {
+		free(vi, M_EROFS);
+		*vpp = NULL;
+		return (error);
+	}
+	error = vfs_hash_insert(vp, hash, flags, td, vpp, erofs_vfs_hash_cmp,
+	    &nid);
+	if (error != 0 || *vpp != NULL)
+		return (error);
+
+	error = erofs_read_inode(sbi, nid, vi);
+	if (error != 0) {
+		*vpp = NULL;
+		vgone(vp);
+		vput(vp);
+		return (error);
+	}
+	erofs_xattr_cache_init(vi);
+	erofs_fill_vnode(sbi, vp, vi);
+	vn_set_state(vp, VSTATE_CONSTRUCTED);
+	if (shared)
+		VOP_LOCK(vp, LK_DOWNGRADE);
+	*vpp = vp;
+	return (0);
+}
+
+int
+erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags,
+    struct vnode **vpp)
+{
+	return (erofs_vget_raw(mp, *(erofs_nid_t *)arg, flags, vpp));
+}
+
+int
+erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp)
+{
+	struct erofs_sb_info *sbi;
+	erofs_nid_t nid;
+	int error;
+
+	*vpp = NULL;
+	sbi = MTOE(mp);
+	error = erofs_ino_to_nid(sbi, ino, &nid);
+	if (error != 0)
+		return (error);
+	if (!erofs_nid_is_valid(sbi, nid))
+		return (ESTALE);
+	return (erofs_vget_raw(mp, nid, flags, vpp));
+}
+
+static int
+erofs_reclaim(struct vop_reclaim_args *ap)
+{
+	struct vnode *vp;
+	struct erofs_inode *vi;
+
+	vp = ap->a_vp;
+	vi = VTOE(vp);
+	if (vi != NULL) {
+		vfs_hash_remove(vp);
+		erofs_xattr_cache_fini(MTOE(vp->v_mount), vi);
+		free(vi, M_EROFS);
+		vp->v_data = NULL;
+	}
+	return (0);
+}
+
+/* Vnode pointer to persistent EROFS file handle. */
+static int
+erofs_vptofh(struct vop_vptofh_args *ap)
+{
+	struct erofs_fid efid;
+	struct erofs_inode *vi;
+
+	vi = VTOE(ap->a_vp);
+	bzero(&efid, sizeof(efid));
+	efid.len = sizeof(efid);
+	efid.nid_hi = vi->nid >> 32;
+	efid.nid_lo = vi->nid;
+	efid.gen = vi->generation;
+	memcpy(ap->a_fhp, &efid, sizeof(efid));
+	return (0);
+}
+
+struct vop_vector erofs_vnodeops = {
+	.vop_default = &default_vnodeops,
+	.vop_inactive = erofs_inactive,
+	.vop_reclaim = erofs_reclaim,
+
+	.vop_lookup = vfs_cache_lookup,
+	.vop_cachedlookup = erofs_lookup,
+	.vop_readdir = erofs_readdir,
+	.vop_readlink = erofs_readlink,
+
+	.vop_open = erofs_open,
+	.vop_read = erofs_read,
+	.vop_bmap = erofs_bmap,
+	.vop_ioctl = erofs_ioctl,
+	.vop_getpages = vnode_pager_local_getpages,
+
+	.vop_getattr = erofs_getattr,
+	.vop_setattr = erofs_setattr,
+	.vop_access = erofs_access,
+	.vop_pathconf = erofs_pathconf,
+
+	.vop_getextattr = erofs_getextattr,
+	.vop_listextattr = erofs_listextattr,
+	.vop_deleteextattr = erofs_deleteextattr,
+	.vop_setextattr = erofs_setextattr,
+#if WITH_EROFS_POSIX_ACL
+	.vop_getacl = erofs_vop_getacl,
+	.vop_aclcheck = erofs_aclcheck,
+	.vop_setacl = erofs_setacl,
+#endif
+
+	.vop_vptofh = erofs_vptofh,
+};
+VFS_VOP_VECTOR_REGISTER(erofs_vnodeops);
+
+struct vop_vector erofs_fifoops = {
+	.vop_default = &fifo_specops,
+	.vop_access = erofs_access,
+#if WITH_EROFS_POSIX_ACL
+	.vop_aclcheck = erofs_aclcheck,
+#endif
+	.vop_deleteextattr = erofs_deleteextattr,
+#if WITH_EROFS_POSIX_ACL
+	.vop_getacl = erofs_vop_getacl,
+#endif
+	.vop_getextattr = erofs_getextattr,
+	.vop_getattr = erofs_getattr,
+	.vop_listextattr = erofs_listextattr,
+	.vop_pathconf = erofs_pathconf,
+	.vop_reclaim = erofs_reclaim,
+#if WITH_EROFS_POSIX_ACL
+	.vop_setacl = erofs_setacl,
+#endif
+	.vop_setattr = erofs_setattr,
+	.vop_setextattr = erofs_setextattr,
+	.vop_vptofh = erofs_vptofh,
+};
+VFS_VOP_VECTOR_REGISTER(erofs_fifoops);

fileio.c

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2024, Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/_maxphys.h>
+#include <sys/proc.h>
+
+#include "internal.h"
+
+#if WITH_EROFS_BACKED_BY_FILE
+static int
+erofs_read_file_source_vnrdwr(struct erofs_device_info *dif, erofs_off_t off,
+    size_t len, void *out, int ioflag)
+{
+	size_t done, want;
+	ssize_t resid;
+	int error;
+
+	/* The credential is taken when the file source is opened. */
+	EROFS_DEBUG_ASSERT(dif->source_cred != NULL,
+	    ("erofs: file source without a credential"));
+	done = 0;
+	while (done < len) {
+		want = MIN(len - done, (size_t)MAXPHYS);
+		/* erofs_bread_device() keeps off + len within INT64_MAX + 1. */
+		error = vn_rdwr(UIO_READ, dif->devvp, (char *)out + done,
+		    (int)want, (off_t)(off + done), UIO_SYSSPACE, ioflag,
+		    dif->source_cred, NOCRED, &resid, curthread);
+		if (error != 0)
+			return (error);
+		if (resid != 0)
+			return (EIO);
+		done += want;
+	}
+	return (0);
+}
+
+static int
+erofs_read_file_source_direct(struct erofs_device_info *dif, erofs_off_t off,
+    size_t len, void *out)
+{
+	return (erofs_read_file_source_vnrdwr(dif, off, len, out, IO_DIRECT));
+}
+
+int
+erofs_read_file_source(struct erofs_sb_info *sbi,
+    struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out)
+{
+	if (test_opt(&sbi->opt, DIRECT_IO))
+		return (erofs_read_file_source_direct(dif, off, len, out));
+	return (erofs_read_file_source_vnrdwr(dif, off, len, out, 0));
+}
+#endif

inode.c

@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/endian.h>
+#include <sys/fnv_hash.h>
+#include <sys/libkern.h>
+#include <sys/limits.h>
+#include <sys/stat.h>
+#include <sys/vnode.h>
+
+#include "trace.h"
+#include "xattr.h"
+
+static uint64_t
+erofs_addrmask(const struct erofs_sb_info *sbi)
+{
+	if (erofs_sb_has_48bit(sbi))
+		return ((1ULL << 48) - 1);
+	return (UINT32_MAX);
+}
+
+static dev_t
+erofs_decode_dev(uint32_t dev)
+{
+	unsigned int major, minor;
+
+	major = (dev & 0xfff00) >> 8;
+	minor = (dev & 0xff) | ((dev >> 12) & 0xfff00);
+	return (makedev(major, minor));
+}
+
+static uint32_t
+erofs_inode_generation(const struct erofs_sb_info *sbi, erofs_nid_t nid,
+    const void *inode, size_t inode_size)
+{
+	uint8_t encoded_nid[sizeof(nid)];
+	uint32_t generation;
+
+	le64enc(encoded_nid, nid);
+	generation = fnv_32_buf(encoded_nid, sizeof(encoded_nid),
+	    sbi->generation_seed);
+	generation = fnv_32_buf(inode, inode_size, generation);
+	return (generation != 0 ? generation : 1);
+}
+
+static int
+erofs_set_timestamp(struct erofs_inode *vi, int64_t seconds,
+    uint32_t nanoseconds)
+{
+	/* Keep va_mtime a valid timespec; mkfs.erofs never writes this. */
+	if (nanoseconds >= 1000000000)
+		return (EINTEGRITY);
+	vi->mtime = seconds;
+	vi->mtime_nsec = nanoseconds;
+	return (0);
+}
+
+static void
+erofs_set_data_blocks(const struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    uint64_t compressed_blocks)
+{
+	if (erofs_inode_is_data_compressed(vi->datalayout))
+		vi->data_blocks = compressed_blocks;
+	else if (vi->size == 0)
+		vi->data_blocks = 0;
+	else
+		/* size is at most OFF_MAX (checked by the caller). */
+		vi->data_blocks = roundup2(vi->size,
+		    (uint64_t)sbi->block_size) >> sbi->blkszbits;
+}
+
+static int
+erofs_validate_inline_data(const struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi)
+{
+	uint64_t image_size, inline_end, inline_off, inline_size, tail_start;
+
+	if (vi->datalayout != EROFS_INODE_FLAT_INLINE || vi->size == 0)
+		return (0);
+	tail_start = roundup2(vi->size, (uint64_t)sbi->block_size) -
+	    sbi->block_size;
+	inline_size = vi->size - tail_start;
+	if (__builtin_add_overflow(vi->inode_off, vi->inode_isize,
+	    &inline_off) ||
+	    __builtin_add_overflow(inline_off, vi->xattr_isize, &inline_off) ||
+	    __builtin_add_overflow(inline_off, inline_size, &inline_end))
+		return (erofs_corrupt(sbi, vi->nid,
+		    "inline tail offset overflows"));
+	if ((inline_off & (sbi->block_size - 1)) + inline_size >
+	    sbi->block_size)
+		return (erofs_corrupt(sbi, vi->nid,
+		    "inline tail of %ju bytes crosses its block",
+		    (uintmax_t)inline_size));
+	if (erofs_nid_in_metabox(vi->nid)) {
+		if (sbi->metabox_en == NULL ||
+		    inline_end > sbi->metabox_en->size)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "inline tail ends at %ju, beyond the metabox",
+			    (uintmax_t)inline_end));
+		return (0);
+	}
+	/* The image fits its provider; see erofs_nid_is_valid(). */
+	image_size = sbi->blocks << sbi->blkszbits;
+	if (inline_end > image_size)
+		return (erofs_corrupt(sbi, vi->nid,
+		    "inline tail ends at %ju, beyond the image",
+		    (uintmax_t)inline_end));
+	return (0);
+}
+
+/*
+ * Convert a logical nid to its inode-table byte offset.  Normal NIDs are
+ * relative to the primary metadata area.  For metabox NIDs, bit 63 selects
+ * the metabox backing inode and the remaining bits are relative to its data.
+ * EROFS_NULL_ADDR is returned when the address cannot be represented.
+ */
+static erofs_off_t
+erofs_iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
+{
+	erofs_off_t meta_offset;
+	erofs_nid_t nid_lo;
+	erofs_off_t result;
+	bool in_metabox;
+
+	in_metabox = erofs_nid_in_metabox(nid);
+	if (in_metabox && !erofs_sb_has_metabox(sbi))
+		return (EROFS_NULL_ADDR);
+	nid_lo = nid & EROFS_DIRENT_NID_MASK;
+	if (nid_lo > (UINT64_MAX >> 5))
+		return (EROFS_NULL_ADDR);
+	result = nid_lo << 5;
+	if (in_metabox)
+		return (result);
+
+	meta_offset = (uint64_t)sbi->meta_blkaddr << sbi->blkszbits;
+	if (result > UINT64_MAX - meta_offset)
+		return (EROFS_NULL_ADDR);
+
+	return (meta_offset + result);
+}
+
+/*
+ * Check that a NID can address at least one compact inode slot without
+ * crossing the declared primary image or metabox backing-file boundary.
+ */
+bool
+erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid)
+{
+	erofs_off_t image_size, off;
+
+	off = erofs_iloc(sbi, nid);
+	if (off == EROFS_NULL_ADDR)
+		return (false);
+	if (erofs_nid_in_metabox(nid)) {
+		if (sbi->metabox_en == NULL || off > sbi->metabox_en->size)
+			return (false);
+		return (sizeof(struct erofs_inode_compact) <=
+		    sbi->metabox_en->size - off);
+	}
+	/*
+	 * The image fits its provider (erofs_validate_device_size()), and a
+	 * block count has at most 48 bits, so neither the shift nor the
+	 * provider size needs a check here.
+	 */
+	image_size = sbi->blocks << sbi->blkszbits;
+	return (off <= image_size &&
+	    sizeof(struct erofs_inode_compact) <= image_size - off);
+}
+
+int
+erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,
+    struct erofs_inode *vi)
+{
+	struct erofs_inode_compact *dic;
+	struct erofs_inode_extended *die;
+	struct erofs_inode_chunk_info chunk_info;
+	union erofs_inode_i_nb inode_nb;
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	uint64_t addrmask;
+	int64_t mtime;
+	erofs_off_t off;
+	erofs_blk_t startblk;
+	uint64_t compressed_blocks;
+	uint32_t raw_rdev, startblk_lo;
+	uint16_t ifmt, startblk_hi;
+	int err;
+
+	if (!erofs_nid_is_valid(sbi, nid))
+		return (erofs_corrupt(sbi, nid, "invalid nid"));
+	off = erofs_iloc(sbi, nid);
+	EROFS_TRACE_FILL_INODE(nid, off >> sbi->blkszbits,
+	    off & (sbi->block_size - 1));
+	err = erofs_read_metadata_range(sbi, nid, off,
+	    sizeof(struct erofs_inode_compact), &buf);
+	if (err != 0)
+		return (err);
+
+	bzero(&vi->size, sizeof(*vi) - offsetof(struct erofs_inode, size));
+	vi->nid = nid;
+	vi->inode_off = off;
+	ifmt = le16dec(buf.data);
+	if ((ifmt & ~EROFS_I_ALL) != 0) {
+		erofs_put_metabuf(&buf);
+		erofs_err(sbi, "unsupported inode format %#x @ nid %ju", ifmt,
+		    (uintmax_t)nid);
+		return (EOPNOTSUPP);
+	}
+	vi->datalayout = erofs_inode_datalayout(ifmt);
+	if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) {
+		erofs_put_metabuf(&buf);
+		erofs_err(sbi, "unsupported data layout %u @ nid %ju",
+		    vi->datalayout, (uintmax_t)nid);
+		return (EOPNOTSUPP);
+	}
+	vi->compact_inode = (erofs_inode_version(ifmt) == 0);
+	if (!vi->compact_inode) {
+		erofs_put_metabuf(&buf);
+		err = erofs_read_metadata_range(sbi, nid, off,
+		    sizeof(struct erofs_inode_extended), &buf);
+		if (err != 0)
+			return (err);
+	}
+	addrmask = erofs_addrmask(sbi);
+	startblk = EROFS_NULL_ADDR;
+	startblk_lo = 0;
+	startblk_hi = 0;
+	compressed_blocks = 0;
+	raw_rdev = 0;
+	bzero(&inode_nb, sizeof(inode_nb));
+	dic = buf.data;
+	if (vi->compact_inode) {
+		vi->inode_isize = sizeof(struct erofs_inode_compact);
+		vi->generation = erofs_inode_generation(sbi, nid, buf.data,
+		    vi->inode_isize);
+		vi->mode = le16toh(dic->i_mode);
+		vi->size = le32toh(dic->i_size);
+		vi->uid = le16toh(dic->i_uid);
+		vi->gid = le16toh(dic->i_gid);
+		vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount);
+		if (__builtin_add_overflow(sbi->epoch,
+		    (int64_t)le32toh(dic->i_mtime), &mtime)) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid,
+			    "mtime overflows the superblock epoch"));
+		}
+		err = erofs_set_timestamp(vi, mtime, sbi->fixed_nsec);
+		if (err != 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid,
+			    "invalid mtime nsec %u", sbi->fixed_nsec));
+		}
+		startblk_lo = le32toh(dic->i_u.startblk_lo);
+		compressed_blocks = le32toh(dic->i_u.blocks_lo);
+		raw_rdev = le32toh(dic->i_u.rdev);
+		if (!S_ISDIR(vi->mode) &&
+		    ((ifmt >> EROFS_I_NLINK_1_BIT) & 0x1) != 0) {
+			vi->nlink = 1;
+			inode_nb = dic->i_nb;
+		} else {
+			vi->nlink = le16toh(dic->i_nb.nlink);
+			addrmask = UINT32_MAX;
+		}
+	} else {
+		die = buf.data;
+		vi->inode_isize = sizeof(struct erofs_inode_extended);
+		vi->generation = erofs_inode_generation(sbi, nid, buf.data,
+		    vi->inode_isize);
+		vi->mode = le16toh(die->i_mode);
+		vi->size = le64toh(die->i_size);
+		vi->uid = le32toh(die->i_uid);
+		vi->gid = le32toh(die->i_gid);
+		vi->nlink = le32toh(die->i_nlink);
+		inode_nb = die->i_nb;
+		vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount);
+		err = erofs_set_timestamp(vi,
+		    (int64_t)le64toh(die->i_mtime),
+		    le32toh(die->i_mtime_nsec));
+		if (err != 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid, "invalid mtime nsec %u",
+			    le32toh(die->i_mtime_nsec)));
+		}
+		startblk_lo = le32toh(die->i_u.startblk_lo);
+		compressed_blocks = le32toh(die->i_u.blocks_lo);
+		raw_rdev = le32toh(die->i_u.rdev);
+	}
+	startblk_hi = le16toh(inode_nb.startblk_hi);
+	compressed_blocks |= (uint64_t)le16toh(inode_nb.blocks_hi) << 32;
+	if (vi->size > (uint64_t)OFF_MAX) {
+		erofs_put_metabuf(&buf);
+		return (erofs_corrupt(sbi, nid, "size %ju is too large",
+		    (uintmax_t)vi->size));
+	}
+
+	vi->vtype = IFTOVT(vi->mode);
+	/* A mode with an unknown file type cannot be given a vnode type. */
+	if (vi->mode != 0 && vi->vtype == VNON) {
+		erofs_put_metabuf(&buf);
+		return (erofs_corrupt(sbi, nid, "invalid mode %#o", vi->mode));
+	}
+	if (vi->vtype == VLNK) {
+		if (vi->size == 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid, "empty symlink"));
+		}
+	}
+	vi->dot_omitted = (vi->vtype == VDIR) &&
+	    (((ifmt >> EROFS_I_DOT_OMITTED_BIT) & 0x1) != 0);
+
+	if (erofs_inode_is_data_compressed(vi->datalayout)) {
+#if WITH_EROFS_ZIP
+		err = z_erofs_fill_inode(sbi, vi);
+#else
+		err = EOPNOTSUPP;
+#endif
+		if (err != 0) {
+			erofs_put_metabuf(&buf);
+			return (err);
+		}
+	} else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) {
+		if (vi->compact_inode)
+			chunk_info = dic->i_u.c;
+		else
+			chunk_info = die->i_u.c;
+		if (le16toh(chunk_info.reserved) != 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid,
+			    "reserved chunk field %#x is set",
+			    le16toh(chunk_info.reserved)));
+		}
+		vi->chunkformat = le16toh(chunk_info.format);
+		if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) {
+			erofs_put_metabuf(&buf);
+			erofs_err(sbi, "unsupported chunk format %#x @ nid %ju",
+			    vi->chunkformat, (uintmax_t)nid);
+			return (EOPNOTSUPP);
+		}
+		if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 &&
+		    (vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) == 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, nid,
+			    "chunk format %#x: 48-bit without chunk indexes",
+			    vi->chunkformat));
+		}
+		vi->chunkbits = sbi->blkszbits +
+		    (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK);
+	} else if (vi->datalayout != EROFS_INODE_FLAT_PLAIN &&
+	    vi->datalayout != EROFS_INODE_FLAT_INLINE) {
+		erofs_put_metabuf(&buf);
+		erofs_err(sbi, "unsupported data layout %u @ nid %ju",
+		    vi->datalayout, (uintmax_t)nid);
+		return (EOPNOTSUPP);
+	}
+
+	switch (vi->vtype) {
+	case VREG:
+	case VDIR:
+	case VLNK:
+		if (vi->datalayout == EROFS_INODE_CHUNK_BASED) {
+			vi->startblk = EROFS_NULL_ADDR;
+			vi->rdev = NODEV;
+			break;
+		}
+		startblk = startblk_lo | ((uint64_t)startblk_hi << 32);
+		if (vi->datalayout == EROFS_INODE_FLAT_PLAIN &&
+		    ((startblk ^ EROFS_NULL_ADDR) & addrmask) == 0)
+			startblk = EROFS_NULL_ADDR;
+		vi->startblk = startblk;
+		vi->rdev = NODEV;
+		break;
+	case VCHR:
+	case VBLK:
+		vi->startblk = EROFS_NULL_ADDR;
+		vi->rdev = erofs_decode_dev(raw_rdev);
+		break;
+	case VFIFO:
+	case VSOCK:
+		vi->startblk = EROFS_NULL_ADDR;
+		vi->rdev = NODEV;
+		break;
+	default:
+		erofs_put_metabuf(&buf);
+		return (erofs_corrupt(sbi, nid, "unknown file type, mode %#o",
+		    vi->mode));
+	}
+	erofs_set_data_blocks(sbi, vi, compressed_blocks);
+	err = erofs_validate_inline_data(sbi, vi);
+	if (err != 0) {
+		erofs_put_metabuf(&buf);
+		return (err);
+	}
+
+	erofs_put_metabuf(&buf);
+	return (0);
+}

internal.h

@@ -0,0 +1,721 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+
+#ifndef _FS_EROFS_INTERNAL_H_
+#define _FS_EROFS_INTERNAL_H_
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/condvar.h>
+#include <sys/malloc.h>
+#include <sys/mount.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/vnode.h>
+
+#include "erofs_compat.h"
+#include "erofs_fs.h"
+
+/*
+ * Linux checks the sizes of the on-disk structures in
+ * erofs_check_ondisk_layout_definitions(); these also check the offsets.
+ */
+_Static_assert(sizeof(struct erofs_deviceslot) == 128,
+	"EROFS device slot ABI size");
+_Static_assert(sizeof(struct erofs_super_block) == 144,
+	"EROFS super block ABI size");
+_Static_assert(sizeof(struct erofs_inode_compact) == 32,
+	"EROFS compact inode ABI size");
+_Static_assert(sizeof(struct erofs_inode_extended) == 64,
+	"EROFS extended inode ABI size");
+_Static_assert(sizeof(struct erofs_xattr_ibody_header) == 12,
+	"EROFS xattr ibody header ABI size");
+_Static_assert(sizeof(struct erofs_xattr_entry) == 4,
+	"EROFS xattr entry ABI size");
+_Static_assert(sizeof(struct erofs_inode_chunk_info) == 4,
+	"EROFS chunk info ABI size");
+_Static_assert(sizeof(struct erofs_inode_chunk_index) == 8,
+	"EROFS chunk index ABI size");
+_Static_assert(sizeof(struct z_erofs_map_header) == 8,
+	"EROFS zmap header ABI size");
+_Static_assert(sizeof(struct z_erofs_lcluster_index) == 8,
+	"EROFS lcluster index ABI size");
+_Static_assert(sizeof(struct z_erofs_extent) == 32,
+	"EROFS compression extent ABI size");
+_Static_assert(sizeof(struct erofs_dirent) == 12,
+	"EROFS dirent ABI size");
+_Static_assert(sizeof(struct erofs_deviceslot) == EROFS_DEVT_SLOT_SIZE,
+	"EROFS device slot ABI macro");
+_Static_assert(
+    __builtin_offsetof(struct erofs_super_block, extra_devices) == 86,
+	"EROFS extra device count ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_super_block, devt_slotoff) == 88,
+	"EROFS device table slot offset ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_super_block, rootnid_8b) == 112,
+	"EROFS 48-bit root nid ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_super_block, metabox_nid) == 128,
+	"EROFS metabox nid ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_inode_compact, i_u) == 16,
+	"EROFS compact inode union ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_inode_extended, i_u) == 16,
+	"EROFS extended inode union ABI offset");
+_Static_assert(
+    __builtin_offsetof(struct erofs_inode_extended, i_reserved2) == 48,
+	"EROFS extended inode reserved ABI offset");
+_Static_assert(
+    __builtin_offsetof(struct erofs_inode_chunk_index, device_id) == 2,
+	"EROFS chunk device id ABI offset");
+_Static_assert(
+    __builtin_offsetof(struct erofs_inode_chunk_index, startblk_lo) == 4,
+	"EROFS chunk start block ABI offset");
+_Static_assert(__builtin_offsetof(struct z_erofs_map_header, h_advise) == 4,
+	"EROFS zmap advise ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_lo) == 64,
+	"EROFS device blocks ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_lo) == 68,
+	"EROFS device unified address ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_deviceslot, blocks_hi) == 72,
+	"EROFS device blocks high ABI offset");
+_Static_assert(__builtin_offsetof(struct erofs_deviceslot, uniaddr_hi) == 74,
+	"EROFS device unified address high ABI offset");
+
+MALLOC_DECLARE(M_EROFS);
+
+#ifndef WITH_EROFS_DEBUG
+#define WITH_EROFS_DEBUG 1
+#endif
+
+#if WITH_EROFS_DEBUG
+#define EROFS_DEBUG_ASSERT(exp, msg) KASSERT((exp), msg)
+#else
+#define EROFS_DEBUG_ASSERT(exp, msg) do { } while (0)
+#endif
+
+/*
+ * Diagnostics.  erofs_corrupt() goes where bad on-disk data is first
+ * noticed, not in the callers that pass the error up.  It logs one
+ * rate-limited line (mount point, function, what was wrong, inode), fires
+ * the erofs:::corrupt DTrace probe (arguments: nid, source line, error)
+ * and returns EINTEGRITY, so a check reads
+ *
+ *	return (erofs_corrupt(sbi, nid, "bad foo %u", foo));
+ *
+ * erofs_err() is the same without the probe, for errors that are not
+ * corruption, such as an unsupported feature.
+ */
+struct erofs_sb_info;
+/* The error is not about an inode. */
+#define EROFS_NID_NONE	((erofs_nid_t)-1)
+void erofs_log_err(const struct erofs_sb_info *sbi, const char *func,
+    const char *fmt, ...) __printflike(3, 4);
+int erofs_log_corrupt(const struct erofs_sb_info *sbi, uint64_t nid,
+    const char *func, int line, const char *fmt, ...) __printflike(5, 6);
+#define erofs_err(sbi, fmt, ...) \
+	erofs_log_err((sbi), __func__, fmt, ##__VA_ARGS__)
+#define erofs_corrupt(sbi, nid, fmt, ...) \
+	erofs_log_corrupt((sbi), (nid), __func__, __LINE__, fmt, ##__VA_ARGS__)
+
+struct cdev;
+struct g_consumer;
+struct erofs_device_info;
+
+typedef uint64_t erofs_nid_t;
+typedef uint64_t erofs_off_t;
+typedef uint64_t erofs_blk_t;
+
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__)
+#error EROFS requires compiler byte-order definitions
+#elif __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+#error EROFS requires a little-endian FreeBSD kernel ABI
+#endif
+/*
+ * The module is LP64-only (see also the architecture list in the Makefile),
+ * so size_t, off_t, ino_t, time_t and erofs_blk_t are all 64 bits wide.
+ * Code below relies on this: do not add 32-bit overflow guards.
+ */
+_Static_assert(sizeof(uintptr_t) == sizeof(uint64_t),
+    "EROFS requires a 64-bit pointer ABI");
+_Static_assert(sizeof(ino_t) >= sizeof(uint64_t),
+    "EROFS requires a 64-bit inode number ABI");
+_Static_assert(sizeof(off_t) >= sizeof(uint64_t),
+    "EROFS requires a 64-bit file offset ABI");
+_Static_assert(sizeof(time_t) >= sizeof(int64_t),
+    "EROFS requires a 64-bit timestamp ABI");
+_Static_assert(sizeof(u_long) == sizeof(uint64_t),
+    "EROFS requires 64-bit FreeBSD atomic long operations");
+/*
+ * The block count of the image and of each device has at most 48 bits, so
+ * (blocks << blkszbits) fits in 64 bits.  This does not hold for
+ * flatdev_blocks, a sum of two such counts: erofs_bread_device() checks it.
+ */
+_Static_assert(PAGE_SHIFT <= 16, "EROFS block sizes are at most 64 KiB");
+
+#define EROFS_MOUNT_POSIX_ACL  0x00000020
+#define EROFS_MOUNT_XATTR_USER 0x00000080
+#define EROFS_MOUNT_DIRECT_IO  0x00000100
+
+#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
+#define set_opt(opt, option)   ((opt)->mount_opt |= EROFS_MOUNT_##option)
+#define test_opt(opt, option)  ((opt)->mount_opt & EROFS_MOUNT_##option)
+
+struct erofs_sb_info_opts {
+	unsigned int mount_opt;
+	uint8_t cache_strategy;
+	uint8_t worker_queue_priority;
+};
+
+enum erofs_sync_decompress {
+	EROFS_SYNC_DECOMPRESS_AUTO,
+	EROFS_SYNC_DECOMPRESS_FORCE_ON,
+	EROFS_SYNC_DECOMPRESS_FORCE_OFF,
+};
+
+enum erofs_zip_cache_strategy {
+	EROFS_ZIP_CACHE_DISABLED,
+	EROFS_ZIP_CACHE_READAHEAD,
+	EROFS_ZIP_CACHE_READAROUND,
+};
+
+struct erofs_device_handle;
+
+enum erofs_source_kind {
+	EROFS_SOURCE_NONE,
+	EROFS_SOURCE_BLOCK,
+	EROFS_SOURCE_FILE,
+};
+
+struct erofs_device_info {
+	struct vnode *devvp;
+	struct cdev *dev;
+	struct g_consumer *cp;
+	struct erofs_device_handle *handle;
+	struct ucred *source_cred;
+	enum erofs_source_kind source_kind;
+	uint64_t mediasize;
+	uint32_t sectorsize;
+	uint64_t fsoff;
+
+	erofs_blk_t blocks;
+	erofs_blk_t uniaddr;
+};
+
+static inline bool
+erofs_source_is_open(const struct erofs_device_info *dif)
+{
+	return (dif != NULL && dif->devvp != NULL && dif->handle != NULL &&
+	    (dif->source_kind == EROFS_SOURCE_BLOCK ||
+	    dif->source_kind == EROFS_SOURCE_FILE));
+}
+
+static inline bool
+erofs_source_is_file(const struct erofs_device_info *dif)
+{
+	return (erofs_source_is_open(dif) &&
+	    dif->source_kind == EROFS_SOURCE_FILE);
+}
+
+struct erofs_xattr_prefix_item {
+	uint8_t base_index;
+	uint8_t infix_len;
+	char *infix;
+};
+
+struct erofs_map_blocks {
+	erofs_off_t m_pa, m_la;
+	uint64_t m_plen, m_llen;
+
+	unsigned short m_deviceid;
+	uint8_t m_algorithmformat;
+	unsigned int m_flags;
+};
+
+enum erofs_zextent_cache_state {
+	EROFS_ZCACHE_EMPTY,
+	EROFS_ZCACHE_INFLIGHT,
+	EROFS_ZCACHE_READY,
+	EROFS_ZCACHE_FAILED,
+};
+
+#define EROFS_ZCACHE_MAX_ENTRIES 8
+
+struct erofs_zextent_key {
+	struct erofs_map_blocks map;
+	erofs_nid_t nid;
+	size_t decoded_size;
+};
+
+struct erofs_zextent_cache_metrics {
+	uint64_t hits;
+	uint64_t misses;
+	uint64_t bypasses;
+	uint64_t evictions;
+	uint64_t reclaims;
+	uint64_t waiter_merges;
+	size_t resident_bytes;
+};
+
+struct erofs_cache_stats {
+	uint64_t hits;
+	uint64_t misses;
+	uint64_t bypasses;
+	uint64_t evictions;
+	uint64_t reclaims;
+	uint64_t waiter_merges;
+	uint64_t entries;
+	uint64_t bytes;
+};
+
+struct erofs_zextent_cache_entry {
+	struct erofs_zextent_key key;
+	void *data;
+	size_t charged_bytes;
+	unsigned int waiters;
+	int error;
+	enum erofs_zextent_cache_state state;
+	struct cv cv;
+	uint64_t lru;
+	bool active;
+};
+
+struct erofs_zextent_cache {
+	struct erofs_zextent_cache_entry entries[EROFS_ZCACHE_MAX_ENTRIES];
+	size_t charged_bytes;
+	size_t budget_bytes;
+	uint64_t minimum_decode_work;
+	struct erofs_zextent_cache_metrics metrics[Z_EROFS_COMPRESSION_MAX];
+	uint64_t lru_clock;
+	unsigned int entry_count;
+	bool closing;
+};
+
+struct erofs_decompress_task {
+	struct task task;
+	struct mtx lock;
+	struct cv cv;
+	struct erofs_sb_info *sbi;
+	struct erofs_inode *vi;
+	struct erofs_map_blocks map;
+	size_t decoded_len;
+	bool partial;
+	void *decoded;
+	int error;
+	bool done;
+};
+
+enum erofs_xattr_cache_state {
+	EROFS_XATTR_CACHE_EMPTY,
+	EROFS_XATTR_CACHE_INFLIGHT,
+	EROFS_XATTR_CACHE_READY,
+	EROFS_XATTR_CACHE_FAILED,
+};
+
+struct erofs_xattr_cache {
+	struct mtx lock;
+	struct cv cv;
+	void *data;
+	size_t charged_bytes;
+	unsigned int waiters;
+	int error;
+	enum erofs_xattr_cache_state state;
+	bool closing;
+	bool initialized;
+};
+
+#define EROFS_STREAM_CTX_WRAPPER_SIZE 256
+
+struct erofs_stream_ctx;
+typedef int erofs_stream_ctx_init_t(struct erofs_stream_ctx *ctx);
+typedef void erofs_stream_ctx_fini_t(struct erofs_stream_ctx *ctx);
+
+struct erofs_stream_ctx {
+	STAILQ_ENTRY(erofs_stream_ctx) link;
+	struct erofs_sb_info *sbi;
+	erofs_stream_ctx_fini_t *fini;
+	size_t charged_bytes;
+	uint8_t algorithm;
+	bool cached;
+	bool allocation_failed;
+};
+
+STAILQ_HEAD(erofs_stream_ctx_head, erofs_stream_ctx);
+
+struct erofs_stream_pool_codec {
+	struct erofs_stream_ctx_head idle;
+	unsigned int contexts;
+	unsigned int cached;
+	unsigned int borrowed;
+	unsigned int idle_count;
+};
+
+struct erofs_stream_pool {
+	struct erofs_stream_pool_codec codec[Z_EROFS_COMPRESSION_MAX];
+	size_t resident_bytes;
+	unsigned int waiters;
+	struct cv cv;
+	bool closing;
+};
+
+struct erofs_sb_info {
+	struct mount *mnt;
+	struct erofs_device_info dif0;
+	struct mtx control_lock;
+	struct sysctl_ctx_list sysctl_ctx;
+	uint64_t control_id;
+	uint64_t dir_ra_bytes;
+	uint64_t dir_ra_max_bytes;
+	uint64_t dir_ra_requests;
+	uint64_t dir_ra_blocks;
+	bool control_initialized;
+	bool sysctl_registered;
+	bool control_closing;
+
+	uint32_t block_size;
+	uint32_t sb_size;
+	uint8_t blkszbits;
+	uint32_t meta_blkaddr;
+	uint32_t xattr_blkaddr;
+	uint32_t xattr_prefix_start;
+	uint8_t xattr_prefix_count;
+	uint8_t xattr_filter_reserved;
+	erofs_nid_t packed_nid;
+	erofs_nid_t metabox_nid;
+	struct erofs_inode *metabox_en;
+	struct erofs_inode *packed_inode;
+	struct erofs_xattr_prefix_item *xattr_prefixes;
+	erofs_blk_t blocks;
+	uint64_t inos;
+	erofs_nid_t root_nid;
+	int64_t epoch;
+	uint32_t fixed_nsec;
+	uint32_t generation_seed;
+	uint32_t feature_compat;
+	uint32_t feature_incompat;
+
+	struct erofs_sb_info_opts opt;
+	uint16_t available_compr_algs;
+	uint32_t lzma_dict_size;
+	uint8_t zstd_windowlog;
+	unsigned int sync_decompress;
+
+	/* Device table */
+	uint16_t extra_devices;
+	uint16_t device_id_mask;
+	bool flatdev;
+	erofs_blk_t total_blocks;
+	erofs_blk_t flatdev_blocks;
+	struct erofs_device_info *devs;
+	struct mtx z_extent_cache_lock;
+	struct erofs_zextent_cache z_extent_cache;
+	LIST_ENTRY(erofs_sb_info) z_extent_cache_link;
+	bool z_extent_cache_initialized;
+	struct taskqueue *decompress_taskqueue;
+	struct cv decompress_taskqueue_cv;
+	unsigned int decompress_taskqueue_users;
+	unsigned int decompress_worker_count;
+	bool decompress_worker_cpu_bound;
+	uint8_t decompress_queue_priority;
+	uint64_t decompress_submissions;
+	uint64_t decompress_worker_runs;
+	uint64_t decompress_queue_fallbacks;
+	struct erofs_stream_pool stream_pool;
+	LIST_ENTRY(erofs_sb_info) stream_pool_link;
+	bool stream_pool_initialized;
+	volatile u_long xattr_cache_resident;
+};
+
+#define MTOE(mp) ((struct erofs_sb_info *)(mp)->mnt_data)
+
+#define EROFS_FEATURE_FUNCS(name, compat, feature)                          \
+	static inline bool                                                  \
+	erofs_sb_has_##name(const struct erofs_sb_info *sbi)                \
+	{                                                                   \
+		return (                                                    \
+		    (sbi->feature_##compat & EROFS_FEATURE_##feature) != 0); \
+	}
+
+EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
+EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
+EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
+EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
+EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
+EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
+EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
+EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
+EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
+EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
+EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT)
+EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX)
+EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
+EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)
+EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX)
+EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX)
+
+static inline bool
+erofs_sb_has_xattr_filter_v1(const struct erofs_sb_info *sbi)
+{
+	return (erofs_sb_has_xattr_filter(sbi) &&
+	    sbi->xattr_filter_reserved == 0);
+}
+
+struct erofs_inode {
+	erofs_nid_t nid;
+	uint64_t size;
+	uint64_t data_blocks;
+	/* Byte offset in the device, or in the metabox for a metabox nid. */
+	erofs_off_t inode_off;
+	erofs_blk_t startblk;
+	uint32_t generation;
+	uint32_t nlink;
+	uid_t uid;
+	gid_t gid;
+	mode_t mode;
+	__enum_uint8(vtype) vtype;
+	dev_t rdev;
+	time_t mtime;
+	uint32_t mtime_nsec;
+	uint8_t datalayout;
+	uint8_t inode_isize;
+	uint32_t xattr_isize;
+	bool compact_inode;
+	bool dot_omitted;
+	/* Compression fields */
+	uint16_t z_advise;
+	uint8_t z_algorithmtype[2];
+	uint8_t z_lclusterbits;
+	uint16_t z_idata_size;
+	erofs_off_t z_fragmentoff;
+	uint64_t z_tailextent_headlcn;
+	uint64_t z_extents;
+	bool z_initialized;
+	/* Chunk-based fields */
+	uint16_t chunkformat;
+	uint8_t chunkbits;
+	/* Fragment fields */
+	struct erofs_xattr_cache xattr_cache;
+};
+
+struct erofs_fid {
+	uint16_t len;
+	uint16_t pad;
+	uint32_t nid_hi;
+	uint32_t nid_lo;
+	uint32_t gen;
+};
+
+_Static_assert(sizeof(struct erofs_fid) == 16,
+    "EROFS file handle ABI must be 16 bytes");
+_Static_assert(sizeof(struct erofs_fid) <= sizeof(struct fid),
+    "struct erofs_fid must fit within struct fid");
+
+#define VTOE(vp) ((struct erofs_inode *)(vp)->v_data)
+
+static inline unsigned int
+erofs_inode_version(unsigned int ifmt)
+{
+	return ((ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK);
+}
+
+static inline unsigned int
+erofs_inode_datalayout(unsigned int ifmt)
+{
+	return ((ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK);
+}
+
+static inline bool
+erofs_nid_in_metabox(erofs_nid_t nid)
+{
+	return ((nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) != 0);
+}
+
+static inline uint64_t
+erofs_nid_to_ino64(const struct erofs_sb_info *sbi, erofs_nid_t nid)
+{
+	if (!erofs_sb_has_metabox(sbi))
+		return (nid);
+	return ((nid << 1) & 0xffffffff00000000ULL) |
+	    (nid & 0x000000007fffffffULL) |
+	    ((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31);
+}
+
+static inline int
+erofs_ino_to_nid(const struct erofs_sb_info *sbi, ino_t ino,
+    erofs_nid_t *nidp)
+{
+	uint64_t ino64;
+	erofs_nid_t nid;
+
+	ino64 = (uint64_t)ino;
+	if (!erofs_sb_has_metabox(sbi))
+		nid = ino64;
+	else
+		nid = ((ino64 >> 1) & UINT64_C(0x7fffffff80000000)) |
+		    (ino64 & UINT64_C(0x000000007fffffff)) |
+		    ((ino64 & UINT64_C(0x0000000080000000)) << 32);
+	if (erofs_nid_to_ino64(sbi, nid) != ino64)
+		return (EOVERFLOW);
+	*nidp = nid;
+	return (0);
+}
+
+/* Allocated on disk at m_pa. */
+#define EROFS_MAP_MAPPED	 0x0001
+/* Located in metadata. */
+#define EROFS_MAP_META		 0x0002
+#define EROFS_MAP_PARTIAL_MAPPED 0x0004
+#define EROFS_MAP_PARTIAL_REF	 0x0008
+#define EROFS_MAP_FRAGMENT	 0x0010
+#define EROFS_MAP_FULL(f) \
+	(!((f) & (EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF)))
+
+#define EROFS_GET_BLOCKS_FIEMAP	  0x0001
+#define EROFS_GET_BLOCKS_FINDTAIL 0x0004
+
+enum {
+	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
+	Z_EROFS_COMPRESSION_INTERLACED,
+	Z_EROFS_COMPRESSION_RUNTIME_MAX
+};
+
+_Static_assert(Z_EROFS_COMPRESSION_RUNTIME_MAX <= UINT8_MAX,
+    "algorithm format must fit in uint8_t");
+
+struct erofs_map_dev {
+	struct erofs_device_info *m_dif;
+
+	erofs_off_t m_pa;
+	unsigned int m_deviceid;
+	uint64_t m_plen;
+};
+
+struct erofs_buf {
+	void *data;
+	void (*release)(void *);
+};
+
+#define EROFS_BUF_INITIALIZER { .data = NULL, .release = NULL }
+
+/* Buffer and device I/O. */
+void erofs_put_metabuf(struct erofs_buf *buf);
+int erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len,
+    void **bufp);
+int erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id,
+    erofs_off_t off, size_t len, void **bufp);
+void erofs_brelse(void *buf);
+int erofs_read_metadata_range(struct erofs_sb_info *sbi, erofs_nid_t nid,
+    erofs_off_t off, size_t len, struct erofs_buf *buf);
+/* Logical mapping and file data. */
+int erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map);
+int erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, void **bufp);
+int erofs_read_data_readahead(struct erofs_sb_info *sbi,
+    struct erofs_inode *vi, erofs_off_t loff, size_t len, bool sequential,
+    void **bufp);
+uint64_t erofs_dir_ra_bytes(struct erofs_sb_info *sbi);
+int erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value);
+void erofs_record_dir_readahead(struct erofs_sb_info *sbi,
+    unsigned int blocks);
+int erofs_read_file(struct vnode *vp, struct uio *uio);
+int erofs_file_llseek(struct vnode *vp, off_t *offset, bool seek_data);
+int erofs_readlink_target(struct vnode *vp, struct uio *uio);
+
+/* File-backed source I/O is owned by fileio.c. */
+#if WITH_EROFS_BACKED_BY_FILE
+int erofs_read_file_source(struct erofs_sb_info *sbi,
+    struct erofs_device_info *dif, erofs_off_t off, size_t len, void *out);
+#else
+static inline int
+erofs_read_file_source(struct erofs_sb_info *sbi, struct erofs_device_info *dif,
+    erofs_off_t off, size_t len, void *out)
+{
+	(void)sbi;
+	(void)dif;
+	(void)off;
+	(void)len;
+	(void)out;
+	return (EOPNOTSUPP);
+}
+#endif
+
+/* Inode and vnode lifecycle. */
+bool erofs_nid_is_valid(struct erofs_sb_info *sbi, erofs_nid_t nid);
+int erofs_read_inode(struct erofs_sb_info *sbi, erofs_nid_t nid,
+    struct erofs_inode *vi);
+int erofs_vget_raw(struct mount *mp, erofs_nid_t nid, int flags,
+    struct vnode **vpp);
+int erofs_vget_raw_alloc(struct mount *mp, void *arg, int flags,
+    struct vnode **vpp);
+int erofs_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp);
+
+/* Directory operations. */
+int erofs_readdir_block(struct vnode *vp, struct uio *uio, int *eofflag,
+    int *ncookies, uint64_t **cookies);
+int erofs_dirblock_ndirents(const char *blk, uint32_t maxsize,
+    uint32_t *ndirentsp);
+int erofs_dirent_span(const char *blk, uint32_t maxsize, uint32_t ndirents,
+    uint32_t idx, uint32_t *nameoffp, uint32_t *endoffp);
+int erofs_lookup(struct vop_cachedlookup_args *ap);
+
+/* Compressed mapping and data. */
+int z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi);
+int z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map);
+void z_erofs_extent_cache_init(struct erofs_sb_info *sbi);
+void z_erofs_extent_cache_fini(struct erofs_sb_info *sbi);
+int z_erofs_extent_cache_drop(struct erofs_sb_info *sbi);
+void z_erofs_extent_cache_stats(struct erofs_sb_info *sbi,
+    struct erofs_cache_stats *stats);
+void z_erofs_stream_pool_init(struct erofs_sb_info *sbi);
+void z_erofs_stream_pool_fini(struct erofs_sb_info *sbi);
+int z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm,
+    size_t context_size, erofs_stream_ctx_init_t *init,
+    erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp);
+void z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable);
+int z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes);
+void z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes);
+void *z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes);
+void z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address);
+int z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, void **bufp);
+int z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct uio *uio);
+int z_erofs_decompress(struct erofs_sb_info *sbi,
+    const struct erofs_map_blocks *map, const void *src, size_t srclen,
+    void *dst, size_t dstlen, bool partial);
+int z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map, size_t decoded_len, bool partial,
+    void **bufp);
+int z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi,
+    struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len,
+    bool partial, void **bufp);
+
+/* Compression configuration. */
+int z_erofs_parse_cfgs(struct erofs_sb_info *sbi,
+    const struct erofs_super_block *dsb);
+
+#if WITH_EROFS_ZIP
+int erofs_parse_zip_options(struct mount *mp,
+    struct erofs_sb_info_opts *opts, unsigned int *syncp,
+    bool *cache_presentp, bool *sync_presentp);
+#endif
+
+int erofs_parse_dir_ra_option(struct erofs_sb_info *sbi,
+    struct vfsoptlist *opts);
+int erofs_parse_dir_ra_value(struct erofs_sb_info *sbi,
+    struct vfsoptlist *opts, uint64_t *valuep, bool *presentp);
+void erofs_management_init(struct erofs_sb_info *sbi);
+void erofs_management_fini(struct erofs_sb_info *sbi);
+int erofs_register_sysfs(struct erofs_sb_info *sbi);
+int erofs_unregister_sysfs(struct erofs_sb_info *sbi);
+
+/* VOP vectors. */
+extern struct vop_vector erofs_vnodeops;
+extern struct vop_vector erofs_fifoops;
+
+#endif /* _FS_EROFS_INTERNAL_H_ */

namei.c

@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ * Copyright (C) 2022, Alibaba Cloud
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/dirent.h>
+#include <sys/libkern.h>
+#include <sys/namei.h>
+#include <sys/vnode.h>
+
+#include "trace.h"
+#include "xattr.h"
+
+struct erofs_qstr {
+	const unsigned char *name;
+	const unsigned char *end;
+};
+
+/*
+ * Compare two directory entry names using an already-matched prefix.
+ * qn: search key (not necessarily null-terminated).
+ * qd: on-disk name range (may not be null-terminated).
+ * matched: in/out count of prefix characters already known to match.
+ *
+ * Returns 0 if equal, 1 if qn > qd, -1 if qn < qd.
+ */
+static int
+erofs_dirnamecmp(const struct erofs_qstr *qn, const struct erofs_qstr *qd,
+    unsigned int *matched)
+{
+	size_t dname_span, qname_span;
+	unsigned int i;
+
+	qname_span = qn->end - qn->name;
+	dname_span = qd->end - qd->name;
+	i = MIN(*matched, qname_span);
+	i = MIN(i, dname_span);
+	while (i < qname_span && i < dname_span && qd->name[i] != '\0') {
+		if (qn->name[i] != qd->name[i]) {
+			*matched = i;
+			return (qn->name[i] > qd->name[i] ? 1 : -1);
+		}
+		++i;
+	}
+	*matched = i;
+	if (i == qname_span)
+		return (i == dname_span || qd->name[i] == '\0' ? 0 : -1);
+	return (1);
+}
+
+/*
+ * Name of the idx'th dirent of a block, as a range inside the block.  Only
+ * the bounds are checked; see erofs_dirent_span().
+ */
+static int
+erofs_dirent_qstr(const char *data, uint32_t datasize, uint32_t ndirents,
+    uint32_t idx, struct erofs_qstr *name)
+{
+	uint32_t endoff, nameoff;
+	int error;
+
+	error = erofs_dirent_span(data, datasize, ndirents, idx, &nameoff,
+	    &endoff);
+	if (error != 0)
+		return (error);
+	name->name = (const unsigned char *)data + nameoff;
+	name->end = (const unsigned char *)data + endoff;
+	return (0);
+}
+
+/*
+ * Binary search within a directory block for the target name.
+ *
+ * Sets *dep to the matching dirent, or NULL on a miss.  Only the entries
+ * the search visits are looked at, and each is bounds-checked as it is
+ * visited, so an unsorted or otherwise odd block makes the search miss but
+ * cannot take it outside the block.
+ */
+static int
+find_target_dirent(const struct erofs_qstr *name, char *data,
+    uint32_t datasize, uint32_t ndirents, struct erofs_dirent **dep)
+{
+	uint32_t head, back;
+	unsigned int startprfx, endprfx;
+	struct erofs_dirent *const de = (struct erofs_dirent *)data;
+	int error;
+
+	*dep = NULL;
+	/* The 1st dirent has already been evaluated by the caller. */
+	head = 1;
+	back = ndirents - 1;
+	startprfx = endprfx = 0;
+
+	while (head <= back) {
+		const uint32_t mid = head + (back - head) / 2;
+		unsigned int matched = MIN(startprfx, endprfx);
+		struct erofs_qstr dname;
+		int ret;
+
+		error = erofs_dirent_qstr(data, datasize, ndirents, mid,
+		    &dname);
+		if (error != 0)
+			return (error);
+
+		/* String comparison without already matched prefix */
+		ret = erofs_dirnamecmp(name, &dname, &matched);
+
+		if (ret == 0) {
+			*dep = de + mid;
+			return (0);
+		} else if (ret > 0) {
+			head = mid + 1;
+			startprfx = matched;
+		} else {
+			back = mid - 1;
+			endprfx = matched;
+		}
+	}
+
+	return (0);
+}
+
+static int
+erofs_read_dirblock(struct erofs_sb_info *sbi, struct erofs_inode *dir,
+    uint64_t block, char **datap, uint32_t *datasizep, uint32_t *ndirentsp)
+{
+	erofs_off_t block_off;
+	uint32_t datasize;
+	char *data;
+	int error;
+
+	*datap = NULL;
+	/* The only caller searches the blocks that hold dir->size bytes. */
+	block_off = block * sbi->block_size;
+	EROFS_DEBUG_ASSERT(block_off < dir->size,
+	    ("erofs: directory block %ju beyond the size", (uintmax_t)block));
+	datasize = MIN((uint64_t)sbi->block_size, dir->size - block_off);
+	error = erofs_read_data(sbi, dir, block_off, datasize, (void **)&data);
+	if (error != 0)
+		return (error);
+	error = erofs_dirblock_ndirents(data, datasize, ndirentsp);
+	if (error != 0) {
+		erofs_brelse(data);
+		return (erofs_corrupt(sbi, dir->nid,
+		    "invalid dirent array in block %ju", (uintmax_t)block));
+	}
+	*datap = data;
+	*datasizep = datasize;
+	return (0);
+}
+
+/*
+ * Find the directory block most likely to contain the target name.
+ *
+ * Uses two-level binary search: first across blocks, then within the
+ * candidate block via find_target_dirent().
+ *
+ * Returns the block buffer on success (caller must erofs_brelse),
+ * or NULL if no block matches, in which case *errorp is 0, or on error,
+ * when *errorp is set to a positive errno.  *_ndirents is set to the number
+ * of dirents in the returned block (0 means the first entry is the match).
+ */
+static char *
+erofs_find_target_block(struct erofs_sb_info *sbi, struct erofs_inode *dir,
+    const struct erofs_qstr *name, uint32_t *_ndirents, uint32_t *_datasize,
+    int *errorp)
+{
+	uint32_t bsz = sbi->block_size;
+	uint64_t head, back;
+	unsigned int startprfx = 0, endprfx = 0;
+	char *candidate = NULL;
+	int error;
+
+	*errorp = 0;
+	*_ndirents = 0;
+	*_datasize = 0;
+
+	if (dir->size == 0)
+		return (NULL);
+
+	head = 0;
+	back = (dir->size - 1) / bsz;
+
+	while (head <= back) {
+		const uint64_t mid = head + (back - head) / 2;
+		uint32_t maxsize;
+		char *blk;
+		int diff;
+		uint32_t ndirents;
+		unsigned int matched;
+		struct erofs_qstr dname;
+
+		error = erofs_read_dirblock(sbi, dir, mid, &blk, &maxsize,
+		    &ndirents);
+		if (error != 0) {
+			*errorp = error;
+			goto out;
+		}
+		error = erofs_dirent_qstr(blk, maxsize, ndirents, 0, &dname);
+		if (error != 0) {
+			erofs_brelse(blk);
+			*errorp = erofs_corrupt(sbi, dir->nid,
+			    "invalid name span of the first dirent in block "
+			    "%ju",
+			    (uintmax_t)mid);
+			goto out;
+		}
+
+		matched = MIN(startprfx, endprfx);
+		/* String comparison without already matched prefix */
+		diff = erofs_dirnamecmp(name, &dname, &matched);
+
+		if (diff < 0) {
+			erofs_brelse(blk);
+			if (mid == 0)
+				break;
+			back = mid - 1;
+			endprfx = matched;
+			continue;
+		}
+
+		/* diff >= 0: this block is a candidate. */
+		if (candidate != NULL)
+			erofs_brelse(candidate);
+		candidate = blk;
+		if (diff == 0) {
+			*_ndirents = 0;
+			*_datasize = maxsize;
+			return (candidate);
+		}
+		head = mid + 1;
+		startprfx = matched;
+		*_ndirents = ndirents;
+		*_datasize = maxsize;
+	}
+	return (candidate);
+out:
+	if (candidate != NULL)
+		erofs_brelse(candidate);
+	return (NULL);
+}
+
+/*
+ * Look up a name in a directory and return its nid.
+ */
+static int
+erofs_namei(struct erofs_sb_info *sbi, struct erofs_inode *dir,
+    const struct erofs_qstr *name, erofs_nid_t *nid)
+{
+	int error;
+	uint32_t ndirents;
+	uint32_t datasize;
+	char *blk;
+	struct erofs_dirent *de;
+
+	blk = erofs_find_target_block(sbi, dir, name, &ndirents,
+	    &datasize, &error);
+	if (blk == NULL)
+		return (error != 0 ? error : ENOENT);
+
+	de = (struct erofs_dirent *)blk;
+	if (ndirents > 0) {
+		error = find_target_dirent(name, blk, datasize, ndirents, &de);
+		if (error != 0)
+			error = erofs_corrupt(sbi, dir->nid,
+			    "invalid name span in a dirent block");
+	}
+
+	if (error == 0 && de != NULL) {
+		erofs_nid_t found_nid;
+
+		found_nid = le64toh(de->nid);
+		if (!erofs_nid_is_valid(sbi, found_nid)) {
+			error = erofs_corrupt(sbi, dir->nid,
+			    "dirent points to invalid nid %ju",
+			    (uintmax_t)found_nid);
+		} else {
+			*nid = found_nid;
+		}
+	}
+	erofs_brelse(blk);
+	if (error != 0)
+		return (error);
+	return (de != NULL ? 0 : ENOENT);
+}
+
+/*
+ * Directory name lookup (VOP_CACHEDLOOKUP entry point).
+ *
+ * FreeBSD-side API requirements:
+ * - "." must be returned under the caller's requested lock mode;
+ * - ".." must go through vn_vget_ino() to avoid holding a child lock while
+ *   acquiring the parent directory lock in reverse;
+ * - Both hit and miss must correctly update the namecache.
+ */
+int
+erofs_lookup(struct vop_cachedlookup_args *ap)
+{
+	struct erofs_qstr qname;
+	struct vnode *dvp, *vp;
+	struct erofs_inode *dir;
+	struct erofs_sb_info *sbi;
+	struct componentname *cnp;
+	erofs_nid_t nid;
+	int error, ltype;
+
+	dvp = ap->a_dvp;
+	cnp = ap->a_cnp;
+	*ap->a_vpp = NULL;
+	if ((cnp->cn_flags & ISLASTCN) != 0 &&
+	    (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
+		return (EROFS);
+	if (cnp->cn_namelen > EROFS_NAME_LEN)
+		return (ENAMETOOLONG);
+	erofs_trace_lookup(VTOE(dvp), cnp->cn_nameptr,
+	    (size_t)cnp->cn_namelen, (uint32_t)cnp->cn_flags);
+	if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') {
+		vref(dvp);
+		ltype = cnp->cn_lkflags & LK_TYPE_MASK;
+		if (ltype != VOP_ISLOCKED(dvp)) {
+			if (ltype == LK_EXCLUSIVE)
+				vn_lock(dvp, LK_UPGRADE | LK_RETRY);
+			else if (ltype == LK_SHARED)
+				vn_lock(dvp, LK_DOWNGRADE | LK_RETRY);
+		}
+		*ap->a_vpp = dvp;
+		return (0);
+	}
+
+	dir = VTOE(dvp);
+	sbi = MTOE(dvp->v_mount);
+	qname.name = (const unsigned char *)cnp->cn_nameptr;
+	qname.end = qname.name + cnp->cn_namelen;
+	error = erofs_namei(sbi, dir, &qname, &nid);
+	if (error != 0) {
+		if (error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0)
+			cache_enter(dvp, NULL, cnp);
+		if (error == ENOENT && (cnp->cn_flags & ISLASTCN) != 0 &&
+		    cnp->cn_nameiop == CREATE)
+			return (EROFS);
+		return (error);
+	}
+	if ((cnp->cn_flags & ISDOTDOT) != 0 && nid == dir->nid) {
+		if (nid != sbi->root_nid)
+			return (erofs_corrupt(sbi, dir->nid,
+			    "'..' of a non-root directory points to itself"));
+		vref(dvp);
+		ltype = cnp->cn_lkflags & LK_TYPE_MASK;
+		if (ltype != VOP_ISLOCKED(dvp)) {
+			if (ltype == LK_EXCLUSIVE)
+				vn_lock(dvp, LK_UPGRADE | LK_RETRY);
+			else if (ltype == LK_SHARED)
+				vn_lock(dvp, LK_DOWNGRADE | LK_RETRY);
+		}
+		*ap->a_vpp = dvp;
+		return (0);
+	}
+	if ((cnp->cn_flags & ISDOTDOT) == 0 && nid == dir->nid)
+		return (erofs_corrupt(sbi, dir->nid,
+		    "directory contains an entry pointing to itself"));
+
+	if ((cnp->cn_flags & ISDOTDOT) != 0)
+		error = vn_vget_ino_gen(dvp, erofs_vget_raw_alloc, &nid,
+		    cnp->cn_lkflags, &vp);
+	else
+		error = erofs_vget_raw(dvp->v_mount, nid, cnp->cn_lkflags, &vp);
+	if (error != 0)
+		return (error);
+	if ((cnp->cn_flags & ISDOTDOT) != 0 && VTOE(vp)->vtype != VDIR) {
+		vput(vp);
+		return (erofs_corrupt(sbi, dir->nid,
+		    "'..' does not lead to a directory (nid %ju)",
+		    (uintmax_t)nid));
+	}
+	*ap->a_vpp = vp;
+	if ((cnp->cn_flags & MAKEENTRY) != 0)
+		cache_enter(dvp, vp, cnp);
+	return (0);
+}

sysfs.c

@@ -0,0 +1,571 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* FreeBSD per-mount management controls for EROFS. */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+
+#include "compress.h"
+#include "internal.h"
+
+#define EROFS_DIR_RA_DEFAULT_BYTES	(16UL * 1024)
+#define EROFS_DIR_RA_HARD_MAX_BYTES	(1024UL * 1024)
+#define EROFS_DIR_RA_SLOTS		256UL
+
+enum erofs_cache_metric {
+	EROFS_CACHE_HITS,
+	EROFS_CACHE_MISSES,
+	EROFS_CACHE_BYPASSES,
+	EROFS_CACHE_EVICTIONS,
+	EROFS_CACHE_RECLAIMS,
+	EROFS_CACHE_WAITER_MERGES,
+	EROFS_CACHE_ENTRIES,
+	EROFS_CACHE_BYTES,
+};
+
+static struct mtx erofs_control_id_lock;
+static uint64_t erofs_control_next_id = 1;
+
+MTX_SYSINIT(erofs_control_id, &erofs_control_id_lock, "erofs control id",
+    MTX_DEF);
+
+SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+    "EROFS filesystem");
+SYSCTL_NODE(_vfs_erofs, OID_AUTO, mounts, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+    "Mounted EROFS filesystems");
+
+void
+erofs_management_init(struct erofs_sb_info *sbi)
+{
+
+	mtx_init(&sbi->control_lock, "erofs control", NULL, MTX_DEF);
+	sbi->dir_ra_bytes = EROFS_DIR_RA_DEFAULT_BYTES;
+	sbi->dir_ra_max_bytes = EROFS_DIR_RA_HARD_MAX_BYTES;
+	sbi->control_initialized = true;
+}
+
+void
+erofs_management_fini(struct erofs_sb_info *sbi)
+{
+
+	if (!sbi->control_initialized)
+		return;
+	KASSERT(!sbi->sysctl_registered,
+	    ("erofs management sysctls remain registered"));
+	mtx_destroy(&sbi->control_lock);
+	sbi->control_initialized = false;
+}
+
+uint64_t
+erofs_dir_ra_bytes(struct erofs_sb_info *sbi)
+{
+	uint64_t value;
+
+	mtx_lock(&sbi->control_lock);
+	value = sbi->dir_ra_bytes;
+	mtx_unlock(&sbi->control_lock);
+	return (value);
+}
+
+/* The readahead is limited by the size of the per-directory run lists. */
+static uint64_t
+erofs_dir_ra_limit(const struct erofs_sb_info *sbi)
+{
+
+	return (MIN((uint64_t)EROFS_DIR_RA_HARD_MAX_BYTES,
+	    (uint64_t)sbi->block_size * EROFS_DIR_RA_SLOTS));
+}
+
+int
+erofs_set_dir_ra_bytes(struct erofs_sb_info *sbi, uint64_t value)
+{
+
+	if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) ||
+	    (value != 0 && value % sbi->block_size != 0))
+		return (EINVAL);
+	mtx_lock(&sbi->control_lock);
+	if (sbi->control_closing) {
+		mtx_unlock(&sbi->control_lock);
+		return (ENXIO);
+	}
+	sbi->dir_ra_bytes = value;
+	mtx_unlock(&sbi->control_lock);
+	return (0);
+}
+
+void
+erofs_record_dir_readahead(struct erofs_sb_info *sbi, unsigned int blocks)
+{
+
+	mtx_lock(&sbi->control_lock);
+	++sbi->dir_ra_requests;
+	sbi->dir_ra_blocks += blocks;
+	mtx_unlock(&sbi->control_lock);
+}
+
+static int
+erofs_parse_uint64(const char *value, size_t len, uint64_t *result)
+{
+	size_t i;
+	uint64_t parsed;
+
+	if (value == NULL || len <= 1 || value[len - 1] != '\0' ||
+	    strnlen(value, len) != len - 1)
+		return (EINVAL);
+	parsed = 0;
+	for (i = 0; i < len - 1; ++i) {
+		if (value[i] < '0' || value[i] > '9')
+			return (EINVAL);
+		if (parsed > (UINT64_MAX - (value[i] - '0')) / 10)
+			return (ERANGE);
+		parsed = parsed * 10 + (value[i] - '0');
+	}
+	*result = parsed;
+	return (0);
+}
+
+int
+erofs_parse_dir_ra_value(struct erofs_sb_info *sbi, struct vfsoptlist *opts,
+    uint64_t *valuep, bool *presentp)
+{
+	struct vfsopt *opt;
+	uint64_t value;
+	int error;
+
+	*presentp = false;
+	TAILQ_FOREACH(opt, opts, link) {
+		if (strcmp(opt->name, "dir_ra_bytes") != 0)
+			continue;
+		error = erofs_parse_uint64(opt->value, opt->len, &value);
+		if (error != 0)
+			return (error);
+		if (sbi->block_size == 0 || value > erofs_dir_ra_limit(sbi) ||
+		    (value != 0 && value % sbi->block_size != 0))
+			return (EINVAL);
+		*valuep = value;
+		*presentp = true;
+		return (0);
+	}
+	return (0);
+}
+
+int
+erofs_parse_dir_ra_option(struct erofs_sb_info *sbi, struct vfsoptlist *opts)
+{
+	uint64_t value;
+	bool present;
+	int error;
+
+	error = erofs_parse_dir_ra_value(sbi, opts, &value, &present);
+	if (error != 0 || !present)
+		return (error);
+	return (erofs_set_dir_ra_bytes(sbi, value));
+}
+
+static int
+erofs_sysctl_dir_ra(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_sb_info *sbi;
+	uint64_t value;
+	int error;
+
+	sbi = arg1;
+	value = erofs_dir_ra_bytes(sbi);
+	error = sysctl_handle_64(oidp, &value, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	return (erofs_set_dir_ra_bytes(sbi, value));
+}
+
+static int
+erofs_sysctl_dir_ra_metric(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_sb_info *sbi;
+	uint64_t value;
+
+	sbi = arg1;
+	mtx_lock(&sbi->control_lock);
+	value = arg2 == 0 ? sbi->dir_ra_requests : sbi->dir_ra_blocks;
+	mtx_unlock(&sbi->control_lock);
+	return (sysctl_handle_64(oidp, &value, 0, req));
+}
+
+#if WITH_EROFS_ZIP
+static int
+erofs_validate_zip_value(const char *value, int len)
+{
+
+	if (value == NULL || len <= 1 || value[len - 1] != '\0' ||
+	    strnlen(value, (size_t)len) != (size_t)len - 1)
+		return (EINVAL);
+	return (0);
+}
+
+static int
+erofs_parse_zip_uint(const char *value, int len, unsigned int *result)
+{
+	size_t i;
+	unsigned int parsed;
+
+	if (erofs_validate_zip_value(value, len) != 0)
+		return (EINVAL);
+	parsed = 0;
+	for (i = 0; i < (size_t)len - 1; ++i) {
+		if (value[i] < '0' || value[i] > '9' ||
+		    parsed > (UINT_MAX - (value[i] - '0')) / 10)
+			return (EINVAL);
+		parsed = parsed * 10 + (value[i] - '0');
+	}
+	*result = parsed;
+	return (0);
+}
+
+int
+erofs_parse_zip_options(struct mount *mp, struct erofs_sb_info_opts *opts,
+    unsigned int *syncp, bool *cache_presentp, bool *sync_presentp)
+{
+	struct vfsopt *opt;
+	unsigned int value;
+	bool queue_priority_present;
+	int error;
+
+	*cache_presentp = false;
+	*sync_presentp = false;
+	queue_priority_present = false;
+	TAILQ_FOREACH(opt, mp->mnt_optnew, link) {
+		if (strcmp(opt->name, "cache_strategy") == 0) {
+			if (opt->value == NULL || opt->len <= 1 ||
+			    ((char *)opt->value)[opt->len - 1] != '\0' ||
+			    strnlen(opt->value, opt->len) !=
+			    (size_t)opt->len - 1)
+				return (EINVAL);
+			if (strcmp(opt->value, "disabled") == 0)
+				opts->cache_strategy = EROFS_ZIP_CACHE_DISABLED;
+			else if (strcmp(opt->value, "readahead") == 0)
+				opts->cache_strategy =
+				    EROFS_ZIP_CACHE_READAHEAD;
+			else if (strcmp(opt->value, "readaround") == 0)
+				opts->cache_strategy =
+				    EROFS_ZIP_CACHE_READAROUND;
+			else
+				return (EINVAL);
+			*cache_presentp = true;
+			continue;
+		}
+		if (strcmp(opt->name, "worker_queue_priority") == 0) {
+			error = erofs_parse_zip_uint(opt->value, opt->len,
+			    &value);
+			if (error != 0 || value > UINT8_MAX)
+				return (EINVAL);
+			opts->worker_queue_priority = (uint8_t)value;
+			queue_priority_present = true;
+			continue;
+		}
+		if (strcmp(opt->name, "sync_decompress") != 0)
+			continue;
+		error = erofs_validate_zip_value(opt->value, opt->len);
+		if (error != 0)
+			return (error);
+		if (strcmp(opt->value, "auto") == 0)
+			value = EROFS_SYNC_DECOMPRESS_AUTO;
+		else if (strcmp(opt->value, "on") == 0)
+			value = EROFS_SYNC_DECOMPRESS_FORCE_ON;
+		else if (strcmp(opt->value, "off") == 0)
+			value = EROFS_SYNC_DECOMPRESS_FORCE_OFF;
+		else {
+			error = erofs_parse_zip_uint(opt->value, opt->len,
+			    &value);
+			if (error != 0 ||
+			    value > EROFS_SYNC_DECOMPRESS_FORCE_OFF)
+				return (EINVAL);
+		}
+		*syncp = value;
+		*sync_presentp = true;
+	}
+	if (*cache_presentp)
+		vfs_deleteopt(mp->mnt_optnew, "cache_strategy");
+	if (*sync_presentp)
+		vfs_deleteopt(mp->mnt_optnew, "sync_decompress");
+	if (queue_priority_present)
+		vfs_deleteopt(mp->mnt_optnew, "worker_queue_priority");
+	return (0);
+}
+
+static int
+erofs_sysctl_drop_caches(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_sb_info *sbi;
+	uint64_t value;
+	int error;
+
+	if (req->newptr == NULL)
+		return (EPERM);
+	sbi = arg1;
+	value = 0;
+	error = sysctl_handle_64(oidp, &value, 0, req);
+	if (error != 0)
+		return (error);
+	if (value != 1)
+		return (EINVAL);
+	return (z_erofs_extent_cache_drop(sbi));
+}
+
+static int
+erofs_sysctl_cache_metric(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_cache_stats stats;
+	struct erofs_sb_info *sbi;
+	uint64_t value;
+
+	sbi = arg1;
+	z_erofs_extent_cache_stats(sbi, &stats);
+	switch (arg2) {
+	case EROFS_CACHE_HITS:
+		value = stats.hits;
+		break;
+	case EROFS_CACHE_MISSES:
+		value = stats.misses;
+		break;
+	case EROFS_CACHE_BYPASSES:
+		value = stats.bypasses;
+		break;
+	case EROFS_CACHE_EVICTIONS:
+		value = stats.evictions;
+		break;
+	case EROFS_CACHE_RECLAIMS:
+		value = stats.reclaims;
+		break;
+	case EROFS_CACHE_WAITER_MERGES:
+		value = stats.waiter_merges;
+		break;
+	case EROFS_CACHE_ENTRIES:
+		value = stats.entries;
+		break;
+	case EROFS_CACHE_BYTES:
+		value = stats.bytes;
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (sysctl_handle_64(oidp, &value, 0, req));
+}
+
+static int
+erofs_sysctl_zip_setting(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_sb_info *sbi;
+	int value;
+	int error;
+
+	sbi = arg1;
+	mtx_lock(&sbi->control_lock);
+	value = arg2 == 0 ? (int)sbi->sync_decompress :
+	    (int)sbi->opt.cache_strategy;
+	mtx_unlock(&sbi->control_lock);
+	error = sysctl_handle_int(oidp, &value, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	if (arg2 == 0) {
+		if (value < EROFS_SYNC_DECOMPRESS_AUTO ||
+		    value > EROFS_SYNC_DECOMPRESS_FORCE_OFF)
+			return (EINVAL);
+	} else {
+		if (value < EROFS_ZIP_CACHE_DISABLED ||
+		    value > EROFS_ZIP_CACHE_READAROUND)
+			return (EINVAL);
+	}
+	mtx_lock(&sbi->control_lock);
+	if (sbi->control_closing) {
+		mtx_unlock(&sbi->control_lock);
+		return (ENXIO);
+	}
+	if (arg2 == 0)
+		sbi->sync_decompress = (unsigned int)value;
+	else
+		sbi->opt.cache_strategy = (uint8_t)value;
+	mtx_unlock(&sbi->control_lock);
+	return (0);
+}
+
+static int
+erofs_sysctl_worker_queue_priority(SYSCTL_HANDLER_ARGS)
+{
+	struct erofs_sb_info *sbi;
+	int error, value;
+
+	sbi = arg1;
+	mtx_lock(&sbi->control_lock);
+	value = sbi->decompress_queue_priority;
+	mtx_unlock(&sbi->control_lock);
+	error = sysctl_handle_int(oidp, &value, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	if (value < 0 || value > UINT8_MAX)
+		return (EINVAL);
+	mtx_lock(&sbi->control_lock);
+	if (sbi->control_closing) {
+		mtx_unlock(&sbi->control_lock);
+		return (ENXIO);
+	}
+	sbi->decompress_queue_priority = (uint8_t)value;
+	sbi->opt.worker_queue_priority = (uint8_t)value;
+	mtx_unlock(&sbi->control_lock);
+	return (0);
+}
+#endif
+
+int
+erofs_register_sysfs(struct erofs_sb_info *sbi)
+{
+	struct sysctl_oid *cache_node, *mount_node;
+	char name[24];
+	int error;
+
+	mtx_lock(&erofs_control_id_lock);
+	sbi->control_id = erofs_control_next_id++;
+	mtx_unlock(&erofs_control_id_lock);
+
+	sbi->dir_ra_max_bytes = erofs_dir_ra_limit(sbi);
+
+	snprintf(name, sizeof(name), "%ju", (uintmax_t)sbi->control_id);
+	sysctl_ctx_init(&sbi->sysctl_ctx);
+	sbi->sysctl_registered = true;
+	mount_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx,
+	    SYSCTL_STATIC_CHILDREN(_vfs_erofs_mounts), OID_AUTO, name,
+	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "EROFS mount");
+	if (mount_node == NULL)
+		goto fail;
+	if (SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "mount_id", CTLFLAG_RD, &sbi->control_id, 0,
+	    "Unique mount control identifier") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "dir_ra_bytes", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
+    sbi, 0, erofs_sysctl_dir_ra, "QU",
+    "Directory readahead window in bytes") == NULL ||
+	    SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "dir_ra_max_bytes", CTLFLAG_RD, &sbi->dir_ra_max_bytes, 0,
+    "Maximum directory readahead window") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "dir_ra_requests",
+	    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, 0, erofs_sysctl_dir_ra_metric, "QU",
+	    "Directory readahead submissions") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "dir_ra_blocks",
+	    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, 1, erofs_sysctl_dir_ra_metric, "QU",
+	    "Directory readahead blocks submitted") == NULL)
+		goto fail;
+#if WITH_EROFS_ZIP
+	if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "drop_caches", CTLTYPE_U64 | CTLFLAG_WR | CTLFLAG_MPSAFE,
+	    sbi, 0, erofs_sysctl_drop_caches, "QU",
+	    "Write 1 to drop the FreeBSD decoded extent cache") == NULL)
+		goto fail;
+	if (SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "sync_decompress",
+	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+	    sbi, 0, erofs_sysctl_zip_setting, "I",
+	    "0=auto, 1=force synchronous, 2=taskqueue fallback") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(mount_node),
+	    OID_AUTO, "cache_strategy",
+	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+	    sbi, 1, erofs_sysctl_zip_setting, "I",
+	    "0=disabled, 1=readahead, 2=readaround") == NULL)
+		goto fail;
+	cache_node = SYSCTL_ADD_NODE(&sbi->sysctl_ctx,
+	    SYSCTL_CHILDREN(mount_node), OID_AUTO, "decoded_cache",
+	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
+	    "FreeBSD decoded extent cache, not Linux managed pclusters");
+	if (cache_node == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "hits", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi,
+	    EROFS_CACHE_HITS, erofs_sysctl_cache_metric, "QU",
+	    "Cache hits") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "misses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi,
+	    EROFS_CACHE_MISSES, erofs_sysctl_cache_metric, "QU",
+	    "Cache misses") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "bypasses", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, EROFS_CACHE_BYPASSES, erofs_sysctl_cache_metric, "QU",
+	    "Cache admission bypasses") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "evictions", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, EROFS_CACHE_EVICTIONS, erofs_sysctl_cache_metric, "QU",
+	    "Cache evictions") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "reclaims", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, EROFS_CACHE_RECLAIMS, erofs_sysctl_cache_metric, "QU",
+	    "Cache pressure reclaims") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "waiter_merges",
+	    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
+	    sbi, EROFS_CACHE_WAITER_MERGES, erofs_sysctl_cache_metric, "QU",
+	    "Inflight cache waiter merges") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "entries", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi,
+	    EROFS_CACHE_ENTRIES, erofs_sysctl_cache_metric, "QU",
+	    "Resident decoded extents") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "bytes", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sbi,
+	    EROFS_CACHE_BYTES, erofs_sysctl_cache_metric, "QU",
+	    "Resident decoded bytes") == NULL ||
+	    SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "worker_submissions", CTLFLAG_RD,
+	    &sbi->decompress_submissions,
+	    0, "Taskqueue decompression submissions") == NULL ||
+	    SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "worker_runs", CTLFLAG_RD, &sbi->decompress_worker_runs,
+	    0, "Taskqueue decompression worker runs") == NULL ||
+	    SYSCTL_ADD_UINT(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "worker_count", CTLFLAG_RD, &sbi->decompress_worker_count,
+	    0, "Bounded decompression worker count") == NULL ||
+	    SYSCTL_ADD_U64(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "queue_fallbacks", CTLFLAG_RD,
+	    &sbi->decompress_queue_fallbacks, 0,
+	    "Direct decode after taskqueue fallback") == NULL ||
+	    SYSCTL_ADD_U8(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "worker_cpu_bound", CTLFLAG_RD,
+	    (uint8_t *)&sbi->decompress_worker_cpu_bound, 0,
+	    "Workers restricted to the online CPU set") == NULL ||
+	    SYSCTL_ADD_PROC(&sbi->sysctl_ctx, SYSCTL_CHILDREN(cache_node),
+	    OID_AUTO, "worker_queue_priority",
+	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sbi, 0,
+	    erofs_sysctl_worker_queue_priority, "I",
+	    "Taskqueue order priority; 0 is highest and 255 is lowest") == NULL)
+		goto fail;
+#else
+	(void)cache_node;
+#endif
+	return (0);
+
+fail:
+	error = erofs_unregister_sysfs(sbi);
+	return (error != 0 ? error : ENOMEM);
+}
+
+int
+erofs_unregister_sysfs(struct erofs_sb_info *sbi)
+{
+	int error;
+
+	if (!sbi->sysctl_registered)
+		return (0);
+	mtx_lock(&sbi->control_lock);
+	sbi->control_closing = true;
+	mtx_unlock(&sbi->control_lock);
+	error = sysctl_ctx_free(&sbi->sysctl_ctx);
+	if (error != 0) {
+		mtx_lock(&sbi->control_lock);
+		sbi->control_closing = false;
+		mtx_unlock(&sbi->control_lock);
+		return (error);
+	}
+	sbi->sysctl_registered = false;
+	return (0);
+}

trace.c

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include "internal.h"
+#include "trace.h"
+
+SDT_PROVIDER_DEFINE(erofs);
+
+SDT_PROBE_DEFINE4(erofs, data, map_blocks, enter,
+    "uint64_t", "uint64_t", "uint64_t", "uint32_t");
+SDT_PROBE_DEFINE6(erofs, data, map_blocks, exit,
+    "uint64_t", "uint64_t", "uint64_t", "uint64_t", "uint64_t",
+    "uint64_t");
+SDT_PROBE_DEFINE5(erofs, data, read_folio, entry,
+    "uint64_t", "uint64_t", "uint64_t", "int", "bool");
+SDT_PROBE_DEFINE4(erofs, data, readahead, entry,
+    "uint64_t", "uint64_t", "uint64_t", "bool");
+SDT_PROBE_DEFINE3(erofs, inode, fill_inode, entry,
+    "uint64_t", "uint64_t", "uint32_t");
+SDT_PROBE_DEFINE4(erofs, namei, lookup, entry,
+    "uint64_t", "uint32_t", "uint64_t", "uint32_t");
+SDT_PROBE_DEFINE3(erofs, , , corrupt, "uint64_t", "int", "int");

trace.h

@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef _FS_EROFS_TRACE_H_
+#define _FS_EROFS_TRACE_H_
+
+#include <sys/sdt.h>
+
+#include "internal.h"
+
+SDT_PROVIDER_DECLARE(erofs);
+
+SDT_PROBE_DECLARE(erofs, data, map_blocks, enter);
+SDT_PROBE_DECLARE(erofs, data, map_blocks, exit);
+SDT_PROBE_DECLARE(erofs, data, read_folio, entry);
+SDT_PROBE_DECLARE(erofs, data, readahead, entry);
+SDT_PROBE_DECLARE(erofs, inode, fill_inode, entry);
+SDT_PROBE_DECLARE(erofs, namei, lookup, entry);
+SDT_PROBE_DECLARE(erofs, , , corrupt);
+
+#if WITH_EROFS_SDT
+#define EROFS_TRACE_CORRUPT(nid, line, error) \
+	SDT_PROBE3(erofs, , , corrupt, (nid), (line), (error))
+#else
+#define EROFS_TRACE_CORRUPT(nid, line, error) do { } while (0)
+#endif
+
+#define EROFS_TRACE_MAP_BLOCKS_ENTER(vi, map, flags) \
+	SDT_PROBE4(erofs, data, map_blocks, enter, \
+	    (vi)->nid, (map)->m_la, (map)->m_llen, (flags))
+
+#ifdef KDTRACE_HOOKS
+#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \
+	uint64_t erofs_trace_status; \
+	if (SDT_PROBES_ENABLED()) { \
+		erofs_trace_status = ((uint64_t)(uint32_t)(error) << 32) | \
+		    (uint64_t)(uint32_t)(map)->m_flags; \
+		SDT_PROBE6(erofs, data, map_blocks, exit, (vi)->nid, \
+		    (map)->m_la, (map)->m_pa, (map)->m_llen, (map)->m_plen, \
+		    erofs_trace_status); \
+	} \
+} while (0)
+#else
+#define EROFS_TRACE_MAP_BLOCKS_EXIT(vi, map, error) do { \
+	(void)(vi); \
+	(void)(map); \
+	(void)(error); \
+} while (0)
+#endif
+
+#define EROFS_TRACE_READ_FOLIO(vi, offset, length, raw) \
+	SDT_PROBE5(erofs, data, read_folio, entry, (vi)->nid, \
+	    (uint64_t)(offset), (uint64_t)(length), (vi)->vtype == VDIR, (raw))
+
+#define EROFS_TRACE_READAHEAD(vi, start, nrpages, raw) \
+	SDT_PROBE4(erofs, data, readahead, entry, (vi)->nid, \
+	    (uint64_t)(start), (uint64_t)(nrpages), (raw))
+
+#define EROFS_TRACE_FILL_INODE(nid, blkaddr, offset) \
+	SDT_PROBE3(erofs, inode, fill_inode, entry, (nid), \
+	    (uint64_t)(blkaddr), (uint32_t)(offset))
+
+static __inline void
+erofs_trace_lookup(const struct erofs_inode *dir, const char *name,
+    size_t length, uint32_t flags)
+{
+#ifdef KDTRACE_HOOKS
+	uint32_t hash;
+	size_t i;
+
+	if (!SDT_PROBES_ENABLED())
+		return;
+	hash = UINT32_C(2166136261);
+	for (i = 0; i < length; ++i) {
+		hash ^= (uint8_t)name[i];
+		hash *= UINT32_C(16777619);
+	}
+	SDT_PROBE4(erofs, namei, lookup, entry, dir->nid, hash,
+	    (uint64_t)length, flags);
+#else
+	(void)dir;
+	(void)name;
+	(void)length;
+	(void)flags;
+#endif
+}
+
+#endif

xattr.h

@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ */
+#ifndef _FS_EROFS_XATTR_H_
+#define _FS_EROFS_XATTR_H_
+
+#if WITH_EROFS_XATTR && WITH_EROFS_POSIX_ACL
+#include <sys/acl.h>
+#endif
+
+#include "internal.h"
+
+#if WITH_EROFS_XATTR
+int erofs_xattr_prefixes_init(struct erofs_sb_info *sbi);
+void erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi);
+int erofs_listxattr(struct vnode *vp, int attrnamespace, struct uio *uio,
+    size_t *sizep);
+int erofs_getxattr(struct vnode *vp, int attrnamespace, const char *name,
+    struct uio *uio, size_t *sizep);
+#if WITH_EROFS_POSIX_ACL
+int erofs_get_acl(struct vnode *vp, acl_type_t type, struct acl *aclp);
+#endif
+void erofs_xattr_cache_init(struct erofs_inode *vi);
+void erofs_xattr_cache_fini(struct erofs_sb_info *sbi,
+    struct erofs_inode *vi);
+#else
+static inline int
+erofs_xattr_prefixes_init(struct erofs_sb_info *sbi __unused)
+{
+	return (0);
+}
+
+static inline void
+erofs_xattr_prefixes_cleanup(struct erofs_sb_info *sbi __unused)
+{
+}
+
+static inline void
+erofs_xattr_cache_init(struct erofs_inode *vi __unused)
+{
+}
+
+static inline void
+erofs_xattr_cache_fini(struct erofs_sb_info *sbi __unused,
+    struct erofs_inode *vi __unused)
+{
+}
+#endif
+#endif

zdata.c

@@ -0,0 +1,937 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/_maxphys.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+#include <sys/smp.h>
+#include <sys/uio.h>
+
+#include "compress.h"
+
+enum z_erofs_cache_lookup {
+	Z_EROFS_CACHE_BYPASS,
+	Z_EROFS_CACHE_HIT,
+	Z_EROFS_CACHE_OWNER,
+	Z_EROFS_CACHE_ERROR,
+};
+
+#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024)
+#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024)
+#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024)
+#define EROFS_ZCACHE_WORKER_HARD_LIMIT EROFS_ZCACHE_MAX_ENTRIES
+
+static int z_erofs_cache_enabled = 1;
+static unsigned long z_erofs_cache_mount_budget =
+    EROFS_ZCACHE_MOUNT_HARD_BUDGET;
+static unsigned long z_erofs_cache_global_budget =
+    EROFS_ZCACHE_GLOBAL_HARD_BUDGET;
+static unsigned long z_erofs_cache_minimum_decode_work =
+    EROFS_ZCACHE_MIN_DECODE_WORK;
+static unsigned long z_erofs_cache_worker_limit =
+    EROFS_ZCACHE_WORKER_HARD_LIMIT;
+
+TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget",
+    &z_erofs_cache_mount_budget);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget",
+    &z_erofs_cache_global_budget);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work",
+    &z_erofs_cache_minimum_decode_work);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.worker_limit",
+    &z_erofs_cache_worker_limit);
+
+static struct mtx z_erofs_cache_list_lock;
+static struct mtx z_erofs_cache_budget_lock;
+static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts =
+    LIST_HEAD_INITIALIZER(z_erofs_cache_mounts);
+static size_t z_erofs_cache_global_charged;
+static eventhandler_tag z_erofs_cache_lowmem_tag;
+
+MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock,
+    "erofs zcache list", MTX_DEF);
+MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock,
+    "erofs zcache budget", MTX_DEF);
+
+static size_t
+z_erofs_extent_cache_global_limit(void)
+{
+
+	return (MIN(z_erofs_cache_global_budget,
+	    EROFS_ZCACHE_GLOBAL_HARD_BUDGET));
+}
+
+static bool
+z_erofs_extent_cache_reserve(size_t bytes)
+{
+	size_t limit;
+	bool reserved;
+
+	reserved = false;
+	mtx_lock(&z_erofs_cache_budget_lock);
+	limit = z_erofs_extent_cache_global_limit();
+	if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) {
+		z_erofs_cache_global_charged += bytes;
+		reserved = true;
+	}
+	mtx_unlock(&z_erofs_cache_budget_lock);
+	return (reserved);
+}
+
+static void
+z_erofs_extent_cache_release(size_t bytes)
+{
+
+	mtx_lock(&z_erofs_cache_budget_lock);
+	KASSERT(bytes <= z_erofs_cache_global_charged,
+	    ("erofs decoded cache budget underflow"));
+	z_erofs_cache_global_charged -= bytes;
+	mtx_unlock(&z_erofs_cache_budget_lock);
+}
+
+static bool
+z_erofs_extent_cache_match(const struct erofs_zextent_cache_entry *entry,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size)
+{
+	const struct erofs_zextent_key *key = &entry->key;
+
+	return (entry->active && entry->state != EROFS_ZCACHE_EMPTY &&
+	    key->nid == vi->nid && key->decoded_size == decoded_size &&
+	    key->map.m_pa == map->m_pa && key->map.m_la == map->m_la &&
+	    key->map.m_plen == map->m_plen && key->map.m_llen == map->m_llen &&
+	    key->map.m_deviceid == map->m_deviceid &&
+	    key->map.m_algorithmformat == map->m_algorithmformat &&
+	    key->map.m_flags == map->m_flags);
+}
+
+static void
+z_erofs_extent_cache_reset_locked(struct erofs_zextent_cache *cache,
+    struct erofs_zextent_cache_entry *entry)
+{
+
+	KASSERT(entry->state != EROFS_ZCACHE_INFLIGHT && entry->waiters == 0,
+	    ("erofs cache entry reset while busy"));
+	if (entry->active) {
+		KASSERT(cache->entry_count != 0,
+		    ("erofs cache entry underflow"));
+		--cache->entry_count;
+	}
+	entry->active = false;
+	entry->state = EROFS_ZCACHE_EMPTY;
+	entry->error = 0;
+	entry->lru = 0;
+	bzero(&entry->key, sizeof(entry->key));
+}
+
+static void *
+z_erofs_extent_cache_drop_entry_locked(struct erofs_sb_info *sbi __unused,
+    struct erofs_zextent_cache *cache, struct erofs_zextent_cache_entry *entry,
+    bool evicted, bool reclaimed)
+{
+	struct erofs_zextent_cache_metrics *metrics;
+	void *data;
+
+	KASSERT(entry->state == EROFS_ZCACHE_READY && entry->waiters == 0,
+	    ("erofs cache drop of busy entry"));
+	data = entry->data;
+	metrics = &cache->metrics[entry->key.map.m_algorithmformat];
+	KASSERT(metrics->resident_bytes >= entry->charged_bytes,
+	    ("erofs decoded cache codec accounting mismatch"));
+	metrics->resident_bytes -= entry->charged_bytes;
+	if (evicted)
+		++metrics->evictions;
+	if (reclaimed)
+		++metrics->reclaims;
+	KASSERT(cache->charged_bytes >= entry->charged_bytes,
+	    ("erofs decoded cache mount accounting underflow"));
+	cache->charged_bytes -= entry->charged_bytes;
+	z_erofs_extent_cache_release(entry->charged_bytes);
+	entry->data = NULL;
+	entry->charged_bytes = 0;
+	z_erofs_extent_cache_reset_locked(cache, entry);
+	cv_broadcast(&entry->cv);
+	return (data);
+}
+
+static enum z_erofs_cache_lookup
+z_erofs_extent_cache_claim(struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_zextent_cache_metrics *metrics;
+	struct erofs_zextent_cache_entry *entry, *victim;
+	void *old;
+	enum z_erofs_cache_lookup result;
+	unsigned int i;
+
+	KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS"));
+	KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff,
+	    ("erofs extent cache copy exceeds decoded extent"));
+	*errorp = 0;
+	if (!sbi->z_extent_cache_initialized)
+		return (Z_EROFS_CACHE_BYPASS);
+	cache = &sbi->z_extent_cache;
+	KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX,
+	    ("erofs decoded cache claim has unknown codec"));
+	metrics = &cache->metrics[map->m_algorithmformat];
+	entry = NULL;
+	victim = NULL;
+	old = NULL;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	if (cache->closing) {
+		++metrics->bypasses;
+		goto bypass;
+	}
+	for (i = 0; i < nitems(cache->entries); ++i) {
+		if (z_erofs_extent_cache_match(&cache->entries[i], vi, map,
+		    decoded_size)) {
+			entry = &cache->entries[i];
+			break;
+		}
+	}
+	if (entry != NULL) {
+		if (entry->state == EROFS_ZCACHE_READY) {
+			KASSERT(entry->data != NULL,
+			    ("erofs ready extent cache has no data"));
+			memcpy(dst, (char *)entry->data + (size_t)mapoff, len);
+			entry->lru = ++cache->lru_clock;
+			++metrics->hits;
+			mtx_unlock(&sbi->z_extent_cache_lock);
+			return (Z_EROFS_CACHE_HIT);
+		}
+		if (entry->state == EROFS_ZCACHE_INFLIGHT) {
+			++metrics->waiter_merges;
+			++entry->waiters;
+			do {
+				cv_wait(&entry->cv, &sbi->z_extent_cache_lock);
+			} while (entry->state == EROFS_ZCACHE_INFLIGHT);
+			if (entry->state == EROFS_ZCACHE_READY) {
+				memcpy(dst,
+				    (char *)entry->data + (size_t)mapoff, len);
+				entry->lru = ++cache->lru_clock;
+				++metrics->hits;
+				result = Z_EROFS_CACHE_HIT;
+			} else {
+				KASSERT(entry->state == EROFS_ZCACHE_FAILED &&
+				    entry->error > 0,
+				    ("erofs inflight cache has no typed "
+				    "result"));
+				*errorp = entry->error;
+				result = Z_EROFS_CACHE_ERROR;
+			}
+			--entry->waiters;
+			if (entry->state == EROFS_ZCACHE_FAILED &&
+			    entry->waiters == 0)
+				z_erofs_extent_cache_reset_locked(cache, entry);
+			if (entry->waiters == 0)
+				cv_broadcast(&entry->cv);
+			mtx_unlock(&sbi->z_extent_cache_lock);
+			return (result);
+		}
+		if (entry->waiters != 0) {
+			++metrics->bypasses;
+			goto bypass;
+		}
+		z_erofs_extent_cache_reset_locked(cache, entry);
+	}
+	for (i = 0; i < nitems(cache->entries); ++i) {
+		if (!cache->entries[i].active) {
+			entry = &cache->entries[i];
+			break;
+		}
+		if (cache->entries[i].state == EROFS_ZCACHE_READY &&
+		    cache->entries[i].waiters == 0 &&
+		    (victim == NULL || cache->entries[i].lru < victim->lru))
+			victim = &cache->entries[i];
+	}
+	if (entry == NULL && victim != NULL) {
+		entry = victim;
+		old = z_erofs_extent_cache_drop_entry_locked(sbi, cache, entry,
+		    true, false);
+	}
+	if (entry == NULL || decoded_size > cache->budget_bytes ||
+	    cache->charged_bytes > cache->budget_bytes - decoded_size ||
+	    !z_erofs_extent_cache_reserve(decoded_size)) {
+		++metrics->bypasses;
+		goto bypass;
+	}
+	entry->key.map = *map;
+	entry->key.nid = vi->nid;
+	entry->key.decoded_size = decoded_size;
+	entry->charged_bytes = decoded_size;
+	entry->error = 0;
+	entry->state = EROFS_ZCACHE_INFLIGHT;
+	entry->active = true;
+	entry->lru = ++cache->lru_clock;
+	++cache->entry_count;
+	cache->charged_bytes += decoded_size;
+	++metrics->misses;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	free(old, M_EROFS);
+	return (Z_EROFS_CACHE_OWNER);
+
+bypass:
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	free(old, M_EROFS);
+	return (Z_EROFS_CACHE_BYPASS);
+}
+
+static void
+z_erofs_extent_cache_complete(struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size, void *decoded, int error)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_zextent_cache_entry *entry;
+	struct erofs_zextent_cache_metrics *metrics;
+	unsigned int i;
+
+	KASSERT((error == 0) == (decoded != NULL),
+	    ("erofs extent cache completion is untyped"));
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	entry = NULL;
+	for (i = 0; i < nitems(cache->entries); ++i) {
+		if (z_erofs_extent_cache_match(&cache->entries[i], vi, map,
+		    decoded_size)) {
+			entry = &cache->entries[i];
+			break;
+		}
+	}
+	KASSERT(entry != NULL && entry->state == EROFS_ZCACHE_INFLIGHT,
+	    ("erofs extent cache owner lost its key"));
+	metrics = &cache->metrics[map->m_algorithmformat];
+	if (error == 0) {
+		KASSERT(entry->charged_bytes == decoded_size,
+		    ("erofs decoded cache success accounting mismatch"));
+		entry->data = decoded;
+		metrics->resident_bytes += decoded_size;
+		entry->error = 0;
+		entry->state = EROFS_ZCACHE_READY;
+		entry->lru = ++cache->lru_clock;
+	} else {
+		KASSERT(error > 0,
+		    ("erofs extent cache published negative errno"));
+		KASSERT(cache->charged_bytes >= entry->charged_bytes,
+		    ("erofs decoded cache accounting underflow"));
+		cache->charged_bytes -= entry->charged_bytes;
+		z_erofs_extent_cache_release(entry->charged_bytes);
+		entry->charged_bytes = 0;
+		entry->error = error;
+		entry->state = EROFS_ZCACHE_FAILED;
+		if (entry->waiters == 0)
+			z_erofs_extent_cache_reset_locked(cache, entry);
+	}
+	cv_broadcast(&entry->cv);
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+static void
+z_erofs_extent_cache_lowmem(void *arg __unused, int howto __unused)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_sb_info *sbi;
+	void *data[EROFS_ZCACHE_MAX_ENTRIES];
+	unsigned int count, i;
+
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) {
+		count = 0;
+		if (!mtx_trylock(&sbi->z_extent_cache_lock))
+			continue;
+		cache = &sbi->z_extent_cache;
+		for (i = 0; i < nitems(cache->entries); ++i) {
+			if (cache->entries[i].state == EROFS_ZCACHE_READY &&
+			    cache->entries[i].waiters == 0)
+				data[count++] =
+				    z_erofs_extent_cache_drop_entry_locked(sbi,
+				    cache, &cache->entries[i], true, true);
+		}
+		mtx_unlock(&sbi->z_extent_cache_lock);
+		while (count != 0)
+			free(data[--count], M_EROFS);
+	}
+	mtx_unlock(&z_erofs_cache_list_lock);
+}
+
+static void
+z_erofs_extent_cache_global_init(void *arg __unused)
+{
+	z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
+	    z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT);
+}
+
+static void
+z_erofs_extent_cache_global_fini(void *arg __unused)
+{
+	if (z_erofs_cache_lowmem_tag != NULL)
+		EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag);
+	KASSERT(LIST_EMPTY(&z_erofs_cache_mounts),
+	    ("erofs decoded cache mounts remain at unload"));
+	KASSERT(z_erofs_cache_global_charged == 0,
+	    ("erofs decoded cache bytes remain at unload"));
+}
+
+SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST,
+    z_erofs_extent_cache_global_init, NULL);
+SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_FIRST,
+    z_erofs_extent_cache_global_fini, NULL);
+
+#if WITH_EROFS_PCPU_KTHREAD
+static unsigned int
+z_erofs_decode_worker_count(void)
+{
+	unsigned long count;
+
+	count = z_erofs_cache_worker_limit;
+	if (count == 0)
+		count = 1;
+	count = MIN(count, (unsigned long)EROFS_ZCACHE_WORKER_HARD_LIMIT);
+	count = MIN(count, (unsigned long)MAX(1, mp_ncpus));
+	return ((unsigned int)count);
+}
+#endif
+
+static void
+z_erofs_decode_taskqueue_init(struct erofs_sb_info *sbi __unused)
+{
+#if WITH_EROFS_PCPU_KTHREAD
+	unsigned int worker_count;
+
+	worker_count = z_erofs_decode_worker_count();
+	sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK,
+	    taskqueue_thread_enqueue, &sbi->decompress_taskqueue);
+	if (sbi->decompress_taskqueue == NULL)
+		return;
+	if (taskqueue_start_threads_cpuset(&sbi->decompress_taskqueue,
+	    (int)worker_count, PWAIT, &all_cpus, "erofs decode") == 0) {
+		sbi->decompress_worker_count = worker_count;
+		sbi->decompress_worker_cpu_bound = true;
+		return;
+	}
+	taskqueue_free(sbi->decompress_taskqueue);
+	sbi->decompress_taskqueue = taskqueue_create("erofs decode", M_WAITOK,
+	    taskqueue_thread_enqueue, &sbi->decompress_taskqueue);
+	if (sbi->decompress_taskqueue == NULL)
+		return;
+	if (taskqueue_start_threads(&sbi->decompress_taskqueue,
+	    (int)worker_count, PWAIT, "erofs decode") != 0) {
+		taskqueue_free(sbi->decompress_taskqueue);
+		sbi->decompress_taskqueue = NULL;
+		return;
+	}
+	sbi->decompress_worker_count = worker_count;
+#endif
+}
+
+static struct taskqueue *
+z_erofs_decode_taskqueue_acquire(struct erofs_sb_info *sbi)
+{
+	struct taskqueue *taskqueue;
+
+	mtx_lock(&sbi->z_extent_cache_lock);
+	if (sbi->z_extent_cache.closing ||
+	    sbi->decompress_taskqueue == NULL) {
+		taskqueue = NULL;
+	} else {
+		taskqueue = sbi->decompress_taskqueue;
+		++sbi->decompress_taskqueue_users;
+	}
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	return (taskqueue);
+}
+
+static void
+z_erofs_decode_taskqueue_release(struct erofs_sb_info *sbi)
+{
+	mtx_lock(&sbi->z_extent_cache_lock);
+	KASSERT(sbi->decompress_taskqueue_users != 0,
+	    ("erofs decode taskqueue user underflow"));
+	if (--sbi->decompress_taskqueue_users == 0)
+		cv_broadcast(&sbi->decompress_taskqueue_cv);
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+void
+z_erofs_extent_cache_init(struct erofs_sb_info *sbi)
+{
+
+	bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache));
+	sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget,
+	    EROFS_ZCACHE_MOUNT_HARD_BUDGET);
+	sbi->z_extent_cache.minimum_decode_work =
+	    z_erofs_cache_minimum_decode_work;
+	mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF);
+	cv_init(&sbi->decompress_taskqueue_cv, "erofs decode tq");
+	for (unsigned int i = 0; i < nitems(sbi->z_extent_cache.entries); ++i)
+		cv_init(&sbi->z_extent_cache.entries[i].cv, "erofs zextent");
+	z_erofs_decode_taskqueue_init(sbi);
+	sbi->z_extent_cache_initialized = true;
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link);
+	mtx_unlock(&z_erofs_cache_list_lock);
+}
+
+void
+z_erofs_extent_cache_fini(struct erofs_sb_info *sbi)
+{
+	struct erofs_zextent_cache *cache;
+	struct taskqueue *taskqueue;
+	void *data[EROFS_ZCACHE_MAX_ENTRIES];
+	unsigned int count, i;
+
+	if (!sbi->z_extent_cache_initialized)
+		return;
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_REMOVE(sbi, z_extent_cache_link);
+	mtx_unlock(&z_erofs_cache_list_lock);
+	mtx_lock(&sbi->z_extent_cache_lock);
+	cache->closing = true;
+	sbi->z_extent_cache_initialized = false;
+	for (;;) {
+		bool busy = false;
+		unsigned int wait_index = 0;
+		for (i = 0; i < nitems(cache->entries); ++i)
+			if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT ||
+			    cache->entries[i].waiters != 0) {
+				busy = true;
+				wait_index = i;
+				break;
+			}
+		if (!busy)
+			break;
+		cv_wait(&cache->entries[wait_index].cv,
+		    &sbi->z_extent_cache_lock);
+	}
+	/*
+	 * The workers were created with &sbi->decompress_taskqueue as their
+	 * context and read it when they first run, which can be after this
+	 * point if the mount failed early.  Keep it valid until
+	 * taskqueue_free() has waited for every worker to exit; closing
+	 * already keeps new dispatches away from the queue.
+	 */
+	taskqueue = sbi->decompress_taskqueue;
+	sbi->decompress_worker_count = 0;
+	sbi->decompress_worker_cpu_bound = false;
+	while (sbi->decompress_taskqueue_users != 0)
+		cv_wait(&sbi->decompress_taskqueue_cv,
+		    &sbi->z_extent_cache_lock);
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	if (taskqueue != NULL) {
+		taskqueue_drain_all(taskqueue);
+		taskqueue_free(taskqueue);
+	}
+	mtx_lock(&sbi->z_extent_cache_lock);
+	sbi->decompress_taskqueue = NULL;
+	count = 0;
+	for (i = 0; i < nitems(cache->entries); ++i) {
+		if (cache->entries[i].state == EROFS_ZCACHE_READY)
+			data[count++] =
+			    z_erofs_extent_cache_drop_entry_locked(sbi,
+			    cache, &cache->entries[i], false, false);
+		else if (cache->entries[i].active)
+			z_erofs_extent_cache_reset_locked(cache,
+			    &cache->entries[i]);
+	}
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	while (count != 0)
+		free(data[--count], M_EROFS);
+	for (i = 0; i < nitems(cache->entries); ++i)
+		cv_destroy(&cache->entries[i].cv);
+	cv_destroy(&sbi->decompress_taskqueue_cv);
+	mtx_destroy(&sbi->z_extent_cache_lock);
+}
+
+int
+z_erofs_extent_cache_drop(struct erofs_sb_info *sbi)
+{
+	struct erofs_zextent_cache *cache;
+	void *data[EROFS_ZCACHE_MAX_ENTRIES];
+	unsigned int count, i;
+
+	if (!sbi->z_extent_cache_initialized)
+		return (ENXIO);
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	if (cache->closing) {
+		mtx_unlock(&sbi->z_extent_cache_lock);
+		return (ENXIO);
+	}
+	for (;;) {
+		bool busy = false;
+		unsigned int wait_index = 0;
+		for (i = 0; i < nitems(cache->entries); ++i)
+			if (cache->entries[i].state == EROFS_ZCACHE_INFLIGHT ||
+			    cache->entries[i].waiters != 0) {
+				busy = true;
+				wait_index = i;
+				break;
+			}
+		if (!busy)
+			break;
+		cv_wait(&cache->entries[wait_index].cv,
+		    &sbi->z_extent_cache_lock);
+	}
+	count = 0;
+	for (i = 0; i < nitems(cache->entries); ++i) {
+		if (cache->entries[i].state == EROFS_ZCACHE_READY)
+			data[count++] =
+			    z_erofs_extent_cache_drop_entry_locked(sbi,
+			    cache, &cache->entries[i], true, false);
+		else if (cache->entries[i].active)
+			z_erofs_extent_cache_reset_locked(cache,
+			    &cache->entries[i]);
+	}
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	while (count != 0)
+		free(data[--count], M_EROFS);
+	return (0);
+}
+
+void
+z_erofs_extent_cache_stats(struct erofs_sb_info *sbi,
+    struct erofs_cache_stats *stats)
+{
+	struct erofs_zextent_cache *cache;
+	unsigned int algorithm;
+
+	bzero(stats, sizeof(*stats));
+	if (!sbi->z_extent_cache_initialized)
+		return;
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
+		stats->hits += cache->metrics[algorithm].hits;
+		stats->misses += cache->metrics[algorithm].misses;
+		stats->bypasses += cache->metrics[algorithm].bypasses;
+		stats->evictions += cache->metrics[algorithm].evictions;
+		stats->reclaims += cache->metrics[algorithm].reclaims;
+		stats->waiter_merges += cache->metrics[algorithm].waiter_merges;
+		stats->bytes += cache->metrics[algorithm].resident_bytes;
+	}
+	stats->entries = cache->entry_count;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+static bool
+z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t len)
+{
+	const struct erofs_zextent_cache *cache;
+	uint64_t decode_work;
+
+	cache = &sbi->z_extent_cache;
+	decode_work = map->m_plen + len;	/* at most 1 MiB + 12 MiB */
+	return (z_erofs_cache_enabled != 0 &&
+	    sbi->opt.cache_strategy != EROFS_ZIP_CACHE_DISABLED &&
+	    len <= MAXPHYS &&
+	    len <= cache->budget_bytes &&
+	    decode_work >= cache->minimum_decode_work &&
+	    (map->m_flags & (EROFS_MAP_META |
+	    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
+	    EROFS_MAP_FRAGMENT)) == 0 &&
+	    map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX &&
+	    sbi->z_extent_cache_initialized && vi != sbi->packed_inode &&
+	    vi != sbi->metabox_en);
+}
+
+static void
+z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi,
+    const struct erofs_map_blocks *map)
+{
+	struct erofs_zextent_cache *cache;
+
+	if (!sbi->z_extent_cache_initialized ||
+	    map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX)
+		return;
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	++cache->metrics[map->m_algorithmformat].bypasses;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+static void
+z_erofs_decode_length(const struct erofs_map_blocks *map,
+    erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial)
+{
+	size_t end;
+
+	/* z_erofs_do_read() has checked that [mapoff, mapoff + want) fits. */
+	EROFS_DEBUG_ASSERT(mapoff <= map->m_llen &&
+	    want <= map->m_llen - mapoff, ("erofs: bad decode range"));
+	end = (size_t)mapoff + want;
+	*partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;
+	*decoded_len = *partial ? end : (size_t)map->m_llen;
+}
+
+int
+z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	const void *input;
+	void *compressed, *decoded;
+	int error;
+
+	*bufp = NULL;
+	/*
+	 * z_erofs_do_read() handles fragment and unmapped extents itself and
+	 * gets decoded_len from z_erofs_decode_length().
+	 */
+	EROFS_DEBUG_ASSERT((map->m_flags &
+	    (EROFS_MAP_FRAGMENT | EROFS_MAP_MAPPED)) == EROFS_MAP_MAPPED,
+	    ("erofs: decoding a map with flags %#x",
+	    (unsigned int)map->m_flags));
+	EROFS_DEBUG_ASSERT(decoded_len != 0 && decoded_len <= map->m_llen &&
+	    (partial || decoded_len == map->m_llen),
+	    ("erofs: bad decoded length %zu", decoded_len));
+
+	if ((map->m_flags & EROFS_MAP_META) != 0) {
+		error = erofs_read_metadata_range(sbi, vi->nid, map->m_pa,
+		    (size_t)map->m_plen, &buf);
+	} else {
+		error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa,
+		    (size_t)map->m_plen, &compressed);
+	}
+	if (error != 0)
+		return (error);
+	input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed;
+
+	decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO);
+	error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,
+	    decoded, decoded_len, partial);
+	if ((map->m_flags & EROFS_MAP_META) != 0)
+		erofs_put_metabuf(&buf);
+	else
+		erofs_brelse(compressed);
+	if (error != 0) {
+		free(decoded, M_EROFS);
+		return (error);
+	}
+	*bufp = decoded;
+	return (0);
+}
+
+static void
+z_erofs_decode_task_run(void *context, int pending __unused)
+{
+	struct erofs_decompress_task *task = context;
+
+	task->error = z_erofs_decode_extent(task->sbi, task->vi, &task->map,
+	    task->decoded_len, task->partial, &task->decoded);
+	mtx_lock(&task->lock);
+	atomic_add_64(&task->sbi->decompress_worker_runs, 1);
+	task->done = true;
+	cv_signal(&task->cv);
+	mtx_unlock(&task->lock);
+}
+
+static uint8_t
+z_erofs_decode_queue_priority(struct erofs_sb_info *sbi)
+{
+	uint8_t priority;
+
+	mtx_lock(&sbi->control_lock);
+	priority = sbi->decompress_queue_priority;
+	mtx_unlock(&sbi->control_lock);
+	return (priority);
+}
+
+int
+z_erofs_decode_extent_dispatch(struct erofs_sb_info *sbi,
+    struct erofs_inode *vi, struct erofs_map_blocks *map, size_t decoded_len,
+    bool partial, void **bufp)
+{
+	struct erofs_decompress_task *task;
+	struct taskqueue *taskqueue;
+	int error;
+	uint8_t queue_priority;
+	bool queued;
+
+	if (sbi->sync_decompress != EROFS_SYNC_DECOMPRESS_FORCE_OFF)
+		return (z_erofs_decode_extent(sbi, vi, map, decoded_len,
+		    partial, bufp));
+	taskqueue = z_erofs_decode_taskqueue_acquire(sbi);
+	if (taskqueue == NULL) {
+		atomic_add_64(&sbi->decompress_queue_fallbacks, 1);
+		return (z_erofs_decode_extent(sbi, vi, map, decoded_len,
+		    partial, bufp));
+	}
+	/*
+	 * A decode worker can get here again: reading the inline input of a
+	 * metabox inode reads the (compressed) metabox.  Waiting for another
+	 * task from inside a worker could use up every worker, so decode in
+	 * place.
+	 */
+	if (taskqueue_member(taskqueue, curthread)) {
+		z_erofs_decode_taskqueue_release(sbi);
+		atomic_add_64(&sbi->decompress_queue_fallbacks, 1);
+		return (z_erofs_decode_extent(sbi, vi, map, decoded_len,
+		    partial, bufp));
+	}
+	task = malloc(sizeof(*task), M_EROFS, M_WAITOK | M_ZERO);
+	task->sbi = sbi;
+	task->vi = vi;
+	task->map = *map;
+	task->decoded_len = decoded_len;
+	task->partial = partial;
+	mtx_init(&task->lock, "erofs decomp task", NULL, MTX_DEF);
+	cv_init(&task->cv, "erofs decomp task");
+	queue_priority = z_erofs_decode_queue_priority(sbi);
+	TASK_INIT(&task->task, queue_priority, z_erofs_decode_task_run, task);
+	error = taskqueue_enqueue(taskqueue, &task->task);
+	queued = error == 0;
+	if (queued) {
+		atomic_add_64(&sbi->decompress_submissions, 1);
+		mtx_lock(&task->lock);
+		while (!task->done)
+			cv_wait(&task->cv, &task->lock);
+		mtx_unlock(&task->lock);
+		taskqueue_drain(taskqueue, &task->task);
+		error = task->error;
+		*bufp = task->decoded;
+	} else {
+		atomic_add_64(&sbi->decompress_queue_fallbacks, 1);
+		error = z_erofs_decode_extent(sbi, vi, map, decoded_len,
+		    partial, bufp);
+	}
+	cv_destroy(&task->cv);
+	mtx_destroy(&task->lock);
+	free(task, M_EROFS);
+	z_erofs_decode_taskqueue_release(sbi);
+	return (error);
+}
+
+static int
+z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, char *out)
+{
+	struct erofs_map_blocks map;
+	void *decoded, *fragment;
+	erofs_off_t mapoff;
+	size_t decoded_len, done, want;
+	bool cache_eligible, partial;
+	enum z_erofs_cache_lookup cache_lookup;
+	int error;
+
+	done = 0;
+	while (done < len) {
+		map = (struct erofs_map_blocks) { .m_la = loff + done };
+		error = erofs_map_blocks(sbi, vi, &map);
+		if (error != 0)
+			return (error);
+		if (map.m_llen == 0 || map.m_la > loff + done ||
+		    loff + done - map.m_la >= map.m_llen)
+			return (EINTEGRITY);
+		mapoff = loff + done - map.m_la;
+		want = MIN((size_t)(map.m_llen - mapoff), len - done);
+
+		if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) {
+			if (sbi->packed_inode == NULL ||
+			    sbi->packed_inode->nid == vi->nid ||
+			    vi->z_fragmentoff > UINT64_MAX - mapoff)
+				return (EINTEGRITY);
+			error = erofs_read_data(sbi, sbi->packed_inode,
+			    vi->z_fragmentoff + mapoff, want, &fragment);
+			if (error != 0)
+				return (error);
+			memcpy(out + done, fragment, want);
+			erofs_brelse(fragment);
+		} else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
+			bzero(out + done, want);
+		} else {
+			z_erofs_decode_length(&map, mapoff, want, &decoded_len,
+			    &partial);
+			cache_eligible = !partial &&
+			    z_erofs_extent_cache_eligible(sbi, vi, &map,
+			    decoded_len);
+			cache_lookup = Z_EROFS_CACHE_BYPASS;
+			if (cache_eligible) {
+				cache_lookup = z_erofs_extent_cache_claim(sbi,
+				    vi, &map, decoded_len, mapoff, want,
+				    out + done, &error);
+				if (cache_lookup == Z_EROFS_CACHE_HIT) {
+					done += want;
+					continue;
+				}
+				if (cache_lookup == Z_EROFS_CACHE_ERROR)
+					return (error);
+			} else
+				z_erofs_extent_cache_record_bypass(sbi, &map);
+			error = z_erofs_decode_extent_dispatch(sbi, vi, &map,
+			    decoded_len, partial, &decoded);
+			if (error != 0) {
+				if (cache_lookup == Z_EROFS_CACHE_OWNER)
+					z_erofs_extent_cache_complete(sbi, vi,
+					    &map, decoded_len, NULL, error);
+				return (error);
+			}
+			memcpy(out + done, (char *)decoded + (size_t)mapoff,
+			    want);
+			if (cache_lookup == Z_EROFS_CACHE_OWNER)
+				z_erofs_extent_cache_complete(sbi, vi, &map,
+				    decoded_len, decoded, 0);
+			else
+				free(decoded, M_EROFS);
+		}
+		done += want;
+	}
+	return (0);
+}
+
+int
+z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, void **bufp)
+{
+	char *out;
+	int error;
+
+	/* erofs_read_data_impl() has checked the arguments and the range. */
+	EROFS_DEBUG_ASSERT(len != 0 && loff <= vi->size &&
+	    len <= vi->size - loff, ("erofs: bad compressed read"));
+	*bufp = NULL;
+	out = malloc(len, M_EROFS, M_WAITOK);
+	error = z_erofs_do_read(sbi, vi, loff, len, out);
+	if (error != 0) {
+		free(out, M_EROFS);
+		return (error);
+	}
+	*bufp = out;
+	return (0);
+}
+
+int
+z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct uio *uio)
+{
+	char *buf;
+	size_t want;
+	int error;
+
+	/* erofs_read_uio() has rejected a negative offset. */
+	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) {
+		want = MIN((size_t)uio->uio_resid,
+		    (size_t)MIN((uint64_t)MAXPHYS,
+		    vi->size - (uint64_t)uio->uio_offset));
+		buf = malloc(want, M_EROFS, M_WAITOK);
+		error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset,
+		    want, buf);
+		if (error == 0)
+			error = uiomove(buf, want, uio);
+		free(buf, M_EROFS);
+		if (error != 0)
+			return (error);
+	}
+	return (0);
+}

zmap.c

@@ -0,0 +1,949 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/endian.h>
+#include <sys/libkern.h>
+
+#include "internal.h"
+
+struct z_erofs_maprecorder {
+	struct erofs_sb_info *sbi;
+	struct erofs_inode *vi;
+	struct erofs_map_blocks *map;
+	uint64_t lcn;
+	uint8_t type, headtype;
+	unsigned int clusterofs;
+	uint16_t delta[2];
+	erofs_blk_t pblk, compressedblks;
+	erofs_off_t nextpackoff;
+	bool partialref;
+};
+
+_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META |
+    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
+    EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag");
+
+/*
+ * The arithmetic in this file is plain on purpose.  Every value is bounded
+ * before it gets here, and a position that did wrap would still be range
+ * checked where it is read (erofs_bread_device(), erofs_read_data_impl()):
+ *
+ * - blkszbits is in [9, PAGE_SHIFT] (super.c) and z_lclusterbits is
+ *   blkszbits + 0..15, rejected from 31 up (z_erofs_fill_inode()), so a
+ *   cluster is at most 1 GiB and a shift by z_lclusterbits is safe.
+ * - vi->size is at most OFF_MAX (erofs_read_inode()) and every request has
+ *   m_la < vi->size (z_erofs_map_blocks()), so lcn << z_lclusterbits stays
+ *   below 2^63 + 2^46 even after adding a 16-bit delta.
+ * - vi->inode_off lies inside the image (erofs_nid_is_valid()), inode_isize
+ *   is 32 or 64 and xattr_isize is below 2^19.
+ * - An extent table has at most 2^48 records of at most 32 bytes each.
+ * - erofs_blk_t is 64 bits wide (internal.h).
+ *
+ * The list is repeated in docs/validation-policy.md.
+ */
+
+/* The map header follows the inode and its xattrs, 8-byte aligned. */
+static inline erofs_off_t
+z_erofs_header_pos(const struct erofs_inode *vi)
+{
+	return (roundup2(vi->inode_off + vi->inode_isize + vi->xattr_isize, 8));
+}
+
+static int
+z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len,
+    struct erofs_buf *buf)
+{
+	return (erofs_read_metadata_range(m->sbi, m->vi->nid, pos, len, buf));
+}
+
+static int
+z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct erofs_inode *vi;
+	struct z_erofs_lcluster_index *di;
+	erofs_off_t base, pos;
+	unsigned int advise;
+	int error;
+
+	vi = m->vi;
+	base = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header) + 8;
+	pos = base + lcn * sizeof(*di);
+	m->nextpackoff = pos + sizeof(*di);
+	error = z_erofs_read_index(m, pos, sizeof(*di), &buf);
+	if (error != 0)
+		return (error);
+
+	di = buf.data;
+	m->lcn = lcn;
+	advise = le16toh(di->di_advise);
+	m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK;
+	if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+		m->clusterofs = 1U << vi->z_lclusterbits;
+		m->delta[0] = le16toh(di->di_u.delta[0]);
+		if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) {
+			if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
+			    Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) {
+				erofs_put_metabuf(&buf);
+				return (erofs_corrupt(m->sbi, vi->nid,
+				    "lcn %ju has a compressed block count but "
+				    "the inode has no big pclusters",
+				    (uintmax_t)lcn));
+			}
+			m->compressedblks =
+			    m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT;
+			m->delta[0] = 1;
+		}
+		m->delta[1] = le16toh(di->di_u.delta[1]);
+	} else {
+		m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0;
+		m->clusterofs = le16toh(di->di_clusterofs);
+		m->pblk = le32toh(di->di_u.blkaddr);
+	}
+	erofs_put_metabuf(&buf);
+	return (0);
+}
+
+static unsigned int
+decode_compactedbits(unsigned int lobits, const uint8_t *in,
+    unsigned int pos, uint8_t *type)
+{
+	uint32_t v;
+	unsigned int lo;
+
+	v = le32dec(in + pos / 8) >> (pos & 7);
+	lo = v & ((1U << lobits) - 1);
+	*type = (v >> lobits) & 3;
+	return (lo);
+}
+
+static unsigned int
+get_compacted_la_distance(unsigned int lobits, unsigned int encodebits,
+    unsigned int vcnt, const uint8_t *in, int i)
+{
+	unsigned int lo, d1;
+	uint8_t type;
+
+	d1 = 0;
+	do {
+		lo = decode_compactedbits(lobits, in, encodebits * i, &type);
+		if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
+			return (d1);
+		++d1;
+	} while (++i < (int)vcnt);
+
+	/* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */
+	if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0)
+		d1 += lo - 1;
+	return (d1);
+}
+
+static int
+z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn,
+    bool lookahead)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct erofs_inode *vi;
+	erofs_off_t ebase, pos;
+	uint64_t compacted_2b, totalidx, original_lcn;
+	unsigned int compacted_4b_initial, amortizedshift;
+	unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize;
+	bool big_pcluster;
+	uint8_t *in, type;
+	int error, i;
+
+	vi = m->vi;
+	/*
+	 * A compact index entry is 16 bits (4-byte units) or 14 bits (2-byte
+	 * units): a 2-bit type and the offset in the lcluster.  Linux has the
+	 * same limit.
+	 */
+	if (vi->z_lclusterbits > 14)
+		return (erofs_corrupt(m->sbi, vi->nid,
+		    "invalid lcluster bits %u", vi->z_lclusterbits));
+	ebase = z_erofs_header_pos(vi) + sizeof(struct z_erofs_map_header);
+	totalidx = (vi->size + (1ULL << vi->z_lclusterbits) - 1) >>
+	    vi->z_lclusterbits;
+	if (lcn >= totalidx)
+		return (EINVAL);
+
+	original_lcn = lcn;
+	m->lcn = lcn;
+	compacted_4b_initial = ((32 - ebase % 32) / 4) & 7;
+	compacted_2b = 0;
+	if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 &&
+	    compacted_4b_initial < totalidx)
+		compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16);
+
+	pos = ebase;
+	amortizedshift = 2;
+	if (lcn >= compacted_4b_initial) {
+		pos += compacted_4b_initial * 4;
+		lcn -= compacted_4b_initial;
+		if (lcn < compacted_2b) {
+			amortizedshift = 1;
+		} else {
+			pos += compacted_2b * 2;
+			lcn -= compacted_2b;
+		}
+	}
+	pos += lcn << amortizedshift;
+
+	if (amortizedshift == 2 && vi->z_lclusterbits <= 14)
+		vcnt = 2;
+	else if (amortizedshift == 1 && vi->z_lclusterbits <= 12)
+		vcnt = 16;
+	else {
+		erofs_err(m->sbi, "unsupported lcluster bits %u @ nid %ju",
+		    vi->z_lclusterbits, (uintmax_t)vi->nid);
+		return (EOPNOTSUPP);
+	}
+
+	packsize = vcnt << amortizedshift;
+	bytes = pos & (packsize - 1);
+	pos -= bytes;
+	m->nextpackoff = pos + packsize;
+	error = z_erofs_read_index(m, pos, packsize, &buf);
+	if (error != 0)
+		return (error);
+	in = buf.data;
+	lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT));
+	encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt;
+	i = bytes >> amortizedshift;
+
+	lo = decode_compactedbits(lobits, in, encodebits * i, &type);
+	m->type = type;
+	if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+		m->clusterofs = 1U << vi->z_lclusterbits;
+		if (lookahead) {
+			m->delta[1] = get_compacted_la_distance(lobits,
+			    encodebits, vcnt, in, i);
+		}
+		big_pcluster =
+		    (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
+		if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) {
+			if (!big_pcluster) {
+				erofs_put_metabuf(&buf);
+				return (erofs_corrupt(m->sbi, vi->nid,
+				    "lcn %ju has a compressed block count but "
+				    "the inode has no big pclusters",
+				    (uintmax_t)original_lcn));
+			}
+			m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT;
+			m->delta[0] = 1;
+		} else if (i + 1 != (int)vcnt) {
+			m->delta[0] = lo;
+		} else {
+			EROFS_DEBUG_ASSERT(i > 0,
+			    ("erofs: pack of %u slots", vcnt));
+			lo = decode_compactedbits(lobits, in,
+			    encodebits * (i - 1), &type);
+			if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
+				lo = 0;
+			else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0)
+				lo = 1;
+			m->delta[0] = lo + 1;
+		}
+		erofs_put_metabuf(&buf);
+		return (0);
+	}
+
+	m->clusterofs = lo;
+	m->delta[0] = 0;
+	big_pcluster =
+	    (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
+	if (!big_pcluster) {
+		nblk = 1;
+		while (i > 0) {
+			--i;
+			lo = decode_compactedbits(lobits, in,
+			    encodebits * i, &type);
+			if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD)
+				i -= lo;
+			if (i >= 0)
+				++nblk;
+		}
+	} else {
+		nblk = 0;
+		while (i > 0) {
+			--i;
+			lo = decode_compactedbits(lobits, in,
+			    encodebits * i, &type);
+			if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+				if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) {
+					--i;
+					nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT;
+					continue;
+				}
+				if (lo <= 1) {
+					erofs_put_metabuf(&buf);
+					return (erofs_corrupt(m->sbi, vi->nid,
+					    "invalid delta %u before lcn %ju",
+					    lo, (uintmax_t)original_lcn));
+				}
+				i -= lo - 2;
+				continue;
+			}
+			++nblk;
+		}
+	}
+	m->pblk = (erofs_blk_t)le32dec(in + packsize - sizeof(uint32_t)) + nblk;
+	erofs_put_metabuf(&buf);
+	m->lcn = original_lcn;
+	return (0);
+}
+
+static int
+z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn,
+    bool lookahead)
+{
+	int err;
+
+	EROFS_DEBUG_ASSERT(m->vi->datalayout ==
+	    EROFS_INODE_COMPRESSED_COMPACT ||
+	    m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL,
+	    ("erofs: data layout %u is not compressed", m->vi->datalayout));
+	if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT)
+		err = z_erofs_load_compact_lcluster(m, lcn, lookahead);
+	else
+		err = z_erofs_load_full_lcluster(m, lcn);
+	if (err != 0)
+		return (err);
+	/* The type is a 2-bit field, so it is always below TYPE_MAX. */
+	if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
+	    m->clusterofs >= (1U << m->vi->z_lclusterbits))
+		return (erofs_corrupt(m->sbi, m->vi->nid,
+		    "clusterofs %u of lcn %ju is out of range", m->clusterofs,
+		    (uintmax_t)lcn));
+	return (0);
+}
+
+static int
+z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
+    unsigned int lookback_distance)
+{
+	uint64_t lcn;
+	int err;
+
+	while (lookback_distance != 0 && m->lcn >= lookback_distance) {
+		lcn = m->lcn - lookback_distance;
+		err = z_erofs_load_lcluster_from_disk(m, lcn, false);
+		if (err != 0)
+			return (err);
+		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+			lookback_distance = m->delta[0];
+			continue;
+		}
+		m->headtype = m->type;
+		m->map->m_la = (lcn << m->vi->z_lclusterbits) + m->clusterofs;
+		return (0);
+	}
+	return (erofs_corrupt(m->sbi, m->vi->nid,
+	    "no head lcluster found looking back from lcn %ju",
+	    (uintmax_t)m->lcn));
+}
+
+static int
+z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m)
+{
+	struct erofs_inode *vi;
+	bool bigpcl1, bigpcl2;
+	uint64_t lcn, lcnpos;
+	int err;
+
+	vi = m->vi;
+	bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
+	bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0;
+	lcn = m->lcn + 1;
+	lcnpos = lcn << vi->z_lclusterbits;
+	if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) ||
+	    ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN ||
+	    m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) ||
+	    lcnpos >= vi->size)
+		m->compressedblks = 1;
+	if (m->compressedblks == 0) {
+		err = z_erofs_load_lcluster_from_disk(m, lcn, false);
+		if (err != 0)
+			return (err);
+		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
+		    m->delta[0] != 1)
+			return (erofs_corrupt(m->sbi, vi->nid,
+			    "lcn %ju: delta %u should be 1 for the compressed "
+			    "block count", (uintmax_t)lcn, m->delta[0]));
+		if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD ||
+		    m->compressedblks == 0)
+			m->compressedblks = 1;
+	}
+	m->map->m_plen = m->compressedblks * m->sbi->block_size;
+	return (0);
+}
+
+static int
+z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
+{
+	struct erofs_inode *vi;
+	struct erofs_map_blocks *map;
+	uint64_t lcn, headlcn, lend, lcnpos;
+	int err;
+
+	vi = m->vi;
+	map = m->map;
+	lcn = m->lcn;
+	headlcn = map->m_la >> vi->z_lclusterbits;
+	for (;;) {
+		lcnpos = lcn << vi->z_lclusterbits;
+		if (lcnpos >= vi->size) {
+			EROFS_DEBUG_ASSERT(map->m_la <= vi->size,
+			    ("erofs: extent at %ju beyond the size %ju",
+			    (uintmax_t)map->m_la, (uintmax_t)vi->size));
+			map->m_llen = vi->size - map->m_la;
+			return (0);
+		}
+		err = z_erofs_load_lcluster_from_disk(m, lcn, true);
+		if (err != 0)
+			return (err);
+		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+			if (m->delta[1] == 0)
+				m->delta[1] = 1;
+		} else {
+			if (lcn != headlcn)
+				break;
+			m->delta[1] = 1;
+		}
+		lcn += m->delta[1];
+	}
+	lend = (lcn << vi->z_lclusterbits) + m->clusterofs;
+	EROFS_DEBUG_ASSERT(lend >= map->m_la,
+	    ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)lend,
+	    (uintmax_t)map->m_la));
+	map->m_llen = lend - map->m_la;
+	return (0);
+}
+
+static inline erofs_off_t
+z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz)
+{
+	return (roundup2(z_erofs_header_pos(vi) +
+	    sizeof(struct z_erofs_map_header), recsz));
+}
+
+static inline erofs_off_t
+z_erofs_extent_record_pos(erofs_off_t table_pos, unsigned int recsz,
+    uint64_t index)
+{
+	return (table_pos + index * recsz);
+}
+
+static int
+z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	int error;
+
+	bzero(ext, sizeof(*ext));
+	error = erofs_read_metadata_range(sbi, vi->nid, pos, recsz, &buf);
+	if (error != 0)
+		return (error);
+	memcpy(ext, buf.data, recsz);
+	erofs_put_metabuf(&buf);
+	return (0);
+}
+
+static uint64_t
+z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz)
+{
+	uint64_t lstart;
+
+	lstart = le32toh(ext->lstart_lo);
+	if (recsz > offsetof(struct z_erofs_extent, lstart_hi))
+		lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32;
+	return (lstart);
+}
+
+static int
+z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map, int flags)
+{
+	bool fragment, ztailpacking;
+	struct z_erofs_maprecorder m;
+	uint64_t initial_lcn, ofs, end;
+	unsigned int endoff;
+	int err;
+
+	fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0;
+	ztailpacking = vi->z_idata_size != 0;
+	bzero(&m, sizeof(m));
+	m.sbi = sbi;
+	m.vi = vi;
+	m.map = map;
+
+	if (vi->size == 0) {
+		map->m_la = 0;
+		map->m_llen = 0;
+		map->m_flags = 0;
+		return (0);
+	}
+	ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ?
+	    vi->size - 1 : map->m_la;
+	if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 &&
+	    vi->z_tailextent_headlcn == 0) {
+		map->m_la = 0;
+		map->m_llen = vi->size;
+		map->m_flags = EROFS_MAP_FRAGMENT;
+		return (0);
+	}
+	initial_lcn = ofs >> vi->z_lclusterbits;
+	endoff = ofs & ((1U << vi->z_lclusterbits) - 1);
+	err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false);
+	if (err != 0)
+		return (err);
+	if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking)
+		vi->z_fragmentoff = m.nextpackoff;
+
+	map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED;
+	end = (m.lcn + 1) << vi->z_lclusterbits;
+	if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
+	    endoff >= m.clusterofs) {
+		m.headtype = m.type;
+		map->m_la = (m.lcn << vi->z_lclusterbits) + m.clusterofs;
+		if (ztailpacking && end > vi->size)
+			end = vi->size;
+	} else {
+		if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
+			end = (m.lcn << vi->z_lclusterbits) + m.clusterofs;
+			map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED;
+			m.delta[0] = 1;
+		}
+		err = z_erofs_extent_lookback(&m, m.delta[0]);
+		if (err != 0)
+			return (err);
+	}
+	if (m.partialref)
+		map->m_flags |= EROFS_MAP_PARTIAL_REF;
+	EROFS_DEBUG_ASSERT(end >= map->m_la,
+	    ("erofs: extent ends at %ju, before its start %ju", (uintmax_t)end,
+	    (uintmax_t)map->m_la));
+	map->m_llen = end - map->m_la;
+
+	if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) {
+		vi->z_tailextent_headlcn = m.lcn;
+		if (fragment &&
+		    vi->datalayout == EROFS_INODE_COMPRESSED_FULL) {
+			/* Both are 32-bit fields of the on-disk index. */
+			vi->z_fragmentoff |= m.pblk << 32;
+		}
+	}
+	if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
+		map->m_flags |= EROFS_MAP_META;
+		map->m_pa = vi->z_fragmentoff;
+		map->m_plen = vi->z_idata_size;
+		/* Same rule as Linux: inline data does not cross a block. */
+		if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen >
+		    sbi->block_size)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "inline pcluster of %ju bytes crosses its block",
+			    (uintmax_t)map->m_plen));
+	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
+		map->m_flags = EROFS_MAP_FRAGMENT;
+	} else {
+		map->m_pa = m.pblk * sbi->block_size;
+		err = z_erofs_get_extent_compressedlen(&m);
+		if (err != 0)
+			return (err);
+	}
+
+	if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) {
+		/*
+		 * Interlaced pclusters are block-aligned on disk.  An inline
+		 * (ztailpacking) pcluster is not, so it can only be shifted,
+		 * whatever the inode advises.
+		 */
+		map->m_algorithmformat =
+		    (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 &&
+		    (map->m_flags & EROFS_MAP_META) == 0 ?
+		    Z_EROFS_COMPRESSION_INTERLACED :
+		    Z_EROFS_COMPRESSION_SHIFTED;
+	} else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) {
+		map->m_algorithmformat = vi->z_algorithmtype[1];
+	} else {
+		map->m_algorithmformat = vi->z_algorithmtype[0];
+	}
+
+	if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0) {
+		err = z_erofs_get_extent_decompressedlen(&m);
+		if (err == 0)
+			map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED;
+		return (err);
+	}
+	return (0);
+}
+
+/*
+ * The extent table is not checked when the inode is loaded: that its
+ * records are sorted and inside the file is fsck.erofs's job, and scanning
+ * it would read a table that grows with the file on every load.  Each
+ * record the search visits is range checked by the read, and a table that
+ * is out of order only makes it return a hole or the wrong extent.
+ */
+static int
+z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct z_erofs_extent ext;
+	unsigned int recsz, bmask;
+	uint8_t fmt;
+	uint64_t cluster_size, extent_idx;
+	erofs_off_t extent_pos, pos, table_pos;
+	uint64_t lend, l, r, mid, pa, la, lstart;
+	bool interlaced, last;
+	int error;
+
+	interlaced =
+	    (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0;
+	recsz = z_erofs_extent_recsize(vi->z_advise);
+	table_pos = z_erofs_extent_table_pos(vi, recsz);
+	pos = table_pos;
+	bmask = sbi->block_size - 1;
+	lend = vi->size;
+	cluster_size = 1ULL << vi->z_lclusterbits;
+	map->m_flags = 0;
+
+	if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) {
+		if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) {
+			error = erofs_read_metadata_range(sbi, vi->nid, pos,
+			    sizeof(uint64_t), &buf);
+			if (error != 0)
+				return (error);
+			pa = le64dec(buf.data);
+			erofs_put_metabuf(&buf);
+			pos += sizeof(uint64_t);
+			lstart = 0;
+			extent_idx = 0;
+		} else {
+			lstart = rounddown2(map->m_la, cluster_size);
+			extent_idx = lstart >> vi->z_lclusterbits;
+			pa = EROFS_NULL_ADDR;
+		}
+		for (;;) {
+			EROFS_DEBUG_ASSERT(extent_idx < vi->z_extents,
+			    ("erofs: extent %ju of %ju", (uintmax_t)extent_idx,
+			    (uintmax_t)vi->z_extents));
+			extent_pos = z_erofs_extent_record_pos(pos, recsz,
+			    extent_idx);
+			error = z_erofs_read_extent(sbi, vi, extent_pos, recsz,
+			    &ext);
+			if (error != 0)
+				return (error);
+			map->m_plen = le32toh(ext.plen);
+			if (pa != EROFS_NULL_ADDR) {
+				map->m_pa = pa;
+				pa += map->m_plen & Z_EROFS_EXTENT_PLEN_MASK;
+			} else {
+				map->m_pa = le32toh(ext.pstart_lo);
+			}
+			extent_idx++;
+			lstart += cluster_size;
+			if (lstart > map->m_la)
+				break;
+		}
+		last = lstart >= roundup2(lend, cluster_size);
+		lend = MIN(lstart, lend);
+		lstart -= cluster_size;
+	} else {
+		lstart = lend;
+		for (l = 0, r = vi->z_extents; l < r;) {
+			mid = l + (r - l) / 2;
+			extent_pos = z_erofs_extent_record_pos(table_pos, recsz,
+			    mid);
+			error = z_erofs_read_extent(sbi, vi, extent_pos,
+			    recsz, &ext);
+			if (error != 0)
+				return (error);
+			la = z_erofs_extent_lstart(&ext, recsz);
+			pa = le32toh(ext.pstart_lo) |
+			    ((uint64_t)le32toh(ext.pstart_hi) << 32);
+			if (la > map->m_la) {
+				r = mid;
+				/*
+				 * The records are not compared with each
+				 * other, but this one must not start past the
+				 * next one or the size: m_llen stays within the
+				 * file.
+				 */
+				if (la > lend)
+					return (erofs_corrupt(sbi, vi->nid,
+					    "extent %ju starts at %ju, past "
+					    "the following extent or the size "
+					    "%ju",
+					    (uintmax_t)mid, (uintmax_t)la,
+					    (uintmax_t)lend));
+				lend = la;
+			} else {
+				l = mid + 1;
+				if (map->m_la == la)
+					r = MIN(l + 1, r);
+				lstart = la;
+				map->m_plen = le32toh(ext.plen);
+				map->m_pa = pa;
+			}
+		}
+		last = l >= vi->z_extents;
+	}
+
+	if (lstart < lend) {
+		map->m_la = lstart;
+		if (last &&
+		    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) {
+			map->m_flags = EROFS_MAP_FRAGMENT;
+			if (recsz > offsetof(struct z_erofs_extent,
+			    pstart_lo)) {
+				vi->z_fragmentoff = map->m_plen |
+				    (map->m_pa << 32);
+			} else {
+				vi->z_fragmentoff = map->m_plen;
+			}
+		} else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) {
+			map->m_flags = EROFS_MAP_MAPPED;
+			fmt = (uint8_t)(map->m_plen >>
+			    Z_EROFS_EXTENT_PLEN_FMT_BIT);
+			if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0)
+				map->m_flags |= EROFS_MAP_PARTIAL_REF;
+			map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK;
+			if (fmt != 0) {
+				if (fmt - 1 >= Z_EROFS_COMPRESSION_MAX) {
+					erofs_err(sbi,
+					    "unsupported compression "
+					    "algorithm %u @ nid %ju", fmt - 1,
+					    (uintmax_t)vi->nid);
+					return (EOPNOTSUPP);
+				}
+				map->m_algorithmformat = (uint8_t)(fmt - 1);
+			} else if (interlaced &&
+			    ((map->m_pa | map->m_plen) & bmask) == 0)
+				map->m_algorithmformat =
+				    Z_EROFS_COMPRESSION_INTERLACED;
+			else
+				map->m_algorithmformat =
+				    Z_EROFS_COMPRESSION_SHIFTED;
+		}
+	}
+	map->m_llen = lend - map->m_la;
+	return (0);
+}
+
+int
+z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	struct z_erofs_map_header *h;
+	struct erofs_map_blocks map;
+	uint64_t cluster_size, raw;
+	erofs_off_t pos;
+	unsigned int recsz;
+	int err;
+
+	if (vi->z_initialized)
+		return (0);
+	pos = z_erofs_header_pos(vi);
+	err = erofs_read_metadata_range(sbi, vi->nid, pos, sizeof(*h), &buf);
+	if (err != 0)
+		return (err);
+	h = buf.data;
+	if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) {
+		if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) {
+			erofs_put_metabuf(&buf);
+			return (erofs_corrupt(sbi, vi->nid,
+			    "fragment inode, but the image has no packed "
+			    "inode"));
+		}
+		raw = le64dec(h);
+		vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
+		vi->z_fragmentoff = raw ^ (1ULL << 63);
+		vi->z_tailextent_headlcn = 0;
+		erofs_put_metabuf(&buf);
+		vi->z_initialized = true;
+		return (0);
+	}
+
+	vi->z_advise = le16toh(h->h_advise);
+	vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15);
+	/* The lcluster size is used as a 32-bit shift count and size below. */
+	if (vi->z_lclusterbits >= 31) {
+		erofs_put_metabuf(&buf);
+		return (erofs_corrupt(sbi, vi->nid,
+		    "lcluster bits %u are too large", vi->z_lclusterbits));
+	}
+	if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL &&
+	    (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) {
+		recsz = z_erofs_extent_recsize(vi->z_advise);
+		if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) {
+			cluster_size = 1ULL << vi->z_lclusterbits;
+			vi->z_extents = roundup2(vi->size, cluster_size) >>
+			    vi->z_lclusterbits;
+		} else {
+			vi->z_extents = le32toh(h->h_extents_lo) |
+			    ((uint64_t)le16toh(h->h_extents_hi) << 32);
+		}
+		erofs_put_metabuf(&buf);
+		if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 &&
+		    (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0))
+			return (erofs_corrupt(sbi, vi->nid,
+			    "fragment inode, but the image has no packed "
+			    "inode"));
+		vi->z_initialized = true;
+		return (0);
+	}
+	vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15);
+	vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4);
+	/*
+	 * Both HEAD formats are on-disk algorithm IDs.  IDs at or above
+	 * Z_EROFS_COMPRESSION_MAX would alias the runtime-only SHIFTED and
+	 * INTERLACED formats, so refuse the inode instead of decoding it as
+	 * uncompressed data.
+	 */
+	if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
+	    vi->z_algorithmtype[1] >= Z_EROFS_COMPRESSION_MAX) {
+		erofs_put_metabuf(&buf);
+		erofs_err(sbi, "unsupported compression algorithm %u/%u "
+		    "@ nid %ju", vi->z_algorithmtype[0], vi->z_algorithmtype[1],
+		    (uintmax_t)vi->nid);
+		return (EOPNOTSUPP);
+	}
+	if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0)
+		vi->z_fragmentoff = le32toh(h->h_fragmentoff);
+	else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0)
+		vi->z_idata_size = le16toh(h->h_idata_size);
+	erofs_put_metabuf(&buf);
+
+	/* Same rule as Linux: big pcluster advise needs the feature flag. */
+	if (!erofs_sb_has_big_pcluster(sbi) &&
+	    (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
+	    Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0)
+		return (erofs_corrupt(sbi, vi->nid,
+		    "big pcluster advise %#x, but the image has no big "
+		    "pclusters",
+		    vi->z_advise));
+	/*
+	 * Same rule as Linux: compact indexes set both big-pcluster bits or
+	 * neither.
+	 */
+	if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT &&
+	    (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) !=
+	    ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0)))
+		return (erofs_corrupt(sbi, vi->nid,
+		    "compact inode with inconsistent big pcluster advise %#x",
+		    vi->z_advise));
+	if (vi->z_idata_size != 0 ||
+	    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) {
+		if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 &&
+		    (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0))
+			return (erofs_corrupt(sbi, vi->nid,
+			    "fragment inode, but the image has no packed "
+			    "inode"));
+		bzero(&map, sizeof(map));
+		err = z_erofs_map_blocks_fo(sbi, vi, &map,
+		    EROFS_GET_BLOCKS_FINDTAIL);
+		if (err != 0)
+			return (err);
+	}
+	vi->z_initialized = true;
+	return (0);
+}
+
+static int
+z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) {
+		/*
+		 * The fragment is read from the packed inode, and that read
+		 * checks the packed inode and the range (z_erofs_do_read(),
+		 * erofs_read_data()).
+		 */
+		EROFS_DEBUG_ASSERT((map->m_flags &
+		    (EROFS_MAP_MAPPED | EROFS_MAP_META)) == 0,
+		    ("erofs: fragment map with flags %#x",
+		    (unsigned int)map->m_flags));
+		return (0);
+	}
+	if ((map->m_flags & EROFS_MAP_MAPPED) == 0)
+		return (0);
+	EROFS_DEBUG_ASSERT(map->m_la < vi->size,
+	    ("erofs: extent at %ju, beyond the size %ju",
+	    (uintmax_t)map->m_la, (uintmax_t)vi->size));
+	/*
+	 * On-disk algorithm IDs are validated where they are read, so only
+	 * the runtime formats can reach this point.
+	 */
+	EROFS_DEBUG_ASSERT(map->m_algorithmformat <
+	    Z_EROFS_COMPRESSION_RUNTIME_MAX,
+	    ("erofs: unvalidated algorithm format %u", map->m_algorithmformat));
+	if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) {
+		if ((sbi->available_compr_algs &
+		    (1U << map->m_algorithmformat)) == 0)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "algorithm %u is not in the image's algorithm list",
+			    map->m_algorithmformat));
+		/*
+		 * A fully mapped compressed extent does not decode to fewer
+		 * bytes than it occupies, unless the end of the file cuts it
+		 * short.
+		 */
+		if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen &&
+		    map->m_llen < vi->size - map->m_la)
+			return (erofs_corrupt(sbi, vi->nid,
+			    "extent at %ju decompresses to %ju bytes from %ju "
+			    "compressed", (uintmax_t)map->m_la,
+			    (uintmax_t)map->m_llen, (uintmax_t)map->m_plen));
+	} else if (map->m_llen > map->m_plen) {
+		/* Stored data is not longer than the pcluster holding it. */
+		return (erofs_corrupt(sbi, vi->nid,
+		    "uncompressed extent at %ju has length %ju > stored %ju",
+		    (uintmax_t)map->m_la, (uintmax_t)map->m_llen,
+		    (uintmax_t)map->m_plen));
+	}
+	if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE ||
+	    map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE) {
+		erofs_err(sbi, "extent at %ju is too large (%ju bytes encoded, "
+		    "%ju decoded) @ nid %ju", (uintmax_t)map->m_la,
+		    (uintmax_t)map->m_plen, (uintmax_t)map->m_llen,
+		    (uintmax_t)vi->nid);
+		return (EOPNOTSUPP);
+	}
+	return (0);
+}
+
+int
+z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map)
+{
+	int err;
+
+	if (map->m_la >= vi->size) {
+		map->m_llen = map->m_la - vi->size + 1;
+		map->m_la = vi->size;
+		map->m_flags = 0;
+		return (0);
+	}
+	err = z_erofs_fill_inode(sbi, vi);
+	if (err == 0) {
+		if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL &&
+		    (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0)
+			err = z_erofs_map_blocks_ext(sbi, vi, map);
+		else
+			err = z_erofs_map_blocks_fo(sbi, vi, map,
+			    EROFS_GET_BLOCKS_FIEMAP);
+	}
+	if (err == 0)
+		err = z_erofs_map_sanity_check(sbi, vi, map);
+	if (err != 0)
+		map->m_llen = 0;
+	return (err);
+}

zutil.c

@@ -0,0 +1,665 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* FreeBSD stream context pool and compression runtime accounting. */
+
+#include <sys/param.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/sysctl.h>
+
+#include <vm/uma.h>
+
+#include "internal.h"
+
+#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024)
+#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024)
+#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024)
+#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U)
+#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U)
+#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U)
+#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U)
+
+static unsigned long erofs_stream_mount_budget =
+    EROFS_STREAM_MOUNT_HARD_BUDGET;
+static unsigned long erofs_stream_global_budget =
+    EROFS_STREAM_GLOBAL_HARD_BUDGET;
+static int erofs_stream_mount_contexts =
+    EROFS_STREAM_MOUNT_HARD_CONTEXTS;
+static int erofs_stream_global_contexts_limit =
+    EROFS_STREAM_GLOBAL_HARD_CONTEXTS;
+static int erofs_stream_mount_cached =
+    EROFS_STREAM_MOUNT_CACHED_PER_CODEC;
+static int erofs_stream_global_cached =
+    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC;
+
+TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget",
+    &erofs_stream_mount_budget);
+TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget",
+    &erofs_stream_global_budget);
+TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts",
+    &erofs_stream_mount_contexts);
+TUNABLE_INT("vfs.erofs.stream_pool.global_contexts",
+    &erofs_stream_global_contexts_limit);
+TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec",
+    &erofs_stream_mount_cached);
+TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec",
+    &erofs_stream_global_cached);
+
+static struct mtx erofs_stream_lock;
+static struct cv erofs_stream_global_cv;
+static unsigned int erofs_stream_global_waiters;
+static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts =
+    LIST_HEAD_INITIALIZER(erofs_stream_mounts);
+static uma_zone_t erofs_stream_zone;
+static eventhandler_tag erofs_stream_lowmem_tag;
+static size_t erofs_stream_global_resident;
+static unsigned int erofs_stream_global_contexts;
+static unsigned int erofs_stream_global_cached_by_codec[
+    Z_EROFS_COMPRESSION_MAX];
+static unsigned long erofs_stream_cached_contexts;
+static unsigned long erofs_stream_borrowed_contexts;
+static unsigned long erofs_stream_idle_contexts;
+static unsigned long erofs_stream_creations;
+static unsigned long erofs_stream_reuses;
+static unsigned long erofs_stream_fallbacks;
+static unsigned long erofs_stream_exhaustions;
+static unsigned long erofs_stream_waits;
+static unsigned long erofs_stream_destroys;
+static unsigned long erofs_stream_reclaims;
+
+MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool",
+    MTX_DEF);
+
+SYSCTL_DECL(_vfs_erofs);
+SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool,
+    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD,
+    &erofs_stream_mount_budget, 0, "Effective per-mount byte budget");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD,
+    &erofs_stream_global_budget, 0, "Effective global byte budget");
+SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD,
+    &erofs_stream_mount_contexts, 0,
+    "Effective per-mount, per-algorithm context limit");
+SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit,
+    CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0,
+    "Effective global context limit");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD,
+    &erofs_stream_global_resident, 0, "Currently charged global bytes");
+SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD,
+    &erofs_stream_global_contexts, 0, "Currently allocated contexts");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD,
+    &erofs_stream_cached_contexts, 0, "Currently cacheable contexts");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD,
+    &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD,
+    &erofs_stream_idle_contexts, 0, "Currently idle contexts");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD,
+    &erofs_stream_creations, 0, "Successful context creations");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD,
+    &erofs_stream_reuses, 0, "Idle context acquisitions");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD,
+    &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD,
+    &erofs_stream_exhaustions, 0, "Unsatisfiable limit or budget rejections");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, waits, CTLFLAG_RD,
+    &erofs_stream_waits, 0, "Sleeps waiting for a context or budget release");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD,
+    &erofs_stream_destroys, 0, "Destroyed contexts");
+SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD,
+    &erofs_stream_reclaims, 0, "Low-memory idle context reclaims");
+
+struct erofs_stream_alloc_header {
+	struct erofs_stream_ctx *ctx;
+	size_t bytes;
+};
+
+static bool
+erofs_stream_mount_budget_available_locked(const struct erofs_sb_info *sbi,
+    size_t bytes)
+{
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	return (bytes <= erofs_stream_mount_budget &&
+	    sbi->stream_pool.resident_bytes <= erofs_stream_mount_budget -
+	    bytes);
+}
+
+static bool
+erofs_stream_global_budget_available_locked(size_t bytes)
+{
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	return (bytes <= erofs_stream_global_budget &&
+	    erofs_stream_global_resident <= erofs_stream_global_budget - bytes);
+}
+
+static bool
+erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes)
+{
+	struct erofs_stream_pool *pool;
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	pool = &sbi->stream_pool;
+	if (!erofs_stream_mount_budget_available_locked(sbi, bytes) ||
+	    !erofs_stream_global_budget_available_locked(bytes))
+		return (false);
+	pool->resident_bytes += bytes;
+	erofs_stream_global_resident += bytes;
+	return (true);
+}
+
+/*
+ * Wake threads sleeping in z_erofs_stream_ctx_get() after a context or byte
+ * budget was released.  Mount-level sleepers and the unmount drain share the
+ * per-mount cv; sleepers blocked on a global limit use the global cv.
+ */
+static void
+erofs_stream_wake_locked(struct erofs_sb_info *sbi)
+{
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	if (sbi->stream_pool.waiters != 0 || sbi->stream_pool.closing)
+		cv_broadcast(&sbi->stream_pool.cv);
+	if (erofs_stream_global_waiters != 0)
+		cv_broadcast(&erofs_stream_global_cv);
+}
+
+static void
+erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes)
+{
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	KASSERT(bytes <= sbi->stream_pool.resident_bytes &&
+	    bytes <= erofs_stream_global_resident,
+	    ("erofs stream pool byte accounting underflow"));
+	sbi->stream_pool.resident_bytes -= bytes;
+	erofs_stream_global_resident -= bytes;
+}
+
+static struct erofs_stream_ctx *
+erofs_stream_take_idle_locked(struct erofs_sb_info *sbi)
+{
+	struct erofs_stream_pool_codec *codec;
+	struct erofs_stream_ctx *ctx;
+	unsigned int algorithm;
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
+		codec = &sbi->stream_pool.codec[algorithm];
+		ctx = STAILQ_FIRST(&codec->idle);
+		if (ctx == NULL)
+			continue;
+		STAILQ_REMOVE_HEAD(&codec->idle, link);
+		KASSERT(codec->idle_count != 0 &&
+		    erofs_stream_idle_contexts != 0,
+		    ("erofs stream idle accounting underflow"));
+		--codec->idle_count;
+		--erofs_stream_idle_contexts;
+		++codec->borrowed;
+		++erofs_stream_borrowed_contexts;
+		return (ctx);
+	}
+	return (NULL);
+}
+
+/*
+ * Detach one idle context so that the caller can destroy it after dropping
+ * the lock.  Idle contexts of this mount are preferred; other mounts are only
+ * considered when the limit being hit is a global one.
+ */
+static struct erofs_stream_ctx *
+erofs_stream_take_reclaimable_locked(struct erofs_sb_info *sbi, bool global)
+{
+	struct erofs_stream_ctx *ctx;
+	struct erofs_sb_info *other;
+
+	ctx = erofs_stream_take_idle_locked(sbi);
+	if (ctx != NULL || !global)
+		return (ctx);
+	LIST_FOREACH(other, &erofs_stream_mounts, stream_pool_link) {
+		ctx = erofs_stream_take_idle_locked(other);
+		if (ctx != NULL)
+			break;
+	}
+	return (ctx);
+}
+
+static void erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx);
+
+/*
+ * Charging never sleeps for budget: the caller already owns a context, and
+ * waiting here would hold part of the budget while waiting for the rest.  Idle
+ * contexts are the only budget that can be returned immediately, so reclaim
+ * them first and fail with ENOMEM only when the budget is truly exhausted.
+ */
+int
+z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes)
+{
+	struct erofs_stream_ctx *idle;
+	bool reserved;
+
+	if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) {
+		ctx->allocation_failed = true;
+		mtx_lock(&erofs_stream_lock);
+		++erofs_stream_exhaustions;
+		mtx_unlock(&erofs_stream_lock);
+		return (ENOMEM);
+	}
+	mtx_lock(&erofs_stream_lock);
+	for (;;) {
+		if (ctx->charged_bytes > SIZE_MAX - bytes ||
+		    bytes > erofs_stream_mount_budget ||
+		    bytes > erofs_stream_global_budget) {
+			reserved = false;
+			break;
+		}
+		reserved = erofs_stream_reserve_locked(ctx->sbi, bytes);
+		if (reserved)
+			break;
+		idle = erofs_stream_take_reclaimable_locked(ctx->sbi,
+		    erofs_stream_mount_budget_available_locked(ctx->sbi,
+		    bytes));
+		if (idle == NULL)
+			break;
+		mtx_unlock(&erofs_stream_lock);
+		erofs_stream_ctx_destroy(idle);
+		mtx_lock(&erofs_stream_lock);
+	}
+	if (reserved)
+		ctx->charged_bytes += bytes;
+	else
+		++erofs_stream_exhaustions;
+	mtx_unlock(&erofs_stream_lock);
+	if (!reserved) {
+		ctx->allocation_failed = true;
+		return (ENOMEM);
+	}
+	return (0);
+}
+
+void
+z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes)
+{
+
+	mtx_lock(&erofs_stream_lock);
+	KASSERT(bytes <= ctx->charged_bytes,
+	    ("erofs stream context byte accounting underflow"));
+	ctx->charged_bytes -= bytes;
+	erofs_stream_release_locked(ctx->sbi, bytes);
+	erofs_stream_wake_locked(ctx->sbi);
+	mtx_unlock(&erofs_stream_lock);
+}
+
+void *
+z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes)
+{
+	struct erofs_stream_alloc_header *header;
+	size_t total;
+
+	if (__builtin_add_overflow(bytes, sizeof(*header), &total) ||
+	    z_erofs_stream_ctx_charge(ctx, total) != 0)
+		return (NULL);
+	header = malloc(total, M_EROFS, M_WAITOK | M_ZERO);
+	header->ctx = ctx;
+	header->bytes = total;
+	return (header + 1);
+}
+
+void
+z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address)
+{
+	struct erofs_stream_alloc_header *header;
+	size_t bytes;
+
+	if (address == NULL)
+		return;
+	header = (struct erofs_stream_alloc_header *)address - 1;
+	KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch"));
+	bytes = header->bytes;
+	free(header, M_EROFS);
+	z_erofs_stream_ctx_uncharge(ctx, bytes);
+}
+
+static void
+erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx)
+{
+	struct erofs_stream_pool_codec *codec;
+	struct erofs_sb_info *sbi;
+	bool cached;
+	uint8_t algorithm;
+
+	sbi = ctx->sbi;
+	algorithm = ctx->algorithm;
+	cached = ctx->cached;
+	ctx->fini(ctx);
+	KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE,
+	    ("erofs stream backend allocation leaked"));
+	uma_zfree(erofs_stream_zone, ctx);
+
+	mtx_lock(&erofs_stream_lock);
+	codec = &sbi->stream_pool.codec[algorithm];
+	KASSERT(codec->contexts != 0 && codec->borrowed != 0 &&
+	    erofs_stream_global_contexts != 0 &&
+	    erofs_stream_borrowed_contexts != 0,
+	    ("erofs stream context accounting underflow"));
+	--codec->contexts;
+	--codec->borrowed;
+	--erofs_stream_global_contexts;
+	--erofs_stream_borrowed_contexts;
+	if (cached) {
+		KASSERT(codec->cached != 0 &&
+		    erofs_stream_global_cached_by_codec[algorithm] != 0 &&
+		    erofs_stream_cached_contexts != 0,
+		    ("erofs stream cache accounting underflow"));
+		--codec->cached;
+		--erofs_stream_global_cached_by_codec[algorithm];
+		--erofs_stream_cached_contexts;
+	}
+	erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE);
+	++erofs_stream_destroys;
+	erofs_stream_wake_locked(sbi);
+	mtx_unlock(&erofs_stream_lock);
+}
+
+/*
+ * Sleep until a context or budget is released.  Limits that only this mount
+ * can release are waited for on the mount cv; anything involving a global
+ * limit is waited for on the global cv, which every release also signals.
+ */
+static void
+erofs_stream_wait_locked(struct erofs_sb_info *sbi, bool global)
+{
+	struct erofs_stream_pool *pool;
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	pool = &sbi->stream_pool;
+	++pool->waiters;
+	++erofs_stream_waits;
+	if (global) {
+		++erofs_stream_global_waiters;
+		cv_wait(&erofs_stream_global_cv, &erofs_stream_lock);
+		--erofs_stream_global_waiters;
+	} else {
+		cv_wait(&pool->cv, &erofs_stream_lock);
+	}
+	KASSERT(pool->waiters != 0, ("erofs stream waiter underflow"));
+	--pool->waiters;
+	if (pool->closing)
+		cv_broadcast(&pool->cv);
+}
+
+int
+z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm,
+    size_t context_size, erofs_stream_ctx_init_t *init,
+    erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp)
+{
+	struct erofs_stream_pool_codec *codec;
+	struct erofs_stream_ctx *ctx;
+	bool cached, global_full, mount_full, reserved __diagused;
+	int error;
+
+	*ctxp = NULL;
+	if (algorithm == Z_EROFS_COMPRESSION_LZ4 ||
+	    algorithm >= Z_EROFS_COMPRESSION_MAX ||
+	    context_size < sizeof(*ctx) ||
+	    context_size > EROFS_STREAM_CTX_WRAPPER_SIZE)
+		return (EINVAL);
+	mtx_lock(&erofs_stream_lock);
+	for (;;) {
+		if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) {
+			mtx_unlock(&erofs_stream_lock);
+			return (ENXIO);
+		}
+		codec = &sbi->stream_pool.codec[algorithm];
+		ctx = STAILQ_FIRST(&codec->idle);
+		if (ctx != NULL) {
+			STAILQ_REMOVE_HEAD(&codec->idle, link);
+			KASSERT(codec->idle_count != 0 &&
+			    erofs_stream_idle_contexts != 0,
+			    ("erofs stream idle accounting underflow"));
+			--codec->idle_count;
+			--erofs_stream_idle_contexts;
+			++codec->borrowed;
+			++erofs_stream_borrowed_contexts;
+			++erofs_stream_reuses;
+			ctx->allocation_failed = false;
+			mtx_unlock(&erofs_stream_lock);
+			*ctxp = ctx;
+			return (0);
+		}
+		/*
+		 * Limits that no release can ever satisfy are configuration
+		 * errors; waiting for them would never end.
+		 */
+		if (erofs_stream_mount_contexts == 0 ||
+		    erofs_stream_global_contexts_limit == 0 ||
+		    EROFS_STREAM_CTX_WRAPPER_SIZE > erofs_stream_mount_budget ||
+		    EROFS_STREAM_CTX_WRAPPER_SIZE >
+		    erofs_stream_global_budget) {
+			++erofs_stream_exhaustions;
+			mtx_unlock(&erofs_stream_lock);
+			return (ENOMEM);
+		}
+		mount_full = codec->contexts >=
+		    (unsigned int)erofs_stream_mount_contexts;
+		global_full = erofs_stream_global_contexts >=
+		    (unsigned int)erofs_stream_global_contexts_limit ||
+		    !erofs_stream_global_budget_available_locked(
+		    EROFS_STREAM_CTX_WRAPPER_SIZE);
+		if (!mount_full && !global_full &&
+		    erofs_stream_mount_budget_available_locked(sbi,
+		    EROFS_STREAM_CTX_WRAPPER_SIZE))
+			break;
+		/*
+		 * Contexts of this codec are all borrowed when mount_full, so
+		 * only a release can help.  Otherwise idle contexts of other
+		 * codecs (or mounts, for a global limit) hold the resource and
+		 * are never released unless reclaimed here.
+		 */
+		if (!mount_full) {
+			ctx = erofs_stream_take_reclaimable_locked(sbi,
+			    global_full);
+			if (ctx != NULL) {
+				mtx_unlock(&erofs_stream_lock);
+				erofs_stream_ctx_destroy(ctx);
+				mtx_lock(&erofs_stream_lock);
+				continue;
+			}
+		}
+		erofs_stream_wait_locked(sbi, !mount_full && global_full);
+	}
+	reserved = erofs_stream_reserve_locked(sbi,
+	    EROFS_STREAM_CTX_WRAPPER_SIZE);
+	KASSERT(reserved, ("erofs stream wrapper reservation failed"));
+	cached = codec->cached < (unsigned int)erofs_stream_mount_cached &&
+	    erofs_stream_global_cached_by_codec[algorithm] <
+	    (unsigned int)erofs_stream_global_cached;
+	++codec->contexts;
+	++codec->borrowed;
+	++erofs_stream_global_contexts;
+	++erofs_stream_borrowed_contexts;
+	if (cached) {
+		++codec->cached;
+		++erofs_stream_global_cached_by_codec[algorithm];
+		++erofs_stream_cached_contexts;
+	} else {
+		++erofs_stream_fallbacks;
+	}
+	mtx_unlock(&erofs_stream_lock);
+
+	ctx = uma_zalloc(erofs_stream_zone, M_WAITOK | M_ZERO);
+	ctx->sbi = sbi;
+	ctx->fini = fini;
+	ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE;
+	ctx->algorithm = algorithm;
+	ctx->cached = cached;
+	error = init(ctx);
+	if (error != 0) {
+		erofs_stream_ctx_destroy(ctx);
+		return (error > 0 ? error : EIO);
+	}
+	mtx_lock(&erofs_stream_lock);
+	++erofs_stream_creations;
+	mtx_unlock(&erofs_stream_lock);
+	*ctxp = ctx;
+	return (0);
+}
+
+void
+z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable)
+{
+	struct erofs_stream_pool_codec *codec;
+
+	mtx_lock(&erofs_stream_lock);
+	codec = &ctx->sbi->stream_pool.codec[ctx->algorithm];
+	if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) {
+		KASSERT(codec->borrowed != 0 &&
+		    erofs_stream_borrowed_contexts != 0,
+		    ("erofs stream borrowed accounting underflow"));
+		--codec->borrowed;
+		--erofs_stream_borrowed_contexts;
+		STAILQ_INSERT_HEAD(&codec->idle, ctx, link);
+		++codec->idle_count;
+		++erofs_stream_idle_contexts;
+		erofs_stream_wake_locked(ctx->sbi);
+		mtx_unlock(&erofs_stream_lock);
+		return;
+	}
+	mtx_unlock(&erofs_stream_lock);
+	erofs_stream_ctx_destroy(ctx);
+}
+
+static bool
+erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool)
+{
+	unsigned int algorithm;
+
+	mtx_assert(&erofs_stream_lock, MA_OWNED);
+	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
+		if (pool->codec[algorithm].contexts != 0)
+			return (false);
+	}
+	return (true);
+}
+
+void
+z_erofs_stream_pool_init(struct erofs_sb_info *sbi)
+{
+	unsigned int algorithm;
+
+	bzero(&sbi->stream_pool, sizeof(sbi->stream_pool));
+	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm)
+		STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle);
+	cv_init(&sbi->stream_pool.cv, "erofs stream drain");
+	mtx_lock(&erofs_stream_lock);
+	sbi->stream_pool_initialized = true;
+	LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link);
+	mtx_unlock(&erofs_stream_lock);
+}
+
+void
+z_erofs_stream_pool_fini(struct erofs_sb_info *sbi)
+{
+	struct erofs_stream_ctx *ctx;
+
+	if (!sbi->stream_pool_initialized)
+		return;
+	mtx_lock(&erofs_stream_lock);
+	sbi->stream_pool.closing = true;
+	erofs_stream_wake_locked(sbi);
+	for (;;) {
+		ctx = erofs_stream_take_idle_locked(sbi);
+		if (ctx == NULL)
+			break;
+		mtx_unlock(&erofs_stream_lock);
+		erofs_stream_ctx_destroy(ctx);
+		mtx_lock(&erofs_stream_lock);
+	}
+	while (!erofs_stream_pool_empty_locked(&sbi->stream_pool) ||
+	    sbi->stream_pool.waiters != 0)
+		cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock);
+	KASSERT(sbi->stream_pool.resident_bytes == 0,
+	    ("erofs stream mount bytes remain at unmount"));
+	LIST_REMOVE(sbi, stream_pool_link);
+	sbi->stream_pool_initialized = false;
+	mtx_unlock(&erofs_stream_lock);
+	cv_destroy(&sbi->stream_pool.cv);
+}
+
+static void
+erofs_stream_pool_lowmem(void *arg __unused, int howto __unused)
+{
+	struct erofs_stream_ctx *ctx;
+	struct erofs_sb_info *sbi;
+
+	for (;;) {
+		ctx = NULL;
+		mtx_lock(&erofs_stream_lock);
+		LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) {
+			ctx = erofs_stream_take_idle_locked(sbi);
+			if (ctx != NULL) {
+				++erofs_stream_reclaims;
+				break;
+			}
+		}
+		mtx_unlock(&erofs_stream_lock);
+		if (ctx == NULL)
+			break;
+		erofs_stream_ctx_destroy(ctx);
+	}
+}
+
+static void
+erofs_stream_pool_global_init(void *arg __unused)
+{
+	erofs_stream_mount_budget = MIN(erofs_stream_mount_budget,
+	    EROFS_STREAM_MOUNT_HARD_BUDGET);
+	erofs_stream_global_budget = MIN(erofs_stream_global_budget,
+	    EROFS_STREAM_GLOBAL_HARD_BUDGET);
+	erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts,
+	    (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS));
+	erofs_stream_global_contexts_limit = MAX(0, MIN(
+	    erofs_stream_global_contexts_limit,
+	    (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS));
+	erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached,
+	    (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC));
+	erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached,
+	    (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC));
+	cv_init(&erofs_stream_global_cv, "erofs stream wait");
+	/*
+	 * The context count is bounded by erofs_stream_global_contexts_limit;
+	 * a UMA item limit would only add a second, unrelated place to sleep.
+	 */
+	erofs_stream_zone = uma_zcreate("erofs stream ctx",
+	    EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL,
+	    UMA_ALIGN_PTR, 0);
+	KASSERT(erofs_stream_zone != NULL,
+	    ("cannot create erofs stream UMA zone"));
+	uma_zone_set_maxcache(erofs_stream_zone,
+	    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3);
+	erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
+	    erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT);
+}
+
+static void
+erofs_stream_pool_global_fini(void *arg __unused)
+{
+	if (erofs_stream_lowmem_tag != NULL)
+		EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag);
+	KASSERT(LIST_EMPTY(&erofs_stream_mounts),
+	    ("erofs stream mounts remain at unload"));
+	KASSERT(erofs_stream_global_contexts == 0 &&
+	    erofs_stream_global_waiters == 0 &&
+	    erofs_stream_global_resident == 0 &&
+	    erofs_stream_borrowed_contexts == 0 &&
+	    erofs_stream_idle_contexts == 0 &&
+	    erofs_stream_cached_contexts == 0,
+	    ("erofs stream resources remain at unload"));
+	uma_zdestroy(erofs_stream_zone);
+	cv_destroy(&erofs_stream_global_cv);
+}
+
+SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST,
+    erofs_stream_pool_global_init, NULL);
+SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST,
+    erofs_stream_pool_global_fini, NULL);