erofs-freebsd-out-tree/super.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/_maxphys.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/eventhandler.h>
#include <sys/fcntl.h>
#include <sys/fnv_hash.h>
#include <sys/gsb_crc32.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mount.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/vnode.h>

#include <geom/geom.h>
#include <geom/geom_vfs.h>
#include <vm/uma.h>

#include "internal.h"
#include "xattr.h"

MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem");

static const uint32_t erofs_crc32c_seed = 0x5045b54aU;

#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024)
#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024)
#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024)
#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U)
#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U)
#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U)
#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U)

static unsigned long erofs_stream_mount_budget =
    EROFS_STREAM_MOUNT_HARD_BUDGET;
static unsigned long erofs_stream_global_budget =
    EROFS_STREAM_GLOBAL_HARD_BUDGET;
static int erofs_stream_mount_contexts =
    EROFS_STREAM_MOUNT_HARD_CONTEXTS;
static int erofs_stream_global_contexts_limit =
    EROFS_STREAM_GLOBAL_HARD_CONTEXTS;
static int erofs_stream_mount_cached =
    EROFS_STREAM_MOUNT_CACHED_PER_CODEC;
static int erofs_stream_global_cached =
    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC;

TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget",
    &erofs_stream_mount_budget);
TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget",
    &erofs_stream_global_budget);
TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts",
    &erofs_stream_mount_contexts);
TUNABLE_INT("vfs.erofs.stream_pool.global_contexts",
    &erofs_stream_global_contexts_limit);
TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec",
    &erofs_stream_mount_cached);
TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec",
    &erofs_stream_global_cached);

static struct mtx erofs_stream_lock;
static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts =
    LIST_HEAD_INITIALIZER(erofs_stream_mounts);
static uma_zone_t erofs_stream_zone;
static eventhandler_tag erofs_stream_lowmem_tag;
static size_t erofs_stream_global_resident;
static unsigned int erofs_stream_global_contexts;
static unsigned int erofs_stream_global_cached_by_codec[
    Z_EROFS_COMPRESSION_MAX];
static unsigned long erofs_stream_cached_contexts;
static unsigned long erofs_stream_borrowed_contexts;
static unsigned long erofs_stream_idle_contexts;
static unsigned long erofs_stream_creations;
static unsigned long erofs_stream_reuses;
static unsigned long erofs_stream_fallbacks;
static unsigned long erofs_stream_exhaustions;
static unsigned long erofs_stream_allocation_failures;
static unsigned long erofs_stream_destroys;
static unsigned long erofs_stream_reclaims;

MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool",
    MTX_DEF);

SYSCTL_NODE(_vfs, OID_AUTO, erofs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
    "EROFS filesystem");
SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool,
    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD,
    &erofs_stream_mount_budget, 0, "Effective per-mount byte budget");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD,
    &erofs_stream_global_budget, 0, "Effective global byte budget");
SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD,
    &erofs_stream_mount_contexts, 0, "Effective per-mount context limit");
SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit,
    CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0,
    "Effective global context limit");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD,
    &erofs_stream_global_resident, 0, "Currently charged global bytes");
SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD,
    &erofs_stream_global_contexts, 0, "Currently allocated contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD,
    &erofs_stream_cached_contexts, 0, "Currently cacheable contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD,
    &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD,
    &erofs_stream_idle_contexts, 0, "Currently idle contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD,
    &erofs_stream_creations, 0, "Successful context creations");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD,
    &erofs_stream_reuses, 0, "Idle context acquisitions");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD,
    &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD,
    &erofs_stream_exhaustions, 0, "Hard-limit allocation rejections");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, allocation_failures,
    CTLFLAG_RD, &erofs_stream_allocation_failures, 0,
    "UMA or malloc allocation failures");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD,
    &erofs_stream_destroys, 0, "Destroyed contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD,
    &erofs_stream_reclaims, 0, "Low-memory idle context reclaims");

struct erofs_stream_alloc_header {
	struct erofs_stream_ctx *ctx;
	size_t bytes;
};

static bool
erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes)
{
	struct erofs_stream_pool *pool;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	pool = &sbi->stream_pool;
	if (bytes > erofs_stream_mount_budget ||
	    pool->resident_bytes > erofs_stream_mount_budget - bytes ||
	    bytes > erofs_stream_global_budget ||
	    erofs_stream_global_resident > erofs_stream_global_budget - bytes)
		return (false);
	pool->resident_bytes += bytes;
	erofs_stream_global_resident += bytes;
	return (true);
}

static void
erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes)
{

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	KASSERT(bytes <= sbi->stream_pool.resident_bytes &&
	    bytes <= erofs_stream_global_resident,
	    ("erofs stream pool byte accounting underflow"));
	sbi->stream_pool.resident_bytes -= bytes;
	erofs_stream_global_resident -= bytes;
}

int
z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes)
{
	bool reserved;

	if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) {
		ctx->allocation_failed = true;
		mtx_lock(&erofs_stream_lock);
		++erofs_stream_exhaustions;
		mtx_unlock(&erofs_stream_lock);
		return (ENOMEM);
	}
	mtx_lock(&erofs_stream_lock);
	reserved = ctx->charged_bytes <= SIZE_MAX - bytes &&
	    erofs_stream_reserve_locked(ctx->sbi, bytes);
	if (reserved)
		ctx->charged_bytes += bytes;
	else
		++erofs_stream_exhaustions;
	mtx_unlock(&erofs_stream_lock);
	if (!reserved) {
		ctx->allocation_failed = true;
		return (ENOMEM);
	}
	return (0);
}

void
z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes)
{

	mtx_lock(&erofs_stream_lock);
	KASSERT(bytes <= ctx->charged_bytes,
	    ("erofs stream context byte accounting underflow"));
	ctx->charged_bytes -= bytes;
	erofs_stream_release_locked(ctx->sbi, bytes);
	mtx_unlock(&erofs_stream_lock);
}

void *
z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes)
{
	struct erofs_stream_alloc_header *header;
	size_t total;

	if (__builtin_add_overflow(bytes, sizeof(*header), &total) ||
	    z_erofs_stream_ctx_charge(ctx, total) != 0)
		return (NULL);
	header = malloc(total, M_EROFS, M_NOWAIT | M_ZERO);
	if (header == NULL) {
		z_erofs_stream_ctx_uncharge(ctx, total);
		ctx->allocation_failed = true;
		mtx_lock(&erofs_stream_lock);
		++erofs_stream_allocation_failures;
		mtx_unlock(&erofs_stream_lock);
		return (NULL);
	}
	header->ctx = ctx;
	header->bytes = total;
	return (header + 1);
}

void
z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address)
{
	struct erofs_stream_alloc_header *header;
	size_t bytes;

	if (address == NULL)
		return;
	header = (struct erofs_stream_alloc_header *)address - 1;
	KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch"));
	bytes = header->bytes;
	free(header, M_EROFS);
	z_erofs_stream_ctx_uncharge(ctx, bytes);
}

static void
erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_sb_info *sbi;
	bool cached;
	uint8_t algorithm;

	sbi = ctx->sbi;
	algorithm = ctx->algorithm;
	cached = ctx->cached;
	ctx->fini(ctx);
	KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE,
	    ("erofs stream backend allocation leaked"));
	uma_zfree(erofs_stream_zone, ctx);

	mtx_lock(&erofs_stream_lock);
	codec = &sbi->stream_pool.codec[algorithm];
	KASSERT(codec->contexts != 0 && codec->borrowed != 0 &&
	    erofs_stream_global_contexts != 0 &&
	    erofs_stream_borrowed_contexts != 0,
	    ("erofs stream context accounting underflow"));
	--codec->contexts;
	--codec->borrowed;
	--erofs_stream_global_contexts;
	--erofs_stream_borrowed_contexts;
	if (cached) {
		KASSERT(codec->cached != 0 &&
		    erofs_stream_global_cached_by_codec[algorithm] != 0 &&
		    erofs_stream_cached_contexts != 0,
		    ("erofs stream cache accounting underflow"));
		--codec->cached;
		--erofs_stream_global_cached_by_codec[algorithm];
		--erofs_stream_cached_contexts;
	}
	erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE);
	++erofs_stream_destroys;
	cv_broadcast(&sbi->stream_pool.cv);
	mtx_unlock(&erofs_stream_lock);
}

int
z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm,
    size_t context_size, erofs_stream_ctx_init_t *init,
    erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_stream_ctx *ctx;
	bool cached;
	int error;

	*ctxp = NULL;
	if (algorithm == Z_EROFS_COMPRESSION_LZ4 ||
	    algorithm >= Z_EROFS_COMPRESSION_MAX ||
	    context_size < sizeof(*ctx) ||
	    context_size > EROFS_STREAM_CTX_WRAPPER_SIZE)
		return (EINVAL);
	mtx_lock(&erofs_stream_lock);
	if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) {
		mtx_unlock(&erofs_stream_lock);
		return (ENXIO);
	}
	codec = &sbi->stream_pool.codec[algorithm];
	ctx = STAILQ_FIRST(&codec->idle);
	if (ctx != NULL) {
		STAILQ_REMOVE_HEAD(&codec->idle, link);
		KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0,
		    ("erofs stream idle accounting underflow"));
		--codec->idle_count;
		--erofs_stream_idle_contexts;
		++codec->borrowed;
		++erofs_stream_borrowed_contexts;
		++erofs_stream_reuses;
		ctx->allocation_failed = false;
		mtx_unlock(&erofs_stream_lock);
		*ctxp = ctx;
		return (0);
	}
	cached = codec->cached < (unsigned int)erofs_stream_mount_cached &&
	    erofs_stream_global_cached_by_codec[algorithm] <
	    (unsigned int)erofs_stream_global_cached;
	if (codec->contexts >= (unsigned int)erofs_stream_mount_contexts ||
	    erofs_stream_global_contexts >=
	    (unsigned int)erofs_stream_global_contexts_limit ||
	    !erofs_stream_reserve_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE)) {
		++erofs_stream_exhaustions;
		mtx_unlock(&erofs_stream_lock);
		return (ENOMEM);
	}
	++codec->contexts;
	++codec->borrowed;
	++erofs_stream_global_contexts;
	++erofs_stream_borrowed_contexts;
	if (cached) {
		++codec->cached;
		++erofs_stream_global_cached_by_codec[algorithm];
		++erofs_stream_cached_contexts;
	} else {
		++erofs_stream_fallbacks;
	}
	mtx_unlock(&erofs_stream_lock);

	ctx = uma_zalloc(erofs_stream_zone, M_NOWAIT | M_ZERO);
	if (ctx == NULL) {
		mtx_lock(&erofs_stream_lock);
		codec = &sbi->stream_pool.codec[algorithm];
		--codec->contexts;
		--codec->borrowed;
		--erofs_stream_global_contexts;
		--erofs_stream_borrowed_contexts;
		if (cached) {
			--codec->cached;
			--erofs_stream_global_cached_by_codec[algorithm];
			--erofs_stream_cached_contexts;
		}
		erofs_stream_release_locked(sbi,
		    EROFS_STREAM_CTX_WRAPPER_SIZE);
		++erofs_stream_allocation_failures;
		cv_broadcast(&sbi->stream_pool.cv);
		mtx_unlock(&erofs_stream_lock);
		return (ENOMEM);
	}
	ctx->sbi = sbi;
	ctx->fini = fini;
	ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE;
	ctx->algorithm = algorithm;
	ctx->cached = cached;
	error = init(ctx);
	if (error != 0) {
		erofs_stream_ctx_destroy(ctx);
		return (error > 0 ? error : EIO);
	}
	mtx_lock(&erofs_stream_lock);
	++erofs_stream_creations;
	mtx_unlock(&erofs_stream_lock);
	*ctxp = ctx;
	return (0);
}

void
z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable)
{
	struct erofs_stream_pool_codec *codec;

	mtx_lock(&erofs_stream_lock);
	codec = &ctx->sbi->stream_pool.codec[ctx->algorithm];
	if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) {
		KASSERT(codec->borrowed != 0 &&
		    erofs_stream_borrowed_contexts != 0,
		    ("erofs stream borrowed accounting underflow"));
		--codec->borrowed;
		--erofs_stream_borrowed_contexts;
		STAILQ_INSERT_HEAD(&codec->idle, ctx, link);
		++codec->idle_count;
		++erofs_stream_idle_contexts;
		cv_broadcast(&ctx->sbi->stream_pool.cv);
		mtx_unlock(&erofs_stream_lock);
		return;
	}
	mtx_unlock(&erofs_stream_lock);
	erofs_stream_ctx_destroy(ctx);
}

static struct erofs_stream_ctx *
erofs_stream_take_idle_locked(struct erofs_sb_info *sbi)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_stream_ctx *ctx;
	unsigned int algorithm;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
		codec = &sbi->stream_pool.codec[algorithm];
		ctx = STAILQ_FIRST(&codec->idle);
		if (ctx == NULL)
			continue;
		STAILQ_REMOVE_HEAD(&codec->idle, link);
		KASSERT(codec->idle_count != 0 && erofs_stream_idle_contexts != 0,
		    ("erofs stream idle accounting underflow"));
		--codec->idle_count;
		--erofs_stream_idle_contexts;
		++codec->borrowed;
		++erofs_stream_borrowed_contexts;
		return (ctx);
	}
	return (NULL);
}

static bool
erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool)
{
	unsigned int algorithm;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
		if (pool->codec[algorithm].contexts != 0)
			return (false);
	}
	return (true);
}

void
z_erofs_stream_pool_init(struct erofs_sb_info *sbi)
{
	unsigned int algorithm;

	bzero(&sbi->stream_pool, sizeof(sbi->stream_pool));
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm)
		STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle);
	cv_init(&sbi->stream_pool.cv, "erofs stream drain");
	mtx_lock(&erofs_stream_lock);
	sbi->stream_pool_initialized = true;
	LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link);
	mtx_unlock(&erofs_stream_lock);
}

void
z_erofs_stream_pool_fini(struct erofs_sb_info *sbi)
{
	struct erofs_stream_ctx *ctx;

	if (!sbi->stream_pool_initialized)
		return;
	mtx_lock(&erofs_stream_lock);
	sbi->stream_pool.closing = true;
	for (;;) {
		ctx = erofs_stream_take_idle_locked(sbi);
		if (ctx == NULL)
			break;
		mtx_unlock(&erofs_stream_lock);
		erofs_stream_ctx_destroy(ctx);
		mtx_lock(&erofs_stream_lock);
	}
	while (!erofs_stream_pool_empty_locked(&sbi->stream_pool))
		cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock);
	KASSERT(sbi->stream_pool.resident_bytes == 0,
	    ("erofs stream mount bytes remain at unmount"));
	LIST_REMOVE(sbi, stream_pool_link);
	sbi->stream_pool_initialized = false;
	mtx_unlock(&erofs_stream_lock);
	cv_destroy(&sbi->stream_pool.cv);
}

static void
erofs_stream_pool_lowmem(void *arg, int howto)
{
	struct erofs_stream_ctx *ctx;
	struct erofs_sb_info *sbi;

	(void)arg;
	(void)howto;
	for (;;) {
		ctx = NULL;
		mtx_lock(&erofs_stream_lock);
		LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) {
			ctx = erofs_stream_take_idle_locked(sbi);
			if (ctx != NULL) {
				++erofs_stream_reclaims;
				break;
			}
		}
		mtx_unlock(&erofs_stream_lock);
		if (ctx == NULL)
			break;
		erofs_stream_ctx_destroy(ctx);
	}
}

static void
erofs_stream_pool_global_init(void *arg)
{

	(void)arg;
	erofs_stream_mount_budget = MIN(erofs_stream_mount_budget,
	    EROFS_STREAM_MOUNT_HARD_BUDGET);
	erofs_stream_global_budget = MIN(erofs_stream_global_budget,
	    EROFS_STREAM_GLOBAL_HARD_BUDGET);
	erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts,
	    (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS));
	erofs_stream_global_contexts_limit = MAX(0, MIN(
	    erofs_stream_global_contexts_limit,
	    (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS));
	erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached,
	    (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC));
	erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached,
	    (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC));
	erofs_stream_zone = uma_zcreate("erofs stream ctx",
	    EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
	KASSERT(erofs_stream_zone != NULL, ("cannot create erofs stream UMA zone"));
	(void)uma_zone_set_max(erofs_stream_zone,
	    EROFS_STREAM_GLOBAL_HARD_CONTEXTS);
	uma_zone_set_maxcache(erofs_stream_zone,
	    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3);
	erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
	    erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT);
}

static void
erofs_stream_pool_global_fini(void *arg)
{

	(void)arg;
	if (erofs_stream_lowmem_tag != NULL)
		EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag);
	KASSERT(LIST_EMPTY(&erofs_stream_mounts),
	    ("erofs stream mounts remain at unload"));
	KASSERT(erofs_stream_global_contexts == 0 &&
	    erofs_stream_global_resident == 0 &&
	    erofs_stream_borrowed_contexts == 0 &&
	    erofs_stream_idle_contexts == 0 &&
	    erofs_stream_cached_contexts == 0,
	    ("erofs stream resources remain at unload"));
	uma_zdestroy(erofs_stream_zone);
}

SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY,
    erofs_stream_pool_global_init, NULL);
SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_ANY,
    erofs_stream_pool_global_fini, NULL);

static const char *erofs_opts[] = {
	"export",
	"from",
	NULL,
};

static vfs_mount_t erofs_mount;
static vfs_root_t erofs_root;
static vfs_statfs_t erofs_statfs;
static vfs_unmount_t erofs_unmount;
static vfs_fhtovp_t erofs_fhtovp;

#define EROFS_DEVICE_OPT_PREFIX "device."

struct erofs_device_arg {
	uint16_t slot;
	char *path;
};

static int
erofs_load_generation_seed(struct erofs_sb_info *sbi, uint32_t sb_size,
    uint32_t *seedp)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	uint32_t seed;
	int error;

	error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sb_size, &buf);
	if (error != 0)
		return (error);
	seed = fnv_32_buf(buf.data, sb_size, FNV1_32_INIT);
	erofs_put_metabuf(&buf);
	*seedp = seed != 0 ? seed : 1;
	return (0);
}

static void
erofs_free_device_args(struct erofs_device_arg *args, unsigned int count)
{
	unsigned int i;

	if (args == NULL)
		return;
	for (i = 0; i < count; ++i)
		free(args[i].path, M_EROFS);
	free(args, M_EROFS);
}

static int
erofs_parse_device_slot(const char *name, uint16_t *slotp)
{
	const char *p;
	unsigned int slot;

	if (strncmp(name, EROFS_DEVICE_OPT_PREFIX,
	    sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0)
		return (ENOENT);
	p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1;
	if (*p < '1' || *p > '9')
		return (EINVAL);
	slot = 0;
	for (; *p != '\0'; ++p) {
		if (*p < '0' || *p > '9' || slot > (UINT16_MAX - (*p - '0')) / 10)
			return (EINVAL);
		slot = slot * 10 + (*p - '0');
	}
	if (slot == 0 || slot > UINT16_MAX)
		return (EINVAL);
	*slotp = slot;
	return (0);
}

static int
erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp,
    unsigned int *countp)
{
	struct erofs_device_arg *args;
	struct vfsopt *opt;
	char name[32];
	unsigned int count, i;
	uint16_t slot;
	int error;

	*argsp = NULL;
	*countp = 0;
	count = 0;
	TAILQ_FOREACH(opt, mp->mnt_optnew, link) {
		error = erofs_parse_device_slot(opt->name, &slot);
		if (error == ENOENT)
			continue;
		if (error != 0 || opt->value == NULL || opt->len <= 1 ||
		    ((char *)opt->value)[opt->len - 1] != '\0') {
			vfs_mount_error(mp, "erofs: invalid external device option %s",
			    opt->name);
			return (EINVAL);
		}
		if (count == UINT16_MAX)
			return (E2BIG);
		++count;
	}
	if (count == 0)
		return (0);

	args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO);
	i = 0;
	TAILQ_FOREACH(opt, mp->mnt_optnew, link) {
		error = erofs_parse_device_slot(opt->name, &slot);
		if (error == ENOENT)
			continue;
		KASSERT(error == 0, ("validated EROFS device option changed"));
		args[i].slot = slot;
		args[i].path = malloc(opt->len, M_EROFS, M_WAITOK);
		memcpy(args[i].path, opt->value, opt->len);
		++i;
	}
	for (i = 0; i < count; ++i) {
		snprintf(name, sizeof(name), EROFS_DEVICE_OPT_PREFIX "%u",
		    args[i].slot);
		vfs_deleteopt(mp->mnt_optnew, name);
	}
	*argsp = args;
	*countp = count;
	return (0);
}

static void
erofs_release_device_info(struct erofs_device_info *dif)
{
	if (dif->cp != NULL) {
		g_topology_lock();
		g_vfs_close(dif->cp);
		g_topology_unlock();
		dif->cp = NULL;
	}
	if (dif->devvp != NULL) {
		vrele(dif->devvp);
		dif->devvp = NULL;
	}
	if (dif->dev != NULL) {
		dev_rel(dif->dev);
		dif->dev = NULL;
	}
}

static bool
erofs_provider_is_duplicate(struct erofs_sb_info *sbi, struct g_provider *pp)
{
	unsigned int i;

	if (sbi == NULL)
		return (false);
	if (sbi->dif0.cp != NULL && sbi->dif0.cp->provider == pp)
		return (true);
	for (i = 0; i < sbi->extra_devices; ++i) {
		if (sbi->devs[i].cp != NULL && sbi->devs[i].cp->provider == pp)
			return (true);
	}
	return (false);
}

static int
erofs_open_device(struct erofs_sb_info *sbi, const char *path,
    struct erofs_device_info *dif)
{
	struct g_provider *pp;
	struct nameidata nd;
	struct vnode *devvp;
	struct cdev *dev;
	int error;

	bzero(dif, sizeof(*dif));
	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path);
	error = namei(&nd);
	if (error != 0)
		return (error);
	devvp = nd.ni_vp;
	NDFREE_PNBUF(&nd);
	if (!vn_isdisk_error(devvp, &error)) {
		vput(devvp);
		return (error);
	}
	error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread);
	if (error != 0)
		error = priv_check(curthread, PRIV_VFS_MOUNT_PERM);
	if (error != 0) {
		vput(devvp);
		return (error);
	}
	dev = devvp->v_rdev;
	dev_ref(dev);
	g_topology_lock();
	pp = g_dev_getprovider(dev);
	if (pp == NULL)
		error = ENXIO;
	else if (erofs_provider_is_duplicate(sbi, pp))
		error = EINVAL;
	else
		error = g_vfs_open(devvp, &dif->cp, "erofs", 0);
	if (error == 0) {
		dif->mediasize = dif->cp->provider->mediasize;
		dif->sectorsize = dif->cp->provider->sectorsize;
	}
	g_topology_unlock();
	VOP_UNLOCK(devvp);
	if (error != 0) {
		dev_rel(dev);
		vrele(devvp);
		return (error);
	}
	dif->devvp = devvp;
	dif->dev = dev;
	if (dif->sectorsize == 0 ||
	    (dif->sectorsize & (dif->sectorsize - 1)) != 0) {
		erofs_release_device_info(dif);
		return (EINVAL);
	}
	return (0);
}

static void
erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif)
{
	u_long iosize;

	iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ?
	    dif->dev->si_iosize_max : MAXPHYS;
	mp->mnt_iosize_max = MIN(mp->mnt_iosize_max, MIN(iosize, (u_long)MAXPHYS));
}

static void
erofs_free_dev_context(struct erofs_sb_info *sbi)
{
	unsigned int i;

	if (sbi->devs != NULL) {
		for (i = sbi->extra_devices; i > 0; --i)
			erofs_release_device_info(&sbi->devs[i - 1]);
		free(sbi->devs, M_EROFS);
	}
}

static void
erofs_drop_internal_inodes(struct erofs_sb_info *sbi)
{
	if (sbi->metabox_en != NULL)
		free(sbi->metabox_en, M_EROFS);
	if (sbi->packed_inode != NULL)
		free(sbi->packed_inode, M_EROFS);
}

static void
erofs_sb_free(struct erofs_sb_info *sbi)
{
	if (sbi == NULL)
		return;
	z_erofs_extent_cache_fini(sbi);
	z_erofs_stream_pool_fini(sbi);
	erofs_xattr_prefixes_cleanup(sbi);
	erofs_drop_internal_inodes(sbi);
	erofs_free_dev_context(sbi);
	erofs_release_device_info(&sbi->dif0);
	free(sbi, M_EROFS);
}

static int
erofs_superblock_csum_verify(struct erofs_sb_info *sbi,
    const struct erofs_super_block *dsb)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	uint32_t expected, crc;
	size_t len;
	int error;

	if (!erofs_sb_has_sb_chksum(sbi))
		return (0);

	len = 1u << dsb->blkszbits;
	if (len > EROFS_SUPER_OFFSET)
		len -= EROFS_SUPER_OFFSET;

	error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, len, &buf);
	if (error != 0)
		return (error);

	crc = calculate_crc32c(erofs_crc32c_seed,
	    (const uint8_t *)buf.data +
	    offsetof(struct erofs_super_block, checksum) +
	    sizeof(dsb->checksum),
	    len - offsetof(struct erofs_super_block, checksum) -
	    sizeof(dsb->checksum));
	expected = le32toh(dsb->checksum);
	erofs_put_metabuf(&buf);

	if (crc != expected) {
		vfs_mount_error(sbi->mnt,
		    "erofs: invalid superblock checksum 0x%08x, "
		    "0x%08x expected", crc, expected);
		return (EINTEGRITY);
	}
	return (0);
}

static void
erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat,
    uint64_t *blocks, erofs_nid_t *root_nid)
{
	*blocks = le32toh(dsb->blocks_lo);
	if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 &&
	    dsb->rootnid_8b != 0) {
		*blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32;
		*root_nid = le64toh(dsb->rootnid_8b);
	} else {
		*root_nid = le16toh(dsb->rb.rootnid_2b);
	}
}

static int
erofs_validate_device_size(struct erofs_sb_info *sbi,
    struct erofs_device_info *dif, erofs_blk_t blocks)
{
	uint64_t bytes;

	if (blocks == 0)
		return (EINTEGRITY);
	if (sbi->block_size < dif->sectorsize ||
	    sbi->block_size % dif->sectorsize != 0)
		return (EINVAL);
	if (blocks > (UINT64_MAX >> sbi->blkszbits))
		return (EINTEGRITY);
	bytes = blocks << sbi->blkszbits;
	if (bytes > dif->mediasize)
		return (ENXIO);
	return (0);
}

static int
erofs_init_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif,
    const char *path)
{
	struct erofs_device_info opened;
	erofs_blk_t blocks, uniaddr;
	int error;

	blocks = dif->blocks;
	uniaddr = dif->uniaddr;
	error = erofs_open_device(sbi, path, &opened);
	if (error != 0)
		return (error);
	erofs_update_iosize_max(sbi->mnt, &opened);
	opened.blocks = blocks;
	opened.uniaddr = uniaddr;
	*dif = opened;
	return (erofs_validate_device_size(sbi, dif, dif->blocks));
}

static const char *
erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count,
    unsigned int slot)
{
	unsigned int i;

	for (i = 0; i < count; ++i) {
		if (args[i].slot == slot)
			return (args[i].path);
	}
	return (NULL);
}

static int
erofs_scan_devices(struct erofs_sb_info *sbi, const struct erofs_super_block *dsb,
    const struct erofs_device_arg *args, unsigned int arg_count)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct erofs_deviceslot *slots;
	struct erofs_device_info *dif;
	const char *path;
	erofs_off_t devt_off;
	uint64_t devt_size, image_size, end, other_end;
	erofs_blk_t maxend;
	unsigned int i, j, mask;
	int error;

	sbi->total_blocks = sbi->dif0.blocks;
	sbi->flatdev_blocks = sbi->dif0.blocks;
	if (sbi->extra_devices == 0) {
		if (arg_count != 0) {
			vfs_mount_error(sbi->mnt,
			    "erofs: external devices given without a device table");
			return (EINVAL);
		}
		return (0);
	}
	devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
	devt_size = (uint64_t)sbi->extra_devices * EROFS_DEVT_SLOT_SIZE;
	if (sbi->dif0.blocks > (UINT64_MAX >> sbi->blkszbits))
		return (EINTEGRITY);
	image_size = sbi->dif0.blocks << sbi->blkszbits;
	if (devt_off > image_size || devt_size > image_size - devt_off ||
	    devt_size > SIZE_MAX)
		return (EINTEGRITY);
	error = erofs_read_metadata(sbi, 0, devt_off, (size_t)devt_size, &buf);
	if (error != 0)
		return (error);
	slots = buf.data;
	sbi->devs = mallocarray(sbi->extra_devices, sizeof(*sbi->devs), M_EROFS,
	    M_WAITOK | M_ZERO);
	maxend = sbi->dif0.blocks;
	for (i = 0; i < sbi->extra_devices; ++i) {
		dif = &sbi->devs[i];
		dif->blocks = le32toh(slots[i].blocks_lo);
		dif->uniaddr = le32toh(slots[i].uniaddr_lo);
		if (erofs_sb_has_48bit(sbi)) {
			dif->blocks |= (uint64_t)le16toh(slots[i].blocks_hi) << 32;
			dif->uniaddr |= (uint64_t)le16toh(slots[i].uniaddr_hi) << 32;
		}
		if (dif->blocks == 0 ||
		    __builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) {
			error = EINTEGRITY;
			goto out;
		}
		if (end >
		    (erofs_sb_has_48bit(sbi) ? (1ULL << 48) : (1ULL << 32))) {
			error = EINTEGRITY;
			goto out;
		}
		if (dif->uniaddr != 0 && dif->uniaddr < sbi->dif0.blocks) {
			error = EINTEGRITY;
			goto out;
		}
		for (j = 0; j < i; ++j) {
			if (dif->uniaddr == 0 || sbi->devs[j].uniaddr == 0)
				continue;
			if (__builtin_add_overflow(sbi->devs[j].uniaddr,
			    sbi->devs[j].blocks, &other_end)) {
				error = EINTEGRITY;
				goto out;
			}
			if (dif->uniaddr < other_end && sbi->devs[j].uniaddr < end) {
				error = EINTEGRITY;
				goto out;
			}
		}
		if (__builtin_add_overflow(sbi->total_blocks, dif->blocks,
		    &sbi->total_blocks)) {
			error = EOVERFLOW;
			goto out;
		}
		maxend = MAX(maxend, (erofs_blk_t)end);
	}
	erofs_put_metabuf(&buf);
	sbi->flatdev_blocks = maxend;
	mask = 1;
	while (mask < (unsigned int)sbi->extra_devices + 1)
		mask <<= 1;
	sbi->device_id_mask = mask - 1;
	sbi->flatdev = arg_count == 0;
	if (sbi->flatdev)
		return (erofs_validate_device_size(sbi, &sbi->dif0,
		    sbi->flatdev_blocks));
	if (arg_count != sbi->extra_devices) {
		vfs_mount_error(sbi->mnt,
		    "erofs: external devices don't match (ondisk %u, given %u)",
		    sbi->extra_devices, arg_count);
		return (arg_count < sbi->extra_devices ? ENXIO : EINVAL);
	}
	for (i = 0; i < arg_count; ++i) {
		if (args[i].slot == 0 || args[i].slot > sbi->extra_devices)
			return (EINVAL);
	}
	for (i = 0; i < sbi->extra_devices; ++i) {
		path = erofs_device_arg_path(args, arg_count, i + 1);
		if (path == NULL)
			return (ENXIO);
		error = erofs_init_device(sbi, &sbi->devs[i], path);
		if (error != 0)
			return (error);
	}
	return (0);
out:
	erofs_put_metabuf(&buf);
	return (error);
}

static int
erofs_init_packed_inode(struct erofs_sb_info *sbi)
{
	int error;

	/* Load the packed carrier before any fragment-backed metabox inode. */
	if (erofs_sb_has_fragments(sbi) && sbi->packed_nid > 0) {
		sbi->packed_inode = malloc(sizeof(*sbi->packed_inode), M_EROFS,
		    M_WAITOK | M_ZERO);
		error = erofs_read_inode(sbi, sbi->packed_nid, sbi->packed_inode);
		if (error != 0) {
			free(sbi->packed_inode, M_EROFS);
			sbi->packed_inode = NULL;
			return (error);
		}
		if (sbi->packed_inode->vtype != VREG || sbi->packed_inode->fragment) {
			vfs_mount_error(sbi->mnt,
			    "erofs: packed inode nid=%ju is not a non-recursive regular file",
			    (uintmax_t)sbi->packed_nid);
			return (EINTEGRITY);
		}
	}
	return (0);
}

static int
erofs_init_metabox_inode(struct erofs_sb_info *sbi)
{
	int error;

	/*
	 * METABOX NIDs address inode slots in this backing inode's data.  The
	 * packed carrier is ready first so a compressed metabox may legally end in
	 * a fragment pcluster without reading an uninitialized dependency.
	 */
	if (erofs_sb_has_metabox(sbi)) {
		struct erofs_map_blocks map;

		sbi->metabox_en = malloc(sizeof(*sbi->metabox_en), M_EROFS,
		    M_WAITOK | M_ZERO);
		error = erofs_read_inode(sbi, sbi->metabox_nid, sbi->metabox_en);
		if (error != 0)
			return (error);
		if (sbi->metabox_en->vtype != VREG) {
			vfs_mount_error(sbi->mnt,
			    "erofs: metabox inode nid=%ju is not a regular file",
			    (uintmax_t)sbi->metabox_nid);
			return (EINTEGRITY);
		}
		if (sbi->metabox_en->fragment) {
			if (sbi->packed_inode == NULL ||
			    sbi->packed_inode->nid == sbi->metabox_en->nid ||
			    sbi->metabox_en->size == 0)
				return (EINTEGRITY);
			map = (struct erofs_map_blocks) {
				.m_la = sbi->metabox_en->size - 1,
			};
			error = erofs_map_blocks(sbi, sbi->metabox_en, &map);
			if (error != 0 || (map.m_flags & EROFS_MAP_FRAGMENT) == 0)
				return (error != 0 ? error : EINTEGRITY);
		}
	}
	return (0);
}

static int
erofs_read_superblock(struct erofs_sb_info *sbi, struct erofs_super_block *dsb)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	uint32_t unsupported;
	int error;

	error = erofs_read_metadata(sbi, 0, EROFS_SUPER_OFFSET, sizeof(*dsb),
	    &buf);
	if (error != 0)
		return (error);
	memcpy(dsb, buf.data, sizeof(*dsb));
	erofs_put_metabuf(&buf);

	if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1)
		return (EINVAL);
	if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT)
		return (EINVAL);
	sbi->blkszbits = dsb->blkszbits;
	sbi->block_size = 1u << sbi->blkszbits;
	sbi->feature_compat = le32toh(dsb->feature_compat);
	error = erofs_superblock_csum_verify(sbi, dsb);
	if (error != 0)
		return (error);

	if (dsb->dirblkbits != 0)
		return (EOPNOTSUPP);
	sbi->feature_incompat = le32toh(dsb->feature_incompat);
	sbi->packed_nid = le64toh(dsb->packed_nid);
	sbi->extra_devices = erofs_sb_has_device_table(sbi) ?
	    le16toh(dsb->extra_devices) : 0;
	unsupported = sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT;
	if (unsupported != 0)
		return (EOPNOTSUPP);
	sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE;
	if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET)
		return (EINVAL);
	sbi->meta_blkaddr = le32toh(dsb->meta_blkaddr);
	sbi->xattr_blkaddr = le32toh(dsb->xattr_blkaddr);
	sbi->xattr_prefix_start = le32toh(dsb->xattr_prefix_start);
	sbi->xattr_prefix_count = dsb->xattr_prefix_count;
	if (erofs_sb_has_ishare_xattrs(sbi) &&
	    dsb->ishare_xattr_prefix_id >= sbi->xattr_prefix_count)
		return (EINTEGRITY);
	/* Preserve the raw feature declaration and gate its format at use sites. */
	sbi->xattr_filter_reserved = dsb->xattr_filter_reserved;
	erofs_sb_blocks_root(dsb, sbi->feature_incompat, &sbi->blocks,
	    &sbi->root_nid);
	sbi->dif0.blocks = sbi->blocks;
	error = erofs_validate_device_size(sbi, &sbi->dif0, sbi->dif0.blocks);
	if (error != 0)
		return (error);
	sbi->inos = le64toh(dsb->inos);
	sbi->epoch = (int64_t)le64toh(dsb->epoch);
	sbi->fixed_nsec = le32toh(dsb->fixed_nsec);
	if (sbi->fixed_nsec >= 1000000000)
		return (EINTEGRITY);
	error = erofs_load_generation_seed(sbi, sbi->sb_size,
	    &sbi->generation_seed);
	if (error != 0)
		return (error);
	if (sbi->packed_nid != 0 && erofs_nid_in_metabox(sbi->packed_nid))
		return (EINTEGRITY);
	if (erofs_sb_has_metabox(sbi)) {
		if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid))
			return (EINTEGRITY);
		sbi->metabox_nid = le64toh(dsb->metabox_nid);
		if (erofs_nid_in_metabox(sbi->metabox_nid))
			return (EINTEGRITY);
	}

	return (z_erofs_parse_cfgs(sbi, dsb));
}

static int
erofs_mountfs(struct erofs_device_info *primary, struct mount *mp,
    const struct erofs_device_arg *args, unsigned int arg_count)
{
	struct erofs_sb_info *sbi;
	struct erofs_inode root;
	struct erofs_super_block dsb;
	int error;

	sbi = malloc(sizeof(*sbi), M_EROFS, M_WAITOK | M_ZERO);
	sbi->mnt = mp;
	z_erofs_extent_cache_init(sbi);
	z_erofs_stream_pool_init(sbi);
	sbi->dif0 = *primary;
	bzero(primary, sizeof(*primary));

	error = erofs_read_superblock(sbi, &dsb);
	if (error != 0)
		goto fail;
	error = erofs_scan_devices(sbi, &dsb, args, arg_count);
	if (error != 0)
		goto fail;

	if (erofs_sb_has_shared_ea_in_metabox(sbi) &&
	    !erofs_sb_has_metabox(sbi)) {
		error = EINTEGRITY;
		goto fail;
	}

	error = erofs_init_packed_inode(sbi);
	if (error != 0)
		goto fail;
	error = erofs_init_metabox_inode(sbi);
	if (error != 0)
		goto fail;
	error = erofs_read_inode(sbi, sbi->root_nid, &root);
	if (error != 0)
		goto fail;
	if (root.vtype != VDIR) {
		vfs_mount_error(mp,
		    "erofs: root inode nid=%ju is not a directory",
		    (uintmax_t)sbi->root_nid);
		error = EINTEGRITY;
		goto fail;
	}
	error = erofs_xattr_prefixes_init(sbi);
	if (error != 0)
		goto fail;
	set_opt(&sbi->opt, POSIX_ACL);
	memcpy(sbi->volume_name, dsb.volume_name, 16);
	sbi->volume_name[16] = '\0';

	mp->mnt_data = sbi;
	mp->mnt_stat.f_fsid.val[0] = dev2udev(sbi->dif0.devvp->v_rdev);
	mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum;
	MNT_ILOCK(mp);
	mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY | MNT_ACLS;
	mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED |
	    MNTK_USES_BCACHE;
	MNT_IUNLOCK(mp);
	return (0);
fail:
	erofs_sb_free(sbi);
	return (error);
}

static int
erofs_mount(struct mount *mp)
{
	struct erofs_device_arg *args;
	struct erofs_device_info primary;
	char *fspec;
	unsigned int arg_count;
	int error, len;

	MNT_ILOCK(mp);
	mp->mnt_flag |= MNT_RDONLY;
	MNT_IUNLOCK(mp);
	if (mp->mnt_flag & MNT_UPDATE) {
		if (vfs_flagopt(mp->mnt_optnew, "export", NULL, 0))
			return (0);
		return (EOPNOTSUPP);
	}
	args = NULL;
	arg_count = 0;
	error = erofs_parse_device_options(mp, &args, &arg_count);
	if (error != 0)
		return (error);
	if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) {
		error = EINVAL;
		goto out_args;
	}
	fspec = NULL;
	error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len);
	if (error != 0 || fspec == NULL || len == 0 ||
	    fspec[len - 1] != '\0') {
		error = EINVAL;
		goto out_args;
	}
	mp->mnt_iosize_max = MAXPHYS;
	error = erofs_open_device(NULL, fspec, &primary);
	if (error != 0)
		goto out_args;
	erofs_update_iosize_max(mp, &primary);
	error = erofs_mountfs(&primary, mp, args, arg_count);
out_args:
	erofs_free_device_args(args, arg_count);
	if (error != 0)
		return (error);
	vfs_mountedfrom(mp, fspec);
	return (erofs_statfs(mp, &mp->mnt_stat));
}

static int
erofs_root(struct mount *mp, int flags, struct vnode **vpp)
{
	int error;

	error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);
	if (error != 0)
		vfs_mount_error(mp, "erofs: failed to load root nid %ju: error %d",
		    (uintmax_t)MTOE(mp)->root_nid, error);
	return (error);
}

static int
erofs_statfs(struct mount *mp, struct statfs *sbp)
{
	struct erofs_sb_info *sbi;

	sbi = MTOE(mp);
	sbp->f_bsize = sbi->block_size;
	sbp->f_iosize = sbi->block_size;
	sbp->f_blocks = sbi->total_blocks;
	sbp->f_bfree = 0;
	sbp->f_bavail = 0;
	sbp->f_files = sbi->inos;
	sbp->f_ffree = 0;
	return (0);
}

static int
erofs_unmount(struct mount *mp, int mntflags)
{
	struct erofs_sb_info *sbi;
	int error, flags;

	flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0;
	error = vflush(mp, 0, flags, curthread);
	if (error != 0)
		return (error);
	sbi = MTOE(mp);
	mp->mnt_data = NULL;
	erofs_sb_free(sbi);
	return (0);
}

/* Persistent EROFS file handle to locked vnode. */
static int
erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp)
{
	struct erofs_fid efid;
	struct erofs_inode *vi;
	struct vnode *vp;
	erofs_nid_t nid;
	int error;

	*vpp = NULL;
	bzero(&efid, sizeof(efid));
	memcpy(&efid, fhp, sizeof(efid));
	if (efid.len != sizeof(efid) || efid.pad != 0)
		return (EINVAL);
	nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo;
	if (!erofs_nid_is_valid(MTOE(mp), nid))
		return (ESTALE);
	error = VFS_VGET(mp, (ino_t)nid, flags, &vp);
	if (error != 0)
		return (error);
	vi = VTOE(vp);
	if (vi->mode == 0 || vi->nlink == 0 || vi->nid != nid ||
	    vi->generation != efid.gen) {
		vput(vp);
		return (ESTALE);
	}
	*vpp = vp;
	return (0);
}

static struct vfsops erofs_vfsops = {
	.vfs_fhtovp = erofs_fhtovp,
	.vfs_mount = erofs_mount,
	.vfs_root = erofs_root,
	.vfs_statfs = erofs_statfs,
	.vfs_unmount = erofs_unmount,
	.vfs_vget = erofs_vget,
};
VFS_SET(erofs_vfsops, erofs, VFCF_READONLY);
MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1);
MODULE_DEPEND(erofs, zlib, 1, 1, 1);
MODULE_VERSION(erofs, 1);