erofs-freebsd-out-tree/super.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/_maxphys.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/fnv_hash.h>
#include <sys/gsb_crc32.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mount.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/stat.h>
#include <sys/vnode.h>

#include <geom/geom.h>
#include <geom/geom_vfs.h>

#include "internal.h"
#include "xattr.h"
#include "erofs_defs.h"

MALLOC_DEFINE(M_EROFS, "erofs", "EROFS filesystem");

static const char *erofs_opts[] = {
	"export",
	"from",
	NULL,
};

static vfs_mount_t erofs_mount;
static vfs_root_t erofs_root;
static vfs_statfs_t erofs_statfs;
static vfs_unmount_t erofs_unmount;
static vfs_vget_t erofs_vgetf;
static vfs_fhtovp_t erofs_fhtovp;

#define EROFS_DEVICE_OPT_PREFIX "device."

struct erofs_device_arg {
	uint16_t slot;
	char *path;
};

static int
erofs_load_generation_seed(struct erofs_mount *em, uint32_t sb_size,
    uint32_t *seedp)
{
	uint32_t seed;
	void *buf;
	int error;

	error = erofs_bread(em, EROFS_SUPER_OFFSET, sb_size, &buf);
	if (error != 0)
		return (error);
	seed = fnv_32_buf(buf, sb_size, FNV1_32_INIT);
	erofs_brelse(buf);
	*seedp = seed != 0 ? seed : 1;
	return (0);
}

static void
erofs_free_device_args(struct erofs_device_arg *args, unsigned int count)
{
	unsigned int i;

	if (args == NULL)
		return;
	for (i = 0; i < count; ++i)
		free(args[i].path, M_EROFS);
	free(args, M_EROFS);
}

static int
erofs_parse_device_slot(const char *name, uint16_t *slotp)
{
	const char *p;
	unsigned int slot;

	if (strncmp(name, EROFS_DEVICE_OPT_PREFIX,
	    sizeof(EROFS_DEVICE_OPT_PREFIX) - 1) != 0)
		return (ENOENT);
	p = name + sizeof(EROFS_DEVICE_OPT_PREFIX) - 1;
	if (*p < '1' || *p > '9')
		return (EINVAL);
	slot = 0;
	for (; *p != '\0'; ++p) {
		if (*p < '0' || *p > '9' || slot > (UINT16_MAX - (*p - '0')) / 10)
			return (EINVAL);
		slot = slot * 10 + (*p - '0');
	}
	if (slot == 0 || slot > UINT16_MAX)
		return (EINVAL);
	*slotp = slot;
	return (0);
}

static int
erofs_parse_device_options(struct mount *mp, struct erofs_device_arg **argsp,
    unsigned int *countp)
{
	struct erofs_device_arg *args;
	struct vfsopt *opt;
	char name[32];
	unsigned int count, i;
	uint16_t slot;
	int error;

	*argsp = NULL;
	*countp = 0;
	count = 0;
	TAILQ_FOREACH(opt, mp->mnt_optnew, link) {
		error = erofs_parse_device_slot(opt->name, &slot);
		if (error == ENOENT)
			continue;
		if (error != 0 || opt->value == NULL || opt->len <= 1 ||
		    ((char *)opt->value)[opt->len - 1] != '\0') {
			vfs_mount_error(mp, "erofs: invalid external device option %s",
			    opt->name);
			return (EINVAL);
		}
		if (count == UINT16_MAX)
			return (E2BIG);
		++count;
	}
	if (count == 0)
		return (0);

	args = mallocarray(count, sizeof(*args), M_EROFS, M_WAITOK | M_ZERO);
	i = 0;
	TAILQ_FOREACH(opt, mp->mnt_optnew, link) {
		error = erofs_parse_device_slot(opt->name, &slot);
		if (error == ENOENT)
			continue;
		KASSERT(error == 0, ("validated EROFS device option changed"));
		args[i].slot = slot;
		args[i].path = malloc(opt->len, M_EROFS, M_WAITOK);
		memcpy(args[i].path, opt->value, opt->len);
		++i;
	}
	for (i = 0; i < count; ++i) {
		snprintf(name, sizeof(name), EROFS_DEVICE_OPT_PREFIX "%u",
		    args[i].slot);
		vfs_deleteopt(mp->mnt_optnew, name);
	}
	*argsp = args;
	*countp = count;
	return (0);
}

static void
erofs_release_device_info(struct erofs_device_info *dif)
{
	if (dif->cp != NULL) {
		g_topology_lock();
		g_vfs_close(dif->cp);
		g_topology_unlock();
		dif->cp = NULL;
	}
	if (dif->devvp != NULL) {
		vrele(dif->devvp);
		dif->devvp = NULL;
	}
	if (dif->dev != NULL) {
		dev_rel(dif->dev);
		dif->dev = NULL;
	}
}

static bool
erofs_provider_is_duplicate(struct erofs_mount *em, struct g_provider *pp)
{
	unsigned int i;

	if (em == NULL)
		return (false);
	if (em->dif0.cp != NULL && em->dif0.cp->provider == pp)
		return (true);
	for (i = 0; i < em->extra_devices; ++i) {
		if (em->devs[i].cp != NULL && em->devs[i].cp->provider == pp)
			return (true);
	}
	return (false);
}

static int
erofs_open_device(struct erofs_mount *em, const char *path,
    struct erofs_device_info *dif)
{
	struct g_provider *pp;
	struct nameidata nd;
	struct vnode *devvp;
	struct cdev *dev;
	int error;

	bzero(dif, sizeof(*dif));
	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, path);
	error = namei(&nd);
	if (error != 0)
		return (error);
	devvp = nd.ni_vp;
	NDFREE_PNBUF(&nd);
	if (!vn_isdisk_error(devvp, &error)) {
		vput(devvp);
		return (error);
	}
	error = VOP_ACCESS(devvp, VREAD, curthread->td_ucred, curthread);
	if (error != 0)
		error = priv_check(curthread, PRIV_VFS_MOUNT_PERM);
	if (error != 0) {
		vput(devvp);
		return (error);
	}
	dev = devvp->v_rdev;
	dev_ref(dev);
	g_topology_lock();
	pp = g_dev_getprovider(dev);
	if (pp == NULL)
		error = ENXIO;
	else if (erofs_provider_is_duplicate(em, pp))
		error = EINVAL;
	else
		error = g_vfs_open(devvp, &dif->cp, "erofs", 0);
	if (error == 0) {
		dif->mediasize = dif->cp->provider->mediasize;
		dif->sectorsize = dif->cp->provider->sectorsize;
	}
	g_topology_unlock();
	VOP_UNLOCK(devvp);
	if (error != 0) {
		dev_rel(dev);
		vrele(devvp);
		return (error);
	}
	dif->devvp = devvp;
	dif->dev = dev;
	if (dif->sectorsize == 0 ||
	    (dif->sectorsize & (dif->sectorsize - 1)) != 0) {
		erofs_release_device_info(dif);
		return (EINVAL);
	}
	return (0);
}

static void
erofs_update_iosize_max(struct mount *mp, const struct erofs_device_info *dif)
{
	u_long iosize;

	iosize = dif->dev != NULL && dif->dev->si_iosize_max != 0 ?
	    dif->dev->si_iosize_max : MAXPHYS;
	mp->mnt_iosize_max = MIN(mp->mnt_iosize_max, MIN(iosize, (u_long)MAXPHYS));
}

static void
erofs_free_dev_context(struct erofs_mount *em)
{
	unsigned int i;

	if (em->devs != NULL) {
		for (i = em->extra_devices; i > 0; --i)
			erofs_release_device_info(&em->devs[i - 1]);
		free(em->devs, M_EROFS);
	}
}

static void
erofs_drop_internal_inodes(struct erofs_mount *em)
{
	if (em->metabox_en != NULL)
		free(em->metabox_en, M_EROFS);
	if (em->packed_inode != NULL)
		free(em->packed_inode, M_EROFS);
}

static void
erofs_sb_free(struct erofs_mount *em)
{
	if (em == NULL)
		return;
	z_erofs_extent_cache_fini(em);
	erofs_xattr_prefixes_cleanup(em);
	erofs_drop_internal_inodes(em);
	erofs_free_dev_context(em);
	erofs_release_device_info(&em->dif0);
	free(em, M_EROFS);
}

static int
erofs_superblock_csum_verify(struct erofs_mount *em,
    const struct erofs_super_block *dsb)
{
	uint32_t expected, crc;
	size_t len;
	void *buf;
	int error;

	if ((le32toh(dsb->feature_compat) & EROFS_FEATURE_COMPAT_SB_CHKSUM) == 0)
		return (0);

	len = 1u << dsb->blkszbits;
	if (len > EROFS_SUPER_OFFSET)
		len -= EROFS_SUPER_OFFSET;

	buf = NULL;
	error = erofs_bread(em, EROFS_SUPER_OFFSET, len, &buf);
	if (error != 0)
		return (error);

	crc = calculate_crc32c(EROFS_CRC32C_SEED,
	    (const uint8_t *)buf + offsetof(struct erofs_super_block, checksum) +
	    sizeof(dsb->checksum),
	    len - offsetof(struct erofs_super_block, checksum) -
	    sizeof(dsb->checksum));
	expected = le32toh(dsb->checksum);
	erofs_brelse(buf);

	if (crc != expected) {
		vfs_mount_error(em->mnt,
		    "erofs: invalid superblock checksum 0x%08x, "
		    "0x%08x expected", crc, expected);
		return (EINTEGRITY);
	}
	return (0);
}

static void
erofs_sb_blocks_root(const struct erofs_super_block *dsb, uint32_t incompat,
    uint64_t *blocks, uint64_t *root_nid)
{
	*blocks = le32toh(dsb->blocks_lo);
	if ((incompat & EROFS_FEATURE_INCOMPAT_48BIT) != 0 &&
	    dsb->rootnid_8b != 0) {
		*blocks |= (uint64_t)le16toh(dsb->rb.blocks_hi) << 32;
		*root_nid = le64toh(dsb->rootnid_8b);
	} else {
		*root_nid = le16toh(dsb->rb.rootnid_2b);
	}
}

static int
erofs_validate_device_size(struct erofs_mount *em,
    struct erofs_device_info *dif, erofs_blk_t blocks)
{
	uint64_t bytes;

	if (blocks == 0)
		return (EINTEGRITY);
	if (em->block_size < dif->sectorsize ||
	    em->block_size % dif->sectorsize != 0)
		return (EINVAL);
	if (blocks > (UINT64_MAX >> em->block_bits))
		return (EINTEGRITY);
	bytes = blocks << em->block_bits;
	if (bytes > dif->mediasize)
		return (ENXIO);
	return (0);
}

static int
erofs_init_device(struct erofs_mount *em, struct erofs_device_info *dif,
    const char *path)
{
	struct erofs_device_info opened;
	erofs_blk_t blocks, uniaddr;
	int error;

	blocks = dif->blocks;
	uniaddr = dif->uniaddr;
	error = erofs_open_device(em, path, &opened);
	if (error != 0)
		return (error);
	erofs_update_iosize_max(em->mnt, &opened);
	opened.blocks = blocks;
	opened.uniaddr = uniaddr;
	*dif = opened;
	return (erofs_validate_device_size(em, dif, dif->blocks));
}

static const char *
erofs_device_arg_path(const struct erofs_device_arg *args, unsigned int count,
    unsigned int slot)
{
	unsigned int i;

	for (i = 0; i < count; ++i) {
		if (args[i].slot == slot)
			return (args[i].path);
	}
	return (NULL);
}

static int
erofs_scan_devices(struct erofs_mount *em, const struct erofs_super_block *dsb,
    const struct erofs_device_arg *args, unsigned int arg_count)
{
	struct erofs_deviceslot *slots;
	struct erofs_device_info *dif;
	const char *path;
	uint64_t devt_off, devt_size, image_size, end, other_end;
	erofs_blk_t maxend;
	unsigned int i, j, mask;
	void *buf;
	int error;

	em->total_blocks = em->dif0.blocks;
	em->flatdev_blocks = em->dif0.blocks;
	if (em->extra_devices == 0) {
		if (arg_count != 0) {
			vfs_mount_error(em->mnt,
			    "erofs: external devices given without a device table");
			return (EINVAL);
		}
		return (0);
	}
	devt_off = (uint64_t)le16toh(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
	devt_size = (uint64_t)em->extra_devices * EROFS_DEVT_SLOT_SIZE;
	if (em->dif0.blocks > (UINT64_MAX >> em->block_bits))
		return (EINTEGRITY);
	image_size = em->dif0.blocks << em->block_bits;
	if (devt_off > image_size || devt_size > image_size - devt_off ||
	    devt_size > SIZE_MAX)
		return (EINTEGRITY);
	error = erofs_bread(em, devt_off, (size_t)devt_size, &buf);
	if (error != 0)
		return (error);
	slots = buf;
	em->devs = mallocarray(em->extra_devices, sizeof(*em->devs), M_EROFS,
	    M_WAITOK | M_ZERO);
	maxend = em->dif0.blocks;
	for (i = 0; i < em->extra_devices; ++i) {
		dif = &em->devs[i];
		dif->blocks = le32toh(slots[i].blocks_lo);
		dif->uniaddr = le32toh(slots[i].uniaddr_lo);
		if (erofs_sb_has_48bit(em)) {
			dif->blocks |= (uint64_t)le16toh(slots[i].blocks_hi) << 32;
			dif->uniaddr |= (uint64_t)le16toh(slots[i].uniaddr_hi) << 32;
		}
		if (dif->blocks == 0 ||
		    __builtin_add_overflow(dif->uniaddr, dif->blocks, &end)) {
			error = EINTEGRITY;
			goto out;
		}
		if (end >
		    (erofs_sb_has_48bit(em) ? (1ULL << 48) : (1ULL << 32))) {
			error = EINTEGRITY;
			goto out;
		}
		if (dif->uniaddr != 0 && dif->uniaddr < em->dif0.blocks) {
			error = EINTEGRITY;
			goto out;
		}
		for (j = 0; j < i; ++j) {
			if (dif->uniaddr == 0 || em->devs[j].uniaddr == 0)
				continue;
			if (__builtin_add_overflow(em->devs[j].uniaddr,
			    em->devs[j].blocks, &other_end)) {
				error = EINTEGRITY;
				goto out;
			}
			if (dif->uniaddr < other_end && em->devs[j].uniaddr < end) {
				error = EINTEGRITY;
				goto out;
			}
		}
		if (__builtin_add_overflow(em->total_blocks, dif->blocks,
		    &em->total_blocks)) {
			error = EOVERFLOW;
			goto out;
		}
		maxend = MAX(maxend, (erofs_blk_t)end);
	}
	erofs_brelse(buf);
	buf = NULL;
	em->flatdev_blocks = maxend;
	mask = 1;
	while (mask < (unsigned int)em->extra_devices + 1)
		mask <<= 1;
	em->device_id_mask = mask - 1;
	em->flatdev = arg_count == 0;
	if (em->flatdev)
		return (erofs_validate_device_size(em, &em->dif0,
		    em->flatdev_blocks));
	if (arg_count != em->extra_devices) {
		vfs_mount_error(em->mnt,
		    "erofs: external devices don't match (ondisk %u, given %u)",
		    em->extra_devices, arg_count);
		return (arg_count < em->extra_devices ? ENXIO : EINVAL);
	}
	for (i = 0; i < arg_count; ++i) {
		if (args[i].slot == 0 || args[i].slot > em->extra_devices)
			return (EINVAL);
	}
	for (i = 0; i < em->extra_devices; ++i) {
		path = erofs_device_arg_path(args, arg_count, i + 1);
		if (path == NULL)
			return (ENXIO);
		error = erofs_init_device(em, &em->devs[i], path);
		if (error != 0)
			return (error);
	}
	return (0);
out:
	erofs_brelse(buf);
	return (error);
}

static int
erofs_init_packed_inode(struct erofs_mount *em)
{
	int error;

	/* Load the packed carrier before any fragment-backed metabox inode. */
	if ((em->feature_incompat & EROFS_FEATURE_INCOMPAT_FRAGMENTS) != 0 &&
	    em->packed_nid > 0) {
		em->packed_inode = malloc(sizeof(*em->packed_inode), M_EROFS,
		    M_WAITOK | M_ZERO);
		if (em->packed_inode == NULL)
			return (ENOMEM);
		error = erofs_read_inode(em, em->packed_nid, em->packed_inode);
		if (error != 0) {
			free(em->packed_inode, M_EROFS);
			em->packed_inode = NULL;
			return (error);
		}
		if (em->packed_inode->vtype != VREG || em->packed_inode->fragment) {
			vfs_mount_error(em->mnt,
			    "erofs: packed inode nid=%ju is not a non-recursive regular file",
			    (uintmax_t)em->packed_nid);
			return (EINTEGRITY);
		}
	}
	return (0);
}

static int
erofs_init_metabox_inode(struct erofs_mount *em)
{
	int error;

	/*
	 * METABOX NIDs address inode slots in this backing inode's data.  The
	 * packed carrier is ready first so a compressed metabox may legally end in
	 * a fragment pcluster without reading an uninitialized dependency.
	 */
	if (erofs_sb_has_metabox(em)) {
		struct erofs_map_blocks map;

		em->metabox_en = malloc(sizeof(*em->metabox_en), M_EROFS,
		    M_WAITOK | M_ZERO);
		if (em->metabox_en == NULL)
			return (ENOMEM);
		error = erofs_read_inode(em, em->metabox_nid, em->metabox_en);
		if (error != 0)
			return (error);
		if (em->metabox_en->vtype != VREG) {
			vfs_mount_error(em->mnt,
			    "erofs: metabox inode nid=%ju is not a regular file",
			    (uintmax_t)em->metabox_nid);
			return (EINTEGRITY);
		}
		if (em->metabox_en->fragment) {
			if (em->packed_inode == NULL ||
			    em->packed_inode->nid == em->metabox_en->nid ||
			    em->metabox_en->size == 0)
				return (EINTEGRITY);
			bzero(&map, sizeof(map));
			map.m_la = em->metabox_en->size - 1;
			error = z_erofs_map_blocks_iter(em, em->metabox_en, &map,
			    EROFS_GET_BLOCKS_FIEMAP);
			if (error != 0 || (map.m_flags & EROFS_MAP_FRAGMENT) == 0)
				return (error != 0 ? error : EINTEGRITY);
		}
	}
	return (0);
}

static int
erofs_mountfs(struct erofs_device_info *primary, struct mount *mp,
    const struct erofs_device_arg *args, unsigned int arg_count)
{
	struct erofs_mount *em;
	struct erofs_super_block *dsb;
	uint32_t unsupported;
	void *buf;
	int error;

	em = malloc(sizeof(*em), M_EROFS, M_WAITOK | M_ZERO);
	em->mnt = mp;
	z_erofs_extent_cache_init(em);
	em->dif0 = *primary;
	bzero(primary, sizeof(*primary));
	buf = NULL;

	error = erofs_bread(em, EROFS_SUPER_OFFSET, sizeof(*dsb), &buf);
	if (error != 0)
		goto fail;
	dsb = buf;
	if (le32toh(dsb->magic) != EROFS_SUPER_MAGIC_V1) {
		error = EINVAL;
		goto fail;
	}
	if (dsb->blkszbits < 9 || dsb->blkszbits > PAGE_SHIFT) {
		error = EINVAL;
		goto fail;
	}
	if (dsb->dirblkbits != 0) {
		error = EOPNOTSUPP;
		goto fail;
	}
	em->feature_compat = le32toh(dsb->feature_compat);
	em->feature_incompat = le32toh(dsb->feature_incompat);
	em->packed_nid = le64toh(dsb->packed_nid);
	em->extra_devices = erofs_sb_has_device_table(em) ?
	    le16toh(dsb->extra_devices) : 0;
	unsupported = em->feature_incompat & ~EROFS_ALL_SUPPORTED_INCOMPAT;
	/*
	 * Narrowly allow one extra combination: long xattr prefixes enabled
	 * with non-plain prefix table stored in a packed inode, which adds
	 * the FRAGMENTS (0x20) incompat bit.  This is NOT a declaration of
	 * general fragments support; per-inode data layout is still gated
	 * by plain/inline checks in erofs_read_inode().
	 */
	if (unsupported != 0) {
		if (unsupported != EROFS_FEATURE_INCOMPAT_FRAGMENTS ||
		    (em->feature_incompat &
			EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES) == 0 ||
		    (em->feature_compat &
			EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX) != 0 ||
		    em->packed_nid == 0) {
			error = EOPNOTSUPP;
			goto fail;
		}
	}
	em->block_bits = dsb->blkszbits;
	em->block_size = 1u << em->block_bits;
	em->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE;
	if (em->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) {
		error = EINVAL;
		goto fail;
	}
	em->meta_blkaddr = le32toh(dsb->meta_blkaddr);
	em->xattr_blkaddr = le32toh(dsb->xattr_blkaddr);
	em->xattr_prefix_start = le32toh(dsb->xattr_prefix_start);
	em->xattr_prefix_count = dsb->xattr_prefix_count;
	if (erofs_sb_has_ishare_xattrs(em) &&
	    dsb->ishare_xattr_prefix_id >= em->xattr_prefix_count) {
		error = EINTEGRITY;
		goto fail;
	}
	/* A non-zero reserved value disables the current name-filter format. */
	if (erofs_sb_has_xattr_filter(em) && dsb->xattr_filter_reserved != 0)
		em->feature_compat &= ~EROFS_FEATURE_COMPAT_XATTR_FILTER;
	erofs_sb_blocks_root(dsb, em->feature_incompat, &em->blocks,
	    &em->root_nid);
	em->dif0.blocks = em->blocks;
	error = erofs_validate_device_size(em, &em->dif0, em->dif0.blocks);
	if (error != 0)
		goto fail;
	error = erofs_superblock_csum_verify(em, dsb);
	if (error != 0)
		goto fail;
	em->inos = le64toh(dsb->inos);
	em->epoch = le64toh(dsb->epoch);
	em->fixed_nsec = le32toh(dsb->fixed_nsec);
	if (em->fixed_nsec >= 1000000000) {
		error = EINTEGRITY;
		goto fail;
	}
	error = erofs_load_generation_seed(em, em->sb_size,
	    &em->generation_seed);
	if (error != 0)
		goto fail;
	if (em->packed_nid != 0 && erofs_nid_in_metabox(em->packed_nid)) {
		error = EINTEGRITY;
		goto fail;
	}
	if (erofs_sb_has_metabox(em)) {
		if (em->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) {
			error = EINTEGRITY;
			goto fail;
		}
		em->metabox_nid = le64toh(dsb->metabox_nid);
		if (erofs_nid_in_metabox(em->metabox_nid)) {
			error = EINTEGRITY;
			goto fail;
		}
	}

	error = z_erofs_parse_cfgs(em, dsb);
	if (error != 0)
		goto fail;
	error = erofs_scan_devices(em, dsb, args, arg_count);
	if (error != 0)
		goto fail;

	if (erofs_sb_has_shared_ea_in_metabox(em) &&
	    !erofs_sb_has_metabox(em)) {
		error = EINTEGRITY;
		goto fail;
	}

	error = erofs_init_packed_inode(em);
	if (error != 0)
		goto fail;
	error = erofs_init_metabox_inode(em);
	if (error != 0)
		goto fail;
	error = erofs_xattr_prefixes_init(em);
	if (error != 0)
		goto fail;
	set_opt(&em->opt, POSIX_ACL);
	memcpy(em->volume_name, dsb->volume_name, 16);
	em->volume_name[16] = '\0';

	erofs_brelse(buf);
	buf = NULL;
	mp->mnt_data = em;
	mp->mnt_stat.f_fsid.val[0] = dev2udev(em->dif0.devvp->v_rdev);
	mp->mnt_stat.f_fsid.val[1] = mp->mnt_vfc->vfc_typenum;
	MNT_ILOCK(mp);
	mp->mnt_flag |= MNT_LOCAL | MNT_RDONLY | MNT_ACLS;
	mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED |
	    MNTK_USES_BCACHE;
	MNT_IUNLOCK(mp);
	return (0);
fail:
	if (buf != NULL)
		erofs_brelse(buf);
	erofs_sb_free(em);
	return (error);
}

static int
erofs_mount(struct mount *mp)
{
	struct erofs_device_arg *args;
	struct erofs_device_info primary;
	char *fspec;
	unsigned int arg_count;
	int error, len;

	MNT_ILOCK(mp);
	mp->mnt_flag |= MNT_RDONLY;
	MNT_IUNLOCK(mp);
	if (mp->mnt_flag & MNT_UPDATE) {
		if (vfs_flagopt(mp->mnt_optnew, "export", NULL, 0))
			return (0);
		return (EOPNOTSUPP);
	}
	args = NULL;
	arg_count = 0;
	error = erofs_parse_device_options(mp, &args, &arg_count);
	if (error != 0)
		return (error);
	if (vfs_filteropt(mp->mnt_optnew, erofs_opts) != 0) {
		erofs_free_device_args(args, arg_count);
		return (EINVAL);
	}
	fspec = NULL;
	error = vfs_getopt(mp->mnt_optnew, "from", (void **)&fspec, &len);
	if (error != 0 || fspec == NULL || len == 0 ||
	    fspec[len - 1] != '\0') {
		erofs_free_device_args(args, arg_count);
		return (EINVAL);
	}
	mp->mnt_iosize_max = MAXPHYS;
	error = erofs_open_device(NULL, fspec, &primary);
	if (error != 0) {
		erofs_free_device_args(args, arg_count);
		return (error);
	}
	erofs_update_iosize_max(mp, &primary);
	error = erofs_mountfs(&primary, mp, args, arg_count);
	erofs_free_device_args(args, arg_count);
	if (error != 0)
		return (error);
	vfs_mountedfrom(mp, fspec);
	return (erofs_statfs(mp, &mp->mnt_stat));
}

static int
erofs_root(struct mount *mp, int flags, struct vnode **vpp)
{
	int error;

	error = erofs_vget(mp, MTOE(mp)->root_nid, flags, vpp);
	if (error != 0)
		vfs_mount_error(mp, "erofs: failed to load root nid %ju: error %d",
		    (uintmax_t)MTOE(mp)->root_nid, error);
	return (error);
}

static int
erofs_statfs(struct mount *mp, struct statfs *sbp)
{
	struct erofs_mount *em;

	em = MTOE(mp);
	sbp->f_bsize = em->block_size;
	sbp->f_iosize = em->block_size;
	sbp->f_blocks = em->total_blocks;
	sbp->f_bfree = 0;
	sbp->f_bavail = 0;
	sbp->f_files = em->inos;
	sbp->f_ffree = 0;
	return (0);
}

static int
erofs_unmount(struct mount *mp, int mntflags)
{
	struct erofs_mount *em;
	int error, flags;

	flags = ((mntflags & MNT_FORCE) != 0) ? FORCECLOSE : 0;
	error = vflush(mp, 0, flags, curthread);
	if (error != 0)
		return (error);
	em = MTOE(mp);
	mp->mnt_data = NULL;
	erofs_sb_free(em);
	return (0);
}

static int
erofs_vgetf(struct mount *mp, ino_t ino, int flags, struct vnode **vpp)
{
	return (erofs_vget(mp, ino, flags, vpp));
}

/* Persistent EROFS file handle to locked vnode. */
static int
erofs_fhtovp(struct mount *mp, struct fid *fhp, int flags, struct vnode **vpp)
{
	struct erofs_fid efid;
	struct erofs_node *en;
	struct vnode *vp;
	uint64_t nid;
	int error;

	*vpp = NULLVP;
	bzero(&efid, sizeof(efid));
	memcpy(&efid, fhp, sizeof(efid));
	if (efid.len != sizeof(efid) || efid.pad != 0)
		return (EINVAL);
	nid = ((uint64_t)efid.nid_hi << 32) | efid.nid_lo;
	if (!erofs_nid_is_valid(MTOE(mp), nid))
		return (ESTALE);
	error = VFS_VGET(mp, (ino_t)nid, flags, &vp);
	if (error != 0)
		return (error);
	en = VTOE(vp);
	if (en->mode == 0 || en->nlink == 0 || en->nid != nid ||
	    en->generation != efid.gen) {
		vput(vp);
		return (ESTALE);
	}
	*vpp = vp;
	return (0);
}

static struct vfsops erofs_vfsops = {
	.vfs_fhtovp = erofs_fhtovp,
	.vfs_mount = erofs_mount,
	.vfs_root = erofs_root,
	.vfs_statfs = erofs_statfs,
	.vfs_unmount = erofs_unmount,
	.vfs_vget = erofs_vgetf,
};
VFS_SET(erofs_vfsops, erofs, VFCF_READONLY);
MODULE_DEPEND(erofs, acl_posix1e, 1, 1, 1);
MODULE_DEPEND(erofs, zlib, 1, 1, 1);
MODULE_VERSION(erofs, 1);