erofs-freebsd-out-tree/zutil.c

// SPDX-License-Identifier: GPL-2.0-only
/* FreeBSD stream context pool and compression runtime accounting. */

#include <sys/param.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/sysctl.h>

#include <vm/uma.h>

#include "internal.h"

#define EROFS_STREAM_MOUNT_HARD_BUDGET (32UL * 1024 * 1024)
#define EROFS_STREAM_GLOBAL_HARD_BUDGET (128UL * 1024 * 1024)
#define EROFS_STREAM_ALLOCATION_HARD_MAX (4UL * 1024 * 1024)
#define EROFS_STREAM_MOUNT_HARD_CONTEXTS (16U)
#define EROFS_STREAM_GLOBAL_HARD_CONTEXTS (64U)
#define EROFS_STREAM_MOUNT_CACHED_PER_CODEC (2U)
#define EROFS_STREAM_GLOBAL_CACHED_PER_CODEC (16U)

static unsigned long erofs_stream_mount_budget =
    EROFS_STREAM_MOUNT_HARD_BUDGET;
static unsigned long erofs_stream_global_budget =
    EROFS_STREAM_GLOBAL_HARD_BUDGET;
static int erofs_stream_mount_contexts =
    EROFS_STREAM_MOUNT_HARD_CONTEXTS;
static int erofs_stream_global_contexts_limit =
    EROFS_STREAM_GLOBAL_HARD_CONTEXTS;
static int erofs_stream_mount_cached =
    EROFS_STREAM_MOUNT_CACHED_PER_CODEC;
static int erofs_stream_global_cached =
    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC;

TUNABLE_ULONG("vfs.erofs.stream_pool.mount_budget",
    &erofs_stream_mount_budget);
TUNABLE_ULONG("vfs.erofs.stream_pool.global_budget",
    &erofs_stream_global_budget);
TUNABLE_INT("vfs.erofs.stream_pool.mount_contexts",
    &erofs_stream_mount_contexts);
TUNABLE_INT("vfs.erofs.stream_pool.global_contexts",
    &erofs_stream_global_contexts_limit);
TUNABLE_INT("vfs.erofs.stream_pool.mount_cached_per_codec",
    &erofs_stream_mount_cached);
TUNABLE_INT("vfs.erofs.stream_pool.global_cached_per_codec",
    &erofs_stream_global_cached);

static struct mtx erofs_stream_lock;
static struct cv erofs_stream_global_cv;
static unsigned int erofs_stream_global_waiters;
static LIST_HEAD(, erofs_sb_info) erofs_stream_mounts =
    LIST_HEAD_INITIALIZER(erofs_stream_mounts);
static uma_zone_t erofs_stream_zone;
static eventhandler_tag erofs_stream_lowmem_tag;
static size_t erofs_stream_global_resident;
static unsigned int erofs_stream_global_contexts;
static unsigned int erofs_stream_global_cached_by_codec[
    Z_EROFS_COMPRESSION_MAX];
static unsigned long erofs_stream_cached_contexts;
static unsigned long erofs_stream_borrowed_contexts;
static unsigned long erofs_stream_idle_contexts;
static unsigned long erofs_stream_creations;
static unsigned long erofs_stream_reuses;
static unsigned long erofs_stream_fallbacks;
static unsigned long erofs_stream_exhaustions;
static unsigned long erofs_stream_waits;
static unsigned long erofs_stream_destroys;
static unsigned long erofs_stream_reclaims;

MTX_SYSINIT(erofs_stream_pool, &erofs_stream_lock, "erofs stream pool",
    MTX_DEF);

SYSCTL_DECL(_vfs_erofs);
SYSCTL_NODE(_vfs_erofs, OID_AUTO, stream_pool,
    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "EROFS stream context pool");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, mount_budget, CTLFLAG_RD,
    &erofs_stream_mount_budget, 0, "Effective per-mount byte budget");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, global_budget, CTLFLAG_RD,
    &erofs_stream_global_budget, 0, "Effective global byte budget");
SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, mount_contexts, CTLFLAG_RD,
    &erofs_stream_mount_contexts, 0,
    "Effective per-mount, per-algorithm context limit");
SYSCTL_INT(_vfs_erofs_stream_pool, OID_AUTO, global_contexts_limit,
    CTLFLAG_RD, &erofs_stream_global_contexts_limit, 0,
    "Effective global context limit");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, resident_bytes, CTLFLAG_RD,
    &erofs_stream_global_resident, 0, "Currently charged global bytes");
SYSCTL_UINT(_vfs_erofs_stream_pool, OID_AUTO, contexts, CTLFLAG_RD,
    &erofs_stream_global_contexts, 0, "Currently allocated contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, cached, CTLFLAG_RD,
    &erofs_stream_cached_contexts, 0, "Currently cacheable contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, borrowed, CTLFLAG_RD,
    &erofs_stream_borrowed_contexts, 0, "Currently borrowed contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, idle, CTLFLAG_RD,
    &erofs_stream_idle_contexts, 0, "Currently idle contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, creations, CTLFLAG_RD,
    &erofs_stream_creations, 0, "Successful context creations");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reuses, CTLFLAG_RD,
    &erofs_stream_reuses, 0, "Idle context acquisitions");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, fallbacks, CTLFLAG_RD,
    &erofs_stream_fallbacks, 0, "Bounded temporary context fallbacks");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, exhaustions, CTLFLAG_RD,
    &erofs_stream_exhaustions, 0, "Unsatisfiable limit or budget rejections");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, waits, CTLFLAG_RD,
    &erofs_stream_waits, 0, "Sleeps waiting for a context or budget release");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, destroys, CTLFLAG_RD,
    &erofs_stream_destroys, 0, "Destroyed contexts");
SYSCTL_ULONG(_vfs_erofs_stream_pool, OID_AUTO, reclaims, CTLFLAG_RD,
    &erofs_stream_reclaims, 0, "Low-memory idle context reclaims");

struct erofs_stream_alloc_header {
	struct erofs_stream_ctx *ctx;
	size_t bytes;
};

static bool
erofs_stream_mount_budget_available_locked(const struct erofs_sb_info *sbi,
    size_t bytes)
{

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	return (bytes <= erofs_stream_mount_budget &&
	    sbi->stream_pool.resident_bytes <= erofs_stream_mount_budget -
	    bytes);
}

static bool
erofs_stream_global_budget_available_locked(size_t bytes)
{

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	return (bytes <= erofs_stream_global_budget &&
	    erofs_stream_global_resident <= erofs_stream_global_budget - bytes);
}

static bool
erofs_stream_reserve_locked(struct erofs_sb_info *sbi, size_t bytes)
{
	struct erofs_stream_pool *pool;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	pool = &sbi->stream_pool;
	if (!erofs_stream_mount_budget_available_locked(sbi, bytes) ||
	    !erofs_stream_global_budget_available_locked(bytes))
		return (false);
	pool->resident_bytes += bytes;
	erofs_stream_global_resident += bytes;
	return (true);
}

/*
 * Wake threads sleeping in z_erofs_stream_ctx_get() after a context or byte
 * budget was released.  Mount-level sleepers and the unmount drain share the
 * per-mount cv; sleepers blocked on a global limit use the global cv.
 */
static void
erofs_stream_wake_locked(struct erofs_sb_info *sbi)
{

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	if (sbi->stream_pool.waiters != 0 || sbi->stream_pool.closing)
		cv_broadcast(&sbi->stream_pool.cv);
	if (erofs_stream_global_waiters != 0)
		cv_broadcast(&erofs_stream_global_cv);
}

static void
erofs_stream_release_locked(struct erofs_sb_info *sbi, size_t bytes)
{

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	KASSERT(bytes <= sbi->stream_pool.resident_bytes &&
	    bytes <= erofs_stream_global_resident,
	    ("erofs stream pool byte accounting underflow"));
	sbi->stream_pool.resident_bytes -= bytes;
	erofs_stream_global_resident -= bytes;
}

static struct erofs_stream_ctx *
erofs_stream_take_idle_locked(struct erofs_sb_info *sbi)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_stream_ctx *ctx;
	unsigned int algorithm;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
		codec = &sbi->stream_pool.codec[algorithm];
		ctx = STAILQ_FIRST(&codec->idle);
		if (ctx == NULL)
			continue;
		STAILQ_REMOVE_HEAD(&codec->idle, link);
		KASSERT(codec->idle_count != 0 &&
		    erofs_stream_idle_contexts != 0,
		    ("erofs stream idle accounting underflow"));
		--codec->idle_count;
		--erofs_stream_idle_contexts;
		++codec->borrowed;
		++erofs_stream_borrowed_contexts;
		return (ctx);
	}
	return (NULL);
}

/*
 * Detach one idle context so that the caller can destroy it after dropping
 * the lock.  Idle contexts of this mount are preferred; other mounts are only
 * considered when the limit being hit is a global one.
 */
static struct erofs_stream_ctx *
erofs_stream_take_reclaimable_locked(struct erofs_sb_info *sbi, bool global)
{
	struct erofs_stream_ctx *ctx;
	struct erofs_sb_info *other;

	ctx = erofs_stream_take_idle_locked(sbi);
	if (ctx != NULL || !global)
		return (ctx);
	LIST_FOREACH(other, &erofs_stream_mounts, stream_pool_link) {
		ctx = erofs_stream_take_idle_locked(other);
		if (ctx != NULL)
			break;
	}
	return (ctx);
}

static void erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx);

/*
 * Charging never sleeps for budget: the caller already owns a context, and
 * waiting here would hold part of the budget while waiting for the rest.  Idle
 * contexts are the only budget that can be returned immediately, so reclaim
 * them first and fail with ENOMEM only when the budget is truly exhausted.
 */
int
z_erofs_stream_ctx_charge(struct erofs_stream_ctx *ctx, size_t bytes)
{
	struct erofs_stream_ctx *idle;
	bool reserved;

	if (bytes == 0 || bytes > EROFS_STREAM_ALLOCATION_HARD_MAX) {
		ctx->allocation_failed = true;
		mtx_lock(&erofs_stream_lock);
		++erofs_stream_exhaustions;
		mtx_unlock(&erofs_stream_lock);
		return (ENOMEM);
	}
	mtx_lock(&erofs_stream_lock);
	for (;;) {
		if (ctx->charged_bytes > SIZE_MAX - bytes ||
		    bytes > erofs_stream_mount_budget ||
		    bytes > erofs_stream_global_budget) {
			reserved = false;
			break;
		}
		reserved = erofs_stream_reserve_locked(ctx->sbi, bytes);
		if (reserved)
			break;
		idle = erofs_stream_take_reclaimable_locked(ctx->sbi,
		    erofs_stream_mount_budget_available_locked(ctx->sbi,
		    bytes));
		if (idle == NULL)
			break;
		mtx_unlock(&erofs_stream_lock);
		erofs_stream_ctx_destroy(idle);
		mtx_lock(&erofs_stream_lock);
	}
	if (reserved)
		ctx->charged_bytes += bytes;
	else
		++erofs_stream_exhaustions;
	mtx_unlock(&erofs_stream_lock);
	if (!reserved) {
		ctx->allocation_failed = true;
		return (ENOMEM);
	}
	return (0);
}

void
z_erofs_stream_ctx_uncharge(struct erofs_stream_ctx *ctx, size_t bytes)
{

	mtx_lock(&erofs_stream_lock);
	KASSERT(bytes <= ctx->charged_bytes,
	    ("erofs stream context byte accounting underflow"));
	ctx->charged_bytes -= bytes;
	erofs_stream_release_locked(ctx->sbi, bytes);
	erofs_stream_wake_locked(ctx->sbi);
	mtx_unlock(&erofs_stream_lock);
}

void *
z_erofs_stream_ctx_alloc(struct erofs_stream_ctx *ctx, size_t bytes)
{
	struct erofs_stream_alloc_header *header;
	size_t total;

	if (__builtin_add_overflow(bytes, sizeof(*header), &total) ||
	    z_erofs_stream_ctx_charge(ctx, total) != 0)
		return (NULL);
	header = malloc(total, M_EROFS, M_WAITOK | M_ZERO);
	header->ctx = ctx;
	header->bytes = total;
	return (header + 1);
}

void
z_erofs_stream_ctx_free(struct erofs_stream_ctx *ctx, void *address)
{
	struct erofs_stream_alloc_header *header;
	size_t bytes;

	if (address == NULL)
		return;
	header = (struct erofs_stream_alloc_header *)address - 1;
	KASSERT(header->ctx == ctx, ("erofs stream allocation owner mismatch"));
	bytes = header->bytes;
	free(header, M_EROFS);
	z_erofs_stream_ctx_uncharge(ctx, bytes);
}

static void
erofs_stream_ctx_destroy(struct erofs_stream_ctx *ctx)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_sb_info *sbi;
	bool cached;
	uint8_t algorithm;

	sbi = ctx->sbi;
	algorithm = ctx->algorithm;
	cached = ctx->cached;
	ctx->fini(ctx);
	KASSERT(ctx->charged_bytes == EROFS_STREAM_CTX_WRAPPER_SIZE,
	    ("erofs stream backend allocation leaked"));
	uma_zfree(erofs_stream_zone, ctx);

	mtx_lock(&erofs_stream_lock);
	codec = &sbi->stream_pool.codec[algorithm];
	KASSERT(codec->contexts != 0 && codec->borrowed != 0 &&
	    erofs_stream_global_contexts != 0 &&
	    erofs_stream_borrowed_contexts != 0,
	    ("erofs stream context accounting underflow"));
	--codec->contexts;
	--codec->borrowed;
	--erofs_stream_global_contexts;
	--erofs_stream_borrowed_contexts;
	if (cached) {
		KASSERT(codec->cached != 0 &&
		    erofs_stream_global_cached_by_codec[algorithm] != 0 &&
		    erofs_stream_cached_contexts != 0,
		    ("erofs stream cache accounting underflow"));
		--codec->cached;
		--erofs_stream_global_cached_by_codec[algorithm];
		--erofs_stream_cached_contexts;
	}
	erofs_stream_release_locked(sbi, EROFS_STREAM_CTX_WRAPPER_SIZE);
	++erofs_stream_destroys;
	erofs_stream_wake_locked(sbi);
	mtx_unlock(&erofs_stream_lock);
}

/*
 * Sleep until a context or budget is released.  Limits that only this mount
 * can release are waited for on the mount cv; anything involving a global
 * limit is waited for on the global cv, which every release also signals.
 */
static void
erofs_stream_wait_locked(struct erofs_sb_info *sbi, bool global)
{
	struct erofs_stream_pool *pool;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	pool = &sbi->stream_pool;
	++pool->waiters;
	++erofs_stream_waits;
	if (global) {
		++erofs_stream_global_waiters;
		cv_wait(&erofs_stream_global_cv, &erofs_stream_lock);
		--erofs_stream_global_waiters;
	} else {
		cv_wait(&pool->cv, &erofs_stream_lock);
	}
	KASSERT(pool->waiters != 0, ("erofs stream waiter underflow"));
	--pool->waiters;
	if (pool->closing)
		cv_broadcast(&pool->cv);
}

int
z_erofs_stream_ctx_get(struct erofs_sb_info *sbi, uint8_t algorithm,
    size_t context_size, erofs_stream_ctx_init_t *init,
    erofs_stream_ctx_fini_t *fini, struct erofs_stream_ctx **ctxp)
{
	struct erofs_stream_pool_codec *codec;
	struct erofs_stream_ctx *ctx;
	bool cached, global_full, mount_full, reserved __diagused;
	int error;

	*ctxp = NULL;
	if (algorithm == Z_EROFS_COMPRESSION_LZ4 ||
	    algorithm >= Z_EROFS_COMPRESSION_MAX ||
	    context_size < sizeof(*ctx) ||
	    context_size > EROFS_STREAM_CTX_WRAPPER_SIZE)
		return (EINVAL);
	mtx_lock(&erofs_stream_lock);
	for (;;) {
		if (!sbi->stream_pool_initialized || sbi->stream_pool.closing) {
			mtx_unlock(&erofs_stream_lock);
			return (ENXIO);
		}
		codec = &sbi->stream_pool.codec[algorithm];
		ctx = STAILQ_FIRST(&codec->idle);
		if (ctx != NULL) {
			STAILQ_REMOVE_HEAD(&codec->idle, link);
			KASSERT(codec->idle_count != 0 &&
			    erofs_stream_idle_contexts != 0,
			    ("erofs stream idle accounting underflow"));
			--codec->idle_count;
			--erofs_stream_idle_contexts;
			++codec->borrowed;
			++erofs_stream_borrowed_contexts;
			++erofs_stream_reuses;
			ctx->allocation_failed = false;
			mtx_unlock(&erofs_stream_lock);
			*ctxp = ctx;
			return (0);
		}
		/*
		 * Limits that no release can ever satisfy are configuration
		 * errors; waiting for them would never end.
		 */
		if (erofs_stream_mount_contexts == 0 ||
		    erofs_stream_global_contexts_limit == 0 ||
		    EROFS_STREAM_CTX_WRAPPER_SIZE > erofs_stream_mount_budget ||
		    EROFS_STREAM_CTX_WRAPPER_SIZE >
		    erofs_stream_global_budget) {
			++erofs_stream_exhaustions;
			mtx_unlock(&erofs_stream_lock);
			return (ENOMEM);
		}
		mount_full = codec->contexts >=
		    (unsigned int)erofs_stream_mount_contexts;
		global_full = erofs_stream_global_contexts >=
		    (unsigned int)erofs_stream_global_contexts_limit ||
		    !erofs_stream_global_budget_available_locked(
		    EROFS_STREAM_CTX_WRAPPER_SIZE);
		if (!mount_full && !global_full &&
		    erofs_stream_mount_budget_available_locked(sbi,
		    EROFS_STREAM_CTX_WRAPPER_SIZE))
			break;
		/*
		 * Contexts of this codec are all borrowed when mount_full, so
		 * only a release can help.  Otherwise idle contexts of other
		 * codecs (or mounts, for a global limit) hold the resource and
		 * are never released unless reclaimed here.
		 */
		if (!mount_full) {
			ctx = erofs_stream_take_reclaimable_locked(sbi,
			    global_full);
			if (ctx != NULL) {
				mtx_unlock(&erofs_stream_lock);
				erofs_stream_ctx_destroy(ctx);
				mtx_lock(&erofs_stream_lock);
				continue;
			}
		}
		erofs_stream_wait_locked(sbi, !mount_full && global_full);
	}
	reserved = erofs_stream_reserve_locked(sbi,
	    EROFS_STREAM_CTX_WRAPPER_SIZE);
	KASSERT(reserved, ("erofs stream wrapper reservation failed"));
	cached = codec->cached < (unsigned int)erofs_stream_mount_cached &&
	    erofs_stream_global_cached_by_codec[algorithm] <
	    (unsigned int)erofs_stream_global_cached;
	++codec->contexts;
	++codec->borrowed;
	++erofs_stream_global_contexts;
	++erofs_stream_borrowed_contexts;
	if (cached) {
		++codec->cached;
		++erofs_stream_global_cached_by_codec[algorithm];
		++erofs_stream_cached_contexts;
	} else {
		++erofs_stream_fallbacks;
	}
	mtx_unlock(&erofs_stream_lock);

	ctx = uma_zalloc(erofs_stream_zone, M_WAITOK | M_ZERO);
	ctx->sbi = sbi;
	ctx->fini = fini;
	ctx->charged_bytes = EROFS_STREAM_CTX_WRAPPER_SIZE;
	ctx->algorithm = algorithm;
	ctx->cached = cached;
	error = init(ctx);
	if (error != 0) {
		erofs_stream_ctx_destroy(ctx);
		return (error > 0 ? error : EIO);
	}
	mtx_lock(&erofs_stream_lock);
	++erofs_stream_creations;
	mtx_unlock(&erofs_stream_lock);
	*ctxp = ctx;
	return (0);
}

void
z_erofs_stream_ctx_put(struct erofs_stream_ctx *ctx, bool reusable)
{
	struct erofs_stream_pool_codec *codec;

	mtx_lock(&erofs_stream_lock);
	codec = &ctx->sbi->stream_pool.codec[ctx->algorithm];
	if (reusable && ctx->cached && !ctx->sbi->stream_pool.closing) {
		KASSERT(codec->borrowed != 0 &&
		    erofs_stream_borrowed_contexts != 0,
		    ("erofs stream borrowed accounting underflow"));
		--codec->borrowed;
		--erofs_stream_borrowed_contexts;
		STAILQ_INSERT_HEAD(&codec->idle, ctx, link);
		++codec->idle_count;
		++erofs_stream_idle_contexts;
		erofs_stream_wake_locked(ctx->sbi);
		mtx_unlock(&erofs_stream_lock);
		return;
	}
	mtx_unlock(&erofs_stream_lock);
	erofs_stream_ctx_destroy(ctx);
}

static bool
erofs_stream_pool_empty_locked(const struct erofs_stream_pool *pool)
{
	unsigned int algorithm;

	mtx_assert(&erofs_stream_lock, MA_OWNED);
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm) {
		if (pool->codec[algorithm].contexts != 0)
			return (false);
	}
	return (true);
}

void
z_erofs_stream_pool_init(struct erofs_sb_info *sbi)
{
	unsigned int algorithm;

	bzero(&sbi->stream_pool, sizeof(sbi->stream_pool));
	for (algorithm = 0; algorithm < Z_EROFS_COMPRESSION_MAX; ++algorithm)
		STAILQ_INIT(&sbi->stream_pool.codec[algorithm].idle);
	cv_init(&sbi->stream_pool.cv, "erofs stream drain");
	mtx_lock(&erofs_stream_lock);
	sbi->stream_pool_initialized = true;
	LIST_INSERT_HEAD(&erofs_stream_mounts, sbi, stream_pool_link);
	mtx_unlock(&erofs_stream_lock);
}

void
z_erofs_stream_pool_fini(struct erofs_sb_info *sbi)
{
	struct erofs_stream_ctx *ctx;

	if (!sbi->stream_pool_initialized)
		return;
	mtx_lock(&erofs_stream_lock);
	sbi->stream_pool.closing = true;
	erofs_stream_wake_locked(sbi);
	for (;;) {
		ctx = erofs_stream_take_idle_locked(sbi);
		if (ctx == NULL)
			break;
		mtx_unlock(&erofs_stream_lock);
		erofs_stream_ctx_destroy(ctx);
		mtx_lock(&erofs_stream_lock);
	}
	while (!erofs_stream_pool_empty_locked(&sbi->stream_pool) ||
	    sbi->stream_pool.waiters != 0)
		cv_wait(&sbi->stream_pool.cv, &erofs_stream_lock);
	KASSERT(sbi->stream_pool.resident_bytes == 0,
	    ("erofs stream mount bytes remain at unmount"));
	LIST_REMOVE(sbi, stream_pool_link);
	sbi->stream_pool_initialized = false;
	mtx_unlock(&erofs_stream_lock);
	cv_destroy(&sbi->stream_pool.cv);
}

static void
erofs_stream_pool_lowmem(void *arg __unused, int howto __unused)
{
	struct erofs_stream_ctx *ctx;
	struct erofs_sb_info *sbi;

	for (;;) {
		ctx = NULL;
		mtx_lock(&erofs_stream_lock);
		LIST_FOREACH(sbi, &erofs_stream_mounts, stream_pool_link) {
			ctx = erofs_stream_take_idle_locked(sbi);
			if (ctx != NULL) {
				++erofs_stream_reclaims;
				break;
			}
		}
		mtx_unlock(&erofs_stream_lock);
		if (ctx == NULL)
			break;
		erofs_stream_ctx_destroy(ctx);
	}
}

static void
erofs_stream_pool_global_init(void *arg __unused)
{
	erofs_stream_mount_budget = MIN(erofs_stream_mount_budget,
	    EROFS_STREAM_MOUNT_HARD_BUDGET);
	erofs_stream_global_budget = MIN(erofs_stream_global_budget,
	    EROFS_STREAM_GLOBAL_HARD_BUDGET);
	erofs_stream_mount_contexts = MAX(0, MIN(erofs_stream_mount_contexts,
	    (int)EROFS_STREAM_MOUNT_HARD_CONTEXTS));
	erofs_stream_global_contexts_limit = MAX(0, MIN(
	    erofs_stream_global_contexts_limit,
	    (int)EROFS_STREAM_GLOBAL_HARD_CONTEXTS));
	erofs_stream_mount_cached = MAX(0, MIN(erofs_stream_mount_cached,
	    (int)EROFS_STREAM_MOUNT_CACHED_PER_CODEC));
	erofs_stream_global_cached = MAX(0, MIN(erofs_stream_global_cached,
	    (int)EROFS_STREAM_GLOBAL_CACHED_PER_CODEC));
	cv_init(&erofs_stream_global_cv, "erofs stream wait");
	/*
	 * The context count is bounded by erofs_stream_global_contexts_limit;
	 * a UMA item limit would only add a second, unrelated place to sleep.
	 */
	erofs_stream_zone = uma_zcreate("erofs stream ctx",
	    EROFS_STREAM_CTX_WRAPPER_SIZE, NULL, NULL, NULL, NULL,
	    UMA_ALIGN_PTR, 0);
	KASSERT(erofs_stream_zone != NULL,
	    ("cannot create erofs stream UMA zone"));
	uma_zone_set_maxcache(erofs_stream_zone,
	    EROFS_STREAM_GLOBAL_CACHED_PER_CODEC * 3);
	erofs_stream_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
	    erofs_stream_pool_lowmem, NULL, LOWMEM_PRI_DEFAULT);
}

static void
erofs_stream_pool_global_fini(void *arg __unused)
{
	if (erofs_stream_lowmem_tag != NULL)
		EVENTHANDLER_DEREGISTER(vm_lowmem, erofs_stream_lowmem_tag);
	KASSERT(LIST_EMPTY(&erofs_stream_mounts),
	    ("erofs stream mounts remain at unload"));
	KASSERT(erofs_stream_global_contexts == 0 &&
	    erofs_stream_global_waiters == 0 &&
	    erofs_stream_global_resident == 0 &&
	    erofs_stream_borrowed_contexts == 0 &&
	    erofs_stream_idle_contexts == 0 &&
	    erofs_stream_cached_contexts == 0,
	    ("erofs stream resources remain at unload"));
	uma_zdestroy(erofs_stream_zone);
	cv_destroy(&erofs_stream_global_cv);
}

SYSINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST,
    erofs_stream_pool_global_init, NULL);
SYSUNINIT(erofs_stream_pool_global, SI_SUB_VFS, SI_ORDER_FIRST,
    erofs_stream_pool_global_fini, NULL);