erofs-freebsd-out-tree/zdata.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2018-2019 HUAWEI, Inc.
 *             https://www.huawei.com/
 */

#include <sys/param.h>
#include <sys/_maxphys.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/uio.h>

#include "compress.h"

enum z_erofs_cache_lookup {
	Z_EROFS_CACHE_BYPASS,
	Z_EROFS_CACHE_HIT,
	Z_EROFS_CACHE_OWNER,
	Z_EROFS_CACHE_ERROR,
};

#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024)
#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024)
#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024)

static int z_erofs_cache_enabled = 1;
static unsigned long z_erofs_cache_mount_budget =
    EROFS_ZCACHE_MOUNT_HARD_BUDGET;
static unsigned long z_erofs_cache_global_budget =
    EROFS_ZCACHE_GLOBAL_HARD_BUDGET;
static unsigned long z_erofs_cache_minimum_decode_work =
    EROFS_ZCACHE_MIN_DECODE_WORK;

TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled);
TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget",
    &z_erofs_cache_mount_budget);
TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget",
    &z_erofs_cache_global_budget);
TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work",
    &z_erofs_cache_minimum_decode_work);

static struct mtx z_erofs_cache_list_lock;
static struct mtx z_erofs_cache_budget_lock;
static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts =
    LIST_HEAD_INITIALIZER(z_erofs_cache_mounts);
static size_t z_erofs_cache_global_charged;
static eventhandler_tag z_erofs_cache_lowmem_tag;

MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock,
    "erofs zcache list", MTX_DEF);
MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock,
    "erofs zcache budget", MTX_DEF);

static size_t
z_erofs_extent_cache_global_limit(void)
{

	return (MIN(z_erofs_cache_global_budget,
	    EROFS_ZCACHE_GLOBAL_HARD_BUDGET));
}

static bool
z_erofs_extent_cache_reserve(size_t bytes)
{
	size_t limit;
	bool reserved;

	reserved = false;
	mtx_lock(&z_erofs_cache_budget_lock);
	limit = z_erofs_extent_cache_global_limit();
	if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) {
		z_erofs_cache_global_charged += bytes;
		reserved = true;
	}
	mtx_unlock(&z_erofs_cache_budget_lock);
	return (reserved);
}

static void
z_erofs_extent_cache_release(size_t bytes)
{

	mtx_lock(&z_erofs_cache_budget_lock);
	KASSERT(bytes <= z_erofs_cache_global_charged,
	    ("erofs decoded cache budget underflow"));
	z_erofs_cache_global_charged -= bytes;
	mtx_unlock(&z_erofs_cache_budget_lock);
}

static void *
z_erofs_extent_cache_drop_locked(struct erofs_zextent_cache *cache,
    bool evicted, bool reclaimed)
{
	struct erofs_zextent_cache_metrics *metrics;
	void *data;

	data = cache->data;
	if (cache->charged_bytes != 0) {
		KASSERT(cache->map.m_algorithmformat < Z_EROFS_COMPRESSION_MAX,
		    ("erofs decoded cache charged unknown codec"));
		metrics = &cache->metrics[cache->map.m_algorithmformat];
		if (cache->state == EROFS_ZCACHE_READY) {
			KASSERT(metrics->resident_bytes == cache->charged_bytes,
			    ("erofs decoded cache codec accounting mismatch"));
			metrics->resident_bytes = 0;
			if (evicted)
				++metrics->evictions;
			if (reclaimed)
				++metrics->reclaims;
		}
		z_erofs_extent_cache_release(cache->charged_bytes);
		cache->charged_bytes = 0;
	}
	cache->data = NULL;
	return (data);
}

static bool
z_erofs_extent_cache_match(const struct erofs_zextent_cache *cache,
    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
    size_t decoded_size)
{

	return (cache->state != EROFS_ZCACHE_EMPTY && cache->nid == vi->nid &&
	    cache->decoded_size == decoded_size &&
	    cache->map.m_pa == map->m_pa && cache->map.m_la == map->m_la &&
	    cache->map.m_plen == map->m_plen &&
	    cache->map.m_llen == map->m_llen &&
	    cache->map.m_deviceid == map->m_deviceid &&
	    cache->map.m_algorithmformat == map->m_algorithmformat &&
	    cache->map.m_flags == map->m_flags);
}

static enum z_erofs_cache_lookup
z_erofs_extent_cache_claim(struct erofs_sb_info *sbi,
    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
    size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp)
{
	struct erofs_zextent_cache *cache;
	struct erofs_zextent_cache_metrics *metrics;
	void *old;
	enum z_erofs_cache_lookup result;

	KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS"));
	KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff,
	    ("erofs extent cache copy exceeds decoded extent"));
	*errorp = 0;
	if (!sbi->z_extent_cache_initialized)
		return (Z_EROFS_CACHE_BYPASS);
	cache = &sbi->z_extent_cache;
	KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX,
	    ("erofs decoded cache claim has unknown codec"));
	old = NULL;
	mtx_lock(&sbi->z_extent_cache_lock);
	metrics = &cache->metrics[map->m_algorithmformat];
	if (cache->closing) {
		++metrics->bypasses;
		goto bypass;
	}
	if (z_erofs_extent_cache_match(cache, vi, map, decoded_size)) {
		switch (cache->state) {
		case EROFS_ZCACHE_READY:
			KASSERT(cache->data != NULL,
			    ("erofs ready extent cache has no data"));
			memcpy(dst, (char *)cache->data + (size_t)mapoff, len);
			++metrics->hits;
			mtx_unlock(&sbi->z_extent_cache_lock);
			return (Z_EROFS_CACHE_HIT);
		case EROFS_ZCACHE_INFLIGHT:
			++cache->waiters;
			do {
				cv_wait(&cache->cv, &sbi->z_extent_cache_lock);
			} while (cache->state == EROFS_ZCACHE_INFLIGHT);
			KASSERT(z_erofs_extent_cache_match(cache, vi, map,
			    decoded_size), ("erofs inflight cache key changed"));
			if (cache->state == EROFS_ZCACHE_READY) {
				KASSERT(cache->data != NULL,
				    ("erofs ready extent cache has no data"));
				memcpy(dst, (char *)cache->data + (size_t)mapoff,
				    len);
				++metrics->hits;
				result = Z_EROFS_CACHE_HIT;
			} else {
				KASSERT(cache->state == EROFS_ZCACHE_FAILED &&
				    cache->error > 0,
				    ("erofs inflight cache has no typed result"));
				*errorp = cache->error;
				result = Z_EROFS_CACHE_ERROR;
			}
			--cache->waiters;
			if (cache->state == EROFS_ZCACHE_FAILED &&
			    cache->waiters == 0) {
				cache->error = 0;
				cache->state = EROFS_ZCACHE_EMPTY;
			}
			if (cache->waiters == 0)
				cv_broadcast(&cache->cv);
			mtx_unlock(&sbi->z_extent_cache_lock);
			return (result);
		case EROFS_ZCACHE_FAILED:
			if (cache->waiters != 0)
				goto bypass;
			cache->error = 0;
			cache->state = EROFS_ZCACHE_EMPTY;
			break;
		case EROFS_ZCACHE_EMPTY:
			break;
		}
	}
	if (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) {
		++metrics->bypasses;
		goto bypass;
	}
	if (cache->state == EROFS_ZCACHE_READY) {
		old = z_erofs_extent_cache_drop_locked(cache, true, false);
		cache->state = EROFS_ZCACHE_EMPTY;
	}
	if (!z_erofs_extent_cache_reserve(decoded_size)) {
		++metrics->bypasses;
		goto bypass;
	}
	cache->map = *map;
	cache->nid = vi->nid;
	cache->decoded_size = decoded_size;
	cache->charged_bytes = decoded_size;
	cache->error = 0;
	cache->state = EROFS_ZCACHE_INFLIGHT;
	++metrics->misses;
	mtx_unlock(&sbi->z_extent_cache_lock);
	free(old, M_EROFS);
	return (Z_EROFS_CACHE_OWNER);

bypass:
	mtx_unlock(&sbi->z_extent_cache_lock);
	free(old, M_EROFS);
	return (Z_EROFS_CACHE_BYPASS);
}

static void
z_erofs_extent_cache_complete(struct erofs_sb_info *sbi,
    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
    size_t decoded_size, void *decoded, int error)
{
	struct erofs_zextent_cache *cache;
	struct erofs_zextent_cache_metrics *metrics;

	KASSERT((error == 0) == (decoded != NULL),
	    ("erofs extent cache completion is untyped"));
	cache = &sbi->z_extent_cache;
	mtx_lock(&sbi->z_extent_cache_lock);
	KASSERT(cache->state == EROFS_ZCACHE_INFLIGHT &&
	    z_erofs_extent_cache_match(cache, vi, map, decoded_size),
	    ("erofs extent cache owner lost its key"));
	metrics = &cache->metrics[map->m_algorithmformat];
	if (error == 0) {
		KASSERT(cache->charged_bytes == decoded_size &&
		    metrics->resident_bytes == 0,
		    ("erofs decoded cache success accounting mismatch"));
		cache->data = decoded;
		metrics->resident_bytes = decoded_size;
		cache->error = 0;
		cache->state = EROFS_ZCACHE_READY;
	} else {
		KASSERT(error > 0, ("erofs extent cache published negative errno"));
		z_erofs_extent_cache_release(cache->charged_bytes);
		cache->charged_bytes = 0;
		cache->error = error;
		cache->state = EROFS_ZCACHE_FAILED;
		if (cache->waiters == 0) {
			cache->error = 0;
			cache->state = EROFS_ZCACHE_EMPTY;
		}
	}
	cv_broadcast(&cache->cv);
	mtx_unlock(&sbi->z_extent_cache_lock);
}

static void
z_erofs_extent_cache_lowmem(void *arg, int howto)
{
	struct erofs_zextent_cache *cache;
	struct erofs_sb_info *sbi;
	void *data;

	(void)arg;
	(void)howto;
	mtx_lock(&z_erofs_cache_list_lock);
	LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) {
		data = NULL;
		if (!mtx_trylock(&sbi->z_extent_cache_lock))
			continue;
		cache = &sbi->z_extent_cache;
		if (cache->state == EROFS_ZCACHE_READY && cache->waiters == 0) {
			data = z_erofs_extent_cache_drop_locked(cache, true, true);
			cache->state = EROFS_ZCACHE_EMPTY;
		}
		mtx_unlock(&sbi->z_extent_cache_lock);
		free(data, M_EROFS);
	}
	mtx_unlock(&z_erofs_cache_list_lock);
}

static void
z_erofs_extent_cache_global_init(void *arg)
{

	(void)arg;
	z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
	    z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT);
}

static void
z_erofs_extent_cache_global_fini(void *arg)
{

	(void)arg;
	if (z_erofs_cache_lowmem_tag != NULL)
		EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag);
	KASSERT(LIST_EMPTY(&z_erofs_cache_mounts),
	    ("erofs decoded cache mounts remain at unload"));
	KASSERT(z_erofs_cache_global_charged == 0,
	    ("erofs decoded cache bytes remain at unload"));
}

SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY,
    z_erofs_extent_cache_global_init, NULL);
SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY,
    z_erofs_extent_cache_global_fini, NULL);

void
z_erofs_extent_cache_init(struct erofs_sb_info *sbi)
{

	bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache));
	sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget,
	    EROFS_ZCACHE_MOUNT_HARD_BUDGET);
	sbi->z_extent_cache.minimum_decode_work =
	    z_erofs_cache_minimum_decode_work;
	mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF);
	cv_init(&sbi->z_extent_cache.cv, "erofs zextent");
	sbi->z_extent_cache_initialized = true;
	mtx_lock(&z_erofs_cache_list_lock);
	LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link);
	mtx_unlock(&z_erofs_cache_list_lock);
}

void
z_erofs_extent_cache_fini(struct erofs_sb_info *sbi)
{
	struct erofs_zextent_cache *cache;
	void *data;

	if (!sbi->z_extent_cache_initialized)
		return;
	cache = &sbi->z_extent_cache;
	mtx_lock(&z_erofs_cache_list_lock);
	LIST_REMOVE(sbi, z_extent_cache_link);
	mtx_unlock(&z_erofs_cache_list_lock);
	mtx_lock(&sbi->z_extent_cache_lock);
	cache->closing = true;
	while (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0)
		cv_wait(&cache->cv, &sbi->z_extent_cache_lock);
	data = z_erofs_extent_cache_drop_locked(cache, false, false);
	cache->error = 0;
	cache->state = EROFS_ZCACHE_EMPTY;
	sbi->z_extent_cache_initialized = false;
	mtx_unlock(&sbi->z_extent_cache_lock);
	free(data, M_EROFS);
	cv_destroy(&cache->cv);
	mtx_destroy(&sbi->z_extent_cache_lock);
}

static bool
z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi,
    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
    size_t len)
{
	const struct erofs_zextent_cache *cache;
	uint64_t decode_work;

	cache = &sbi->z_extent_cache;
	if (map->m_plen > UINT64_MAX - len)
		decode_work = UINT64_MAX;
	else
		decode_work = map->m_plen + len;
	return (z_erofs_cache_enabled != 0 && len <= MAXPHYS &&
	    len <= cache->budget_bytes &&
	    decode_work >= cache->minimum_decode_work &&
	    (map->m_flags & (EROFS_MAP_META |
	    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
	    EROFS_MAP_FRAGMENT)) == 0 &&
	    map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX &&
	    sbi->z_extent_cache_initialized && vi != sbi->packed_inode &&
	    vi != sbi->metabox_en);
}

static void
z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi,
    const struct erofs_map_blocks *map)
{
	struct erofs_zextent_cache *cache;

	if (!sbi->z_extent_cache_initialized ||
	    map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX)
		return;
	cache = &sbi->z_extent_cache;
	mtx_lock(&sbi->z_extent_cache_lock);
	++cache->metrics[map->m_algorithmformat].bypasses;
	mtx_unlock(&sbi->z_extent_cache_lock);
}

static int
z_erofs_decode_length(const struct erofs_map_blocks *map,
    erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial)
{
	size_t end;

	if (mapoff > map->m_llen || want > map->m_llen - mapoff)
		return (EINTEGRITY);
#if SIZE_MAX < UINT64_MAX
	if (map->m_llen > SIZE_MAX || mapoff > SIZE_MAX)
		return (EOVERFLOW);
#endif
	if ((size_t)mapoff > SIZE_MAX - want)
		return (EOVERFLOW);
	end = (size_t)mapoff + want;
	*partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;
	if (!*partial && (!z_erofs_decompress_supports_subextent(map) ||
	    end == map->m_llen)) {
		*decoded_len = (size_t)map->m_llen;
		return (0);
	}
	*decoded_len = end;
	*partial = true;
	return (0);
}

static int
z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	const void *input;
	void *compressed, *decoded;
	int error;

	*bufp = NULL;
	if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0)
		return (EINTEGRITY);
	if ((map->m_flags & EROFS_MAP_MAPPED) == 0)
		return (EINTEGRITY);
#if SIZE_MAX < UINT64_MAX
	if (map->m_plen > SIZE_MAX || map->m_llen > SIZE_MAX)
		return (EOVERFLOW);
#endif
	if (decoded_len == 0 || decoded_len > map->m_llen)
		return (EINTEGRITY);
	if (!partial && decoded_len != map->m_llen)
		return (EINTEGRITY);

	if ((map->m_flags & EROFS_MAP_META) != 0) {
		error = erofs_read_metadata(sbi, vi->nid, map->m_pa,
		    (size_t)map->m_plen, &buf);
	} else {
		error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa,
		    (size_t)map->m_plen, &compressed);
	}
	if (error != 0)
		return (error);
	input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed;

	decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO);
	error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,
	    decoded, decoded_len, partial);
	if ((map->m_flags & EROFS_MAP_META) != 0)
		erofs_put_metabuf(&buf);
	else
		erofs_brelse(compressed);
	if (error != 0) {
		free(decoded, M_EROFS);
		return (error);
	}
	*bufp = decoded;
	return (0);
}

static int
z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t loff, size_t len, char *out)
{
	struct erofs_map_blocks map;
	void *decoded, *fragment;
	erofs_off_t mapoff;
	size_t decoded_len, done, want;
	bool cache_eligible, partial;
	enum z_erofs_cache_lookup cache_lookup;
	int error;

	done = 0;
	while (done < len) {
		map = (struct erofs_map_blocks) { .m_la = loff + done };
		error = erofs_map_blocks(sbi, vi, &map);
		if (error != 0)
			return (error);
		if (map.m_llen == 0 || map.m_la > loff + done ||
		    loff + done - map.m_la >= map.m_llen)
			return (EINTEGRITY);
		mapoff = loff + done - map.m_la;
#if SIZE_MAX < UINT64_MAX
		if (mapoff > SIZE_MAX)
			return (EOVERFLOW);
		if (map.m_llen - mapoff > SIZE_MAX)
			return (EOVERFLOW);
#endif
		want = MIN((size_t)(map.m_llen - mapoff), len - done);
		if (want == 0)
			return (EINTEGRITY);

		if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) {
			if (sbi->packed_inode == NULL ||
			    sbi->packed_inode->nid == vi->nid ||
			    vi->z_fragmentoff > UINT64_MAX - mapoff)
				return (EINTEGRITY);
			error = erofs_read_data(sbi, sbi->packed_inode,
			    vi->z_fragmentoff + mapoff, want, &fragment);
			if (error != 0)
				return (error);
			memcpy(out + done, fragment, want);
			erofs_brelse(fragment);
		} else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
			bzero(out + done, want);
		} else {
			error = z_erofs_decode_length(&map, mapoff, want,
			    &decoded_len, &partial);
			if (error != 0)
				return (error);
			cache_eligible = !partial &&
			    z_erofs_extent_cache_eligible(sbi, vi, &map, decoded_len);
			cache_lookup = Z_EROFS_CACHE_BYPASS;
			if (cache_eligible) {
				cache_lookup = z_erofs_extent_cache_claim(sbi, vi, &map,
				    decoded_len, mapoff, want, out + done, &error);
				if (cache_lookup == Z_EROFS_CACHE_HIT) {
					done += want;
					continue;
				}
				if (cache_lookup == Z_EROFS_CACHE_ERROR)
					return (error);
			} else
				z_erofs_extent_cache_record_bypass(sbi, &map);
			error = z_erofs_decode_extent(sbi, vi, &map, decoded_len,
			    partial, &decoded);
			if (error != 0) {
				if (cache_lookup == Z_EROFS_CACHE_OWNER)
					z_erofs_extent_cache_complete(sbi, vi, &map,
					    decoded_len, NULL, error);
				return (error);
			}
			memcpy(out + done, (char *)decoded + (size_t)mapoff, want);
			if (cache_lookup == Z_EROFS_CACHE_OWNER)
				z_erofs_extent_cache_complete(sbi, vi, &map, decoded_len,
				    decoded, 0);
			else
				free(decoded, M_EROFS);
		}
		done += want;
	}
	return (0);
}

int
z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t loff, size_t len, void **bufp)
{
	char *out;
	int error;

	if (bufp == NULL)
		return (EINVAL);
	*bufp = NULL;
	if (len == 0)
		return (0);
	if (loff > UINT64_MAX - len)
		return (EOVERFLOW);
	if (loff > vi->size || len > vi->size - loff)
		return (EINTEGRITY);

	out = malloc(len, M_EROFS, M_WAITOK);
	error = z_erofs_do_read(sbi, vi, loff, len, out);
	if (error != 0) {
		free(out, M_EROFS);
		return (error);
	}
	*bufp = out;
	return (0);
}

int
z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct uio *uio)
{
	char *buf;
	size_t want;
	int error;

	if (uio->uio_offset < 0)
		return (EINVAL);
	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) {
		want = MIN((size_t)uio->uio_resid,
		    (size_t)MIN((uint64_t)MAXPHYS,
		    vi->size - (uint64_t)uio->uio_offset));
		buf = malloc(want, M_EROFS, M_WAITOK);
		error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset,
		    want, buf);
		if (error == 0)
			error = uiomove(buf, want, uio);
		free(buf, M_EROFS);
		if (error != 0)
			return (error);
	}
	return (0);
}