update

2026-08-18 09:38:06 +02:00
commit 770d83819b
21 changed files with 9067 additions and 0 deletions
@@ -0,0 +1,625 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             https://www.huawei.com/
+ */
+
+#include <sys/param.h>
+#include <sys/_maxphys.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include "compress.h"
+
+enum z_erofs_cache_lookup {
+	Z_EROFS_CACHE_BYPASS,
+	Z_EROFS_CACHE_HIT,
+	Z_EROFS_CACHE_OWNER,
+	Z_EROFS_CACHE_ERROR,
+};
+
+#define EROFS_ZCACHE_MOUNT_HARD_BUDGET (256UL * 1024)
+#define EROFS_ZCACHE_GLOBAL_HARD_BUDGET (512UL * 1024)
+#define EROFS_ZCACHE_MIN_DECODE_WORK (128UL * 1024)
+
+static int z_erofs_cache_enabled = 1;
+static unsigned long z_erofs_cache_mount_budget =
+    EROFS_ZCACHE_MOUNT_HARD_BUDGET;
+static unsigned long z_erofs_cache_global_budget =
+    EROFS_ZCACHE_GLOBAL_HARD_BUDGET;
+static unsigned long z_erofs_cache_minimum_decode_work =
+    EROFS_ZCACHE_MIN_DECODE_WORK;
+
+TUNABLE_INT("vfs.erofs.decoded_cache.enabled", &z_erofs_cache_enabled);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.mount_budget",
+    &z_erofs_cache_mount_budget);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.global_budget",
+    &z_erofs_cache_global_budget);
+TUNABLE_ULONG("vfs.erofs.decoded_cache.minimum_decode_work",
+    &z_erofs_cache_minimum_decode_work);
+
+static struct mtx z_erofs_cache_list_lock;
+static struct mtx z_erofs_cache_budget_lock;
+static LIST_HEAD(, erofs_sb_info) z_erofs_cache_mounts =
+    LIST_HEAD_INITIALIZER(z_erofs_cache_mounts);
+static size_t z_erofs_cache_global_charged;
+static eventhandler_tag z_erofs_cache_lowmem_tag;
+
+MTX_SYSINIT(erofs_zcache_list, &z_erofs_cache_list_lock,
+    "erofs zcache list", MTX_DEF);
+MTX_SYSINIT(erofs_zcache_budget, &z_erofs_cache_budget_lock,
+    "erofs zcache budget", MTX_DEF);
+
+static size_t
+z_erofs_extent_cache_global_limit(void)
+{
+
+	return (MIN(z_erofs_cache_global_budget,
+	    EROFS_ZCACHE_GLOBAL_HARD_BUDGET));
+}
+
+static bool
+z_erofs_extent_cache_reserve(size_t bytes)
+{
+	size_t limit;
+	bool reserved;
+
+	reserved = false;
+	mtx_lock(&z_erofs_cache_budget_lock);
+	limit = z_erofs_extent_cache_global_limit();
+	if (bytes <= limit && z_erofs_cache_global_charged <= limit - bytes) {
+		z_erofs_cache_global_charged += bytes;
+		reserved = true;
+	}
+	mtx_unlock(&z_erofs_cache_budget_lock);
+	return (reserved);
+}
+
+static void
+z_erofs_extent_cache_release(size_t bytes)
+{
+
+	mtx_lock(&z_erofs_cache_budget_lock);
+	KASSERT(bytes <= z_erofs_cache_global_charged,
+	    ("erofs decoded cache budget underflow"));
+	z_erofs_cache_global_charged -= bytes;
+	mtx_unlock(&z_erofs_cache_budget_lock);
+}
+
+static void *
+z_erofs_extent_cache_drop_locked(struct erofs_zextent_cache *cache,
+    bool evicted, bool reclaimed)
+{
+	struct erofs_zextent_cache_metrics *metrics;
+	void *data;
+
+	data = cache->data;
+	if (cache->charged_bytes != 0) {
+		KASSERT(cache->map.m_algorithmformat < Z_EROFS_COMPRESSION_MAX,
+		    ("erofs decoded cache charged unknown codec"));
+		metrics = &cache->metrics[cache->map.m_algorithmformat];
+		if (cache->state == EROFS_ZCACHE_READY) {
+			KASSERT(metrics->resident_bytes == cache->charged_bytes,
+			    ("erofs decoded cache codec accounting mismatch"));
+			metrics->resident_bytes = 0;
+			if (evicted)
+				++metrics->evictions;
+			if (reclaimed)
+				++metrics->reclaims;
+		}
+		z_erofs_extent_cache_release(cache->charged_bytes);
+		cache->charged_bytes = 0;
+	}
+	cache->data = NULL;
+	return (data);
+}
+
+static bool
+z_erofs_extent_cache_match(const struct erofs_zextent_cache *cache,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size)
+{
+
+	return (cache->state != EROFS_ZCACHE_EMPTY && cache->nid == vi->nid &&
+	    cache->decoded_size == decoded_size &&
+	    cache->map.m_pa == map->m_pa && cache->map.m_la == map->m_la &&
+	    cache->map.m_plen == map->m_plen &&
+	    cache->map.m_llen == map->m_llen &&
+	    cache->map.m_deviceid == map->m_deviceid &&
+	    cache->map.m_algorithmformat == map->m_algorithmformat &&
+	    cache->map.m_flags == map->m_flags);
+}
+
+static enum z_erofs_cache_lookup
+z_erofs_extent_cache_claim(struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size, erofs_off_t mapoff, size_t len, void *dst, int *errorp)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_zextent_cache_metrics *metrics;
+	void *old;
+	enum z_erofs_cache_lookup result;
+
+	KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS"));
+	KASSERT(mapoff <= decoded_size && len <= decoded_size - mapoff,
+	    ("erofs extent cache copy exceeds decoded extent"));
+	*errorp = 0;
+	if (!sbi->z_extent_cache_initialized)
+		return (Z_EROFS_CACHE_BYPASS);
+	cache = &sbi->z_extent_cache;
+	KASSERT(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX,
+	    ("erofs decoded cache claim has unknown codec"));
+	old = NULL;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	metrics = &cache->metrics[map->m_algorithmformat];
+	if (cache->closing) {
+		++metrics->bypasses;
+		goto bypass;
+	}
+	if (z_erofs_extent_cache_match(cache, vi, map, decoded_size)) {
+		switch (cache->state) {
+		case EROFS_ZCACHE_READY:
+			KASSERT(cache->data != NULL,
+			    ("erofs ready extent cache has no data"));
+			memcpy(dst, (char *)cache->data + (size_t)mapoff, len);
+			++metrics->hits;
+			mtx_unlock(&sbi->z_extent_cache_lock);
+			return (Z_EROFS_CACHE_HIT);
+		case EROFS_ZCACHE_INFLIGHT:
+			++cache->waiters;
+			do {
+				cv_wait(&cache->cv, &sbi->z_extent_cache_lock);
+			} while (cache->state == EROFS_ZCACHE_INFLIGHT);
+			KASSERT(z_erofs_extent_cache_match(cache, vi, map,
+			    decoded_size), ("erofs inflight cache key changed"));
+			if (cache->state == EROFS_ZCACHE_READY) {
+				KASSERT(cache->data != NULL,
+				    ("erofs ready extent cache has no data"));
+				memcpy(dst, (char *)cache->data + (size_t)mapoff,
+				    len);
+				++metrics->hits;
+				result = Z_EROFS_CACHE_HIT;
+			} else {
+				KASSERT(cache->state == EROFS_ZCACHE_FAILED &&
+				    cache->error > 0,
+				    ("erofs inflight cache has no typed result"));
+				*errorp = cache->error;
+				result = Z_EROFS_CACHE_ERROR;
+			}
+			--cache->waiters;
+			if (cache->state == EROFS_ZCACHE_FAILED &&
+			    cache->waiters == 0) {
+				cache->error = 0;
+				cache->state = EROFS_ZCACHE_EMPTY;
+			}
+			if (cache->waiters == 0)
+				cv_broadcast(&cache->cv);
+			mtx_unlock(&sbi->z_extent_cache_lock);
+			return (result);
+		case EROFS_ZCACHE_FAILED:
+			if (cache->waiters != 0)
+				goto bypass;
+			cache->error = 0;
+			cache->state = EROFS_ZCACHE_EMPTY;
+			break;
+		case EROFS_ZCACHE_EMPTY:
+			break;
+		}
+	}
+	if (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0) {
+		++metrics->bypasses;
+		goto bypass;
+	}
+	if (cache->state == EROFS_ZCACHE_READY) {
+		old = z_erofs_extent_cache_drop_locked(cache, true, false);
+		cache->state = EROFS_ZCACHE_EMPTY;
+	}
+	if (!z_erofs_extent_cache_reserve(decoded_size)) {
+		++metrics->bypasses;
+		goto bypass;
+	}
+	cache->map = *map;
+	cache->nid = vi->nid;
+	cache->decoded_size = decoded_size;
+	cache->charged_bytes = decoded_size;
+	cache->error = 0;
+	cache->state = EROFS_ZCACHE_INFLIGHT;
+	++metrics->misses;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	free(old, M_EROFS);
+	return (Z_EROFS_CACHE_OWNER);
+
+bypass:
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	free(old, M_EROFS);
+	return (Z_EROFS_CACHE_BYPASS);
+}
+
+static void
+z_erofs_extent_cache_complete(struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t decoded_size, void *decoded, int error)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_zextent_cache_metrics *metrics;
+
+	KASSERT((error == 0) == (decoded != NULL),
+	    ("erofs extent cache completion is untyped"));
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	KASSERT(cache->state == EROFS_ZCACHE_INFLIGHT &&
+	    z_erofs_extent_cache_match(cache, vi, map, decoded_size),
+	    ("erofs extent cache owner lost its key"));
+	metrics = &cache->metrics[map->m_algorithmformat];
+	if (error == 0) {
+		KASSERT(cache->charged_bytes == decoded_size &&
+		    metrics->resident_bytes == 0,
+		    ("erofs decoded cache success accounting mismatch"));
+		cache->data = decoded;
+		metrics->resident_bytes = decoded_size;
+		cache->error = 0;
+		cache->state = EROFS_ZCACHE_READY;
+	} else {
+		KASSERT(error > 0, ("erofs extent cache published negative errno"));
+		z_erofs_extent_cache_release(cache->charged_bytes);
+		cache->charged_bytes = 0;
+		cache->error = error;
+		cache->state = EROFS_ZCACHE_FAILED;
+		if (cache->waiters == 0) {
+			cache->error = 0;
+			cache->state = EROFS_ZCACHE_EMPTY;
+		}
+	}
+	cv_broadcast(&cache->cv);
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+static void
+z_erofs_extent_cache_lowmem(void *arg, int howto)
+{
+	struct erofs_zextent_cache *cache;
+	struct erofs_sb_info *sbi;
+	void *data;
+
+	(void)arg;
+	(void)howto;
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_FOREACH(sbi, &z_erofs_cache_mounts, z_extent_cache_link) {
+		data = NULL;
+		if (!mtx_trylock(&sbi->z_extent_cache_lock))
+			continue;
+		cache = &sbi->z_extent_cache;
+		if (cache->state == EROFS_ZCACHE_READY && cache->waiters == 0) {
+			data = z_erofs_extent_cache_drop_locked(cache, true, true);
+			cache->state = EROFS_ZCACHE_EMPTY;
+		}
+		mtx_unlock(&sbi->z_extent_cache_lock);
+		free(data, M_EROFS);
+	}
+	mtx_unlock(&z_erofs_cache_list_lock);
+}
+
+static void
+z_erofs_extent_cache_global_init(void *arg)
+{
+
+	(void)arg;
+	z_erofs_cache_lowmem_tag = EVENTHANDLER_REGISTER(vm_lowmem,
+	    z_erofs_extent_cache_lowmem, NULL, LOWMEM_PRI_DEFAULT);
+}
+
+static void
+z_erofs_extent_cache_global_fini(void *arg)
+{
+
+	(void)arg;
+	if (z_erofs_cache_lowmem_tag != NULL)
+		EVENTHANDLER_DEREGISTER(vm_lowmem, z_erofs_cache_lowmem_tag);
+	KASSERT(LIST_EMPTY(&z_erofs_cache_mounts),
+	    ("erofs decoded cache mounts remain at unload"));
+	KASSERT(z_erofs_cache_global_charged == 0,
+	    ("erofs decoded cache bytes remain at unload"));
+}
+
+SYSINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY,
+    z_erofs_extent_cache_global_init, NULL);
+SYSUNINIT(erofs_zcache_global, SI_SUB_VFS, SI_ORDER_ANY,
+    z_erofs_extent_cache_global_fini, NULL);
+
+void
+z_erofs_extent_cache_init(struct erofs_sb_info *sbi)
+{
+
+	bzero(&sbi->z_extent_cache, sizeof(sbi->z_extent_cache));
+	sbi->z_extent_cache.budget_bytes = MIN(z_erofs_cache_mount_budget,
+	    EROFS_ZCACHE_MOUNT_HARD_BUDGET);
+	sbi->z_extent_cache.minimum_decode_work =
+	    z_erofs_cache_minimum_decode_work;
+	mtx_init(&sbi->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF);
+	cv_init(&sbi->z_extent_cache.cv, "erofs zextent");
+	sbi->z_extent_cache_initialized = true;
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_INSERT_HEAD(&z_erofs_cache_mounts, sbi, z_extent_cache_link);
+	mtx_unlock(&z_erofs_cache_list_lock);
+}
+
+void
+z_erofs_extent_cache_fini(struct erofs_sb_info *sbi)
+{
+	struct erofs_zextent_cache *cache;
+	void *data;
+
+	if (!sbi->z_extent_cache_initialized)
+		return;
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&z_erofs_cache_list_lock);
+	LIST_REMOVE(sbi, z_extent_cache_link);
+	mtx_unlock(&z_erofs_cache_list_lock);
+	mtx_lock(&sbi->z_extent_cache_lock);
+	cache->closing = true;
+	while (cache->state == EROFS_ZCACHE_INFLIGHT || cache->waiters != 0)
+		cv_wait(&cache->cv, &sbi->z_extent_cache_lock);
+	data = z_erofs_extent_cache_drop_locked(cache, false, false);
+	cache->error = 0;
+	cache->state = EROFS_ZCACHE_EMPTY;
+	sbi->z_extent_cache_initialized = false;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+	free(data, M_EROFS);
+	cv_destroy(&cache->cv);
+	mtx_destroy(&sbi->z_extent_cache_lock);
+}
+
+static bool
+z_erofs_extent_cache_eligible(const struct erofs_sb_info *sbi,
+    const struct erofs_inode *vi, const struct erofs_map_blocks *map,
+    size_t len)
+{
+	const struct erofs_zextent_cache *cache;
+	uint64_t decode_work;
+
+	cache = &sbi->z_extent_cache;
+	if (map->m_plen > UINT64_MAX - len)
+		decode_work = UINT64_MAX;
+	else
+		decode_work = map->m_plen + len;
+	return (z_erofs_cache_enabled != 0 && len <= MAXPHYS &&
+	    len <= cache->budget_bytes &&
+	    decode_work >= cache->minimum_decode_work &&
+	    (map->m_flags & (EROFS_MAP_META |
+	    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
+	    EROFS_MAP_FRAGMENT)) == 0 &&
+	    map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX &&
+	    sbi->z_extent_cache_initialized && vi != sbi->packed_inode &&
+	    vi != sbi->metabox_en);
+}
+
+static void
+z_erofs_extent_cache_record_bypass(struct erofs_sb_info *sbi,
+    const struct erofs_map_blocks *map)
+{
+	struct erofs_zextent_cache *cache;
+
+	if (!sbi->z_extent_cache_initialized ||
+	    map->m_algorithmformat >= Z_EROFS_COMPRESSION_MAX)
+		return;
+	cache = &sbi->z_extent_cache;
+	mtx_lock(&sbi->z_extent_cache_lock);
+	++cache->metrics[map->m_algorithmformat].bypasses;
+	mtx_unlock(&sbi->z_extent_cache_lock);
+}
+
+static int
+z_erofs_decode_length(const struct erofs_map_blocks *map,
+    erofs_off_t mapoff, size_t want, size_t *decoded_len, bool *partial)
+{
+	size_t end;
+
+	if (mapoff > map->m_llen || want > map->m_llen - mapoff)
+		return (EINTEGRITY);
+#if SIZE_MAX < UINT64_MAX
+	if (map->m_llen > SIZE_MAX || mapoff > SIZE_MAX)
+		return (EOVERFLOW);
+#endif
+	if ((size_t)mapoff > SIZE_MAX - want)
+		return (EOVERFLOW);
+	end = (size_t)mapoff + want;
+	*partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;
+	if (!*partial && (!z_erofs_decompress_supports_subextent(map) ||
+	    end == map->m_llen)) {
+		*decoded_len = (size_t)map->m_llen;
+		return (0);
+	}
+	*decoded_len = end;
+	*partial = true;
+	return (0);
+}
+
+static int
+z_erofs_decode_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct erofs_map_blocks *map, size_t decoded_len, bool partial, void **bufp)
+{
+	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
+	const void *input;
+	void *compressed, *decoded;
+	int error;
+
+	*bufp = NULL;
+	if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0)
+		return (EINTEGRITY);
+	if ((map->m_flags & EROFS_MAP_MAPPED) == 0)
+		return (EINTEGRITY);
+#if SIZE_MAX < UINT64_MAX
+	if (map->m_plen > SIZE_MAX || map->m_llen > SIZE_MAX)
+		return (EOVERFLOW);
+#endif
+	if (decoded_len == 0 || decoded_len > map->m_llen)
+		return (EINTEGRITY);
+	if (!partial && decoded_len != map->m_llen)
+		return (EINTEGRITY);
+
+	if ((map->m_flags & EROFS_MAP_META) != 0) {
+		error = erofs_read_metadata(sbi, vi->nid, map->m_pa,
+		    (size_t)map->m_plen, &buf);
+	} else {
+		error = erofs_read_physical(sbi, map->m_deviceid, map->m_pa,
+		    (size_t)map->m_plen, &compressed);
+	}
+	if (error != 0)
+		return (error);
+	input = (map->m_flags & EROFS_MAP_META) != 0 ? buf.data : compressed;
+
+	decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO);
+	error = z_erofs_decompress(sbi, map, input, (size_t)map->m_plen,
+	    decoded, decoded_len, partial);
+	if ((map->m_flags & EROFS_MAP_META) != 0)
+		erofs_put_metabuf(&buf);
+	else
+		erofs_brelse(compressed);
+	if (error != 0) {
+		free(decoded, M_EROFS);
+		return (error);
+	}
+	*bufp = decoded;
+	return (0);
+}
+
+static int
+z_erofs_do_read(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, char *out)
+{
+	struct erofs_map_blocks map;
+	void *decoded, *fragment;
+	erofs_off_t mapoff;
+	size_t decoded_len, done, want;
+	bool cache_eligible, partial;
+	enum z_erofs_cache_lookup cache_lookup;
+	int error;
+
+	done = 0;
+	while (done < len) {
+		map = (struct erofs_map_blocks) { .m_la = loff + done };
+		error = erofs_map_blocks(sbi, vi, &map);
+		if (error != 0)
+			return (error);
+		if (map.m_llen == 0 || map.m_la > loff + done ||
+		    loff + done - map.m_la >= map.m_llen)
+			return (EINTEGRITY);
+		mapoff = loff + done - map.m_la;
+#if SIZE_MAX < UINT64_MAX
+		if (mapoff > SIZE_MAX)
+			return (EOVERFLOW);
+		if (map.m_llen - mapoff > SIZE_MAX)
+			return (EOVERFLOW);
+#endif
+		want = MIN((size_t)(map.m_llen - mapoff), len - done);
+		if (want == 0)
+			return (EINTEGRITY);
+
+		if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) {
+			if (sbi->packed_inode == NULL ||
+			    sbi->packed_inode->nid == vi->nid ||
+			    vi->z_fragmentoff > UINT64_MAX - mapoff)
+				return (EINTEGRITY);
+			error = erofs_read_data(sbi, sbi->packed_inode,
+			    vi->z_fragmentoff + mapoff, want, &fragment);
+			if (error != 0)
+				return (error);
+			memcpy(out + done, fragment, want);
+			erofs_brelse(fragment);
+		} else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
+			bzero(out + done, want);
+		} else {
+			error = z_erofs_decode_length(&map, mapoff, want,
+			    &decoded_len, &partial);
+			if (error != 0)
+				return (error);
+			cache_eligible = !partial &&
+			    z_erofs_extent_cache_eligible(sbi, vi, &map, decoded_len);
+			cache_lookup = Z_EROFS_CACHE_BYPASS;
+			if (cache_eligible) {
+				cache_lookup = z_erofs_extent_cache_claim(sbi, vi, &map,
+				    decoded_len, mapoff, want, out + done, &error);
+				if (cache_lookup == Z_EROFS_CACHE_HIT) {
+					done += want;
+					continue;
+				}
+				if (cache_lookup == Z_EROFS_CACHE_ERROR)
+					return (error);
+			} else
+				z_erofs_extent_cache_record_bypass(sbi, &map);
+			error = z_erofs_decode_extent(sbi, vi, &map, decoded_len,
+			    partial, &decoded);
+			if (error != 0) {
+				if (cache_lookup == Z_EROFS_CACHE_OWNER)
+					z_erofs_extent_cache_complete(sbi, vi, &map,
+					    decoded_len, NULL, error);
+				return (error);
+			}
+			memcpy(out + done, (char *)decoded + (size_t)mapoff, want);
+			if (cache_lookup == Z_EROFS_CACHE_OWNER)
+				z_erofs_extent_cache_complete(sbi, vi, &map, decoded_len,
+				    decoded, 0);
+			else
+				free(decoded, M_EROFS);
+		}
+		done += want;
+	}
+	return (0);
+}
+
+int
+z_erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    erofs_off_t loff, size_t len, void **bufp)
+{
+	char *out;
+	int error;
+
+	if (bufp == NULL)
+		return (EINVAL);
+	*bufp = NULL;
+	if (len == 0)
+		return (0);
+	if (loff > UINT64_MAX - len)
+		return (EOVERFLOW);
+	if (loff > vi->size || len > vi->size - loff)
+		return (EINTEGRITY);
+
+	out = malloc(len, M_EROFS, M_WAITOK);
+	error = z_erofs_do_read(sbi, vi, loff, len, out);
+	if (error != 0) {
+		free(out, M_EROFS);
+		return (error);
+	}
+	*bufp = out;
+	return (0);
+}
+
+int
+z_erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi,
+    struct uio *uio)
+{
+	char *buf;
+	size_t want;
+	int error;
+
+	if (uio->uio_offset < 0)
+		return (EINVAL);
+	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) {
+		want = MIN((size_t)uio->uio_resid,
+		    (size_t)MIN((uint64_t)MAXPHYS,
+		    vi->size - (uint64_t)uio->uio_offset));
+		buf = malloc(want, M_EROFS, M_WAITOK);
+		error = z_erofs_do_read(sbi, vi, (uint64_t)uio->uio_offset,
+		    want, buf);
+		if (error == 0)
+			error = uiomove(buf, want, uio);
+		free(buf, M_EROFS);
+		if (error != 0)
+			return (error);
+	}
+	return (0);
+}