erofs-freebsd-out-tree/zdata.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2018-2019 HUAWEI, Inc.
 *             https://www.huawei.com/
 */

#include <sys/param.h>
#include <sys/_maxphys.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/uio.h>

#include "internal.h"

static bool
z_erofs_extent_cache_match(const struct erofs_zextent_cache *cache,
    const struct erofs_node *en, const struct erofs_map_blocks *map)
{

	return (cache->data != NULL && cache->m_nid == en->nid &&
	    cache->m_pa == map->m_pa &&
	    cache->m_la == map->m_la && cache->m_plen == map->m_plen &&
	    cache->m_llen == map->m_llen &&
	    cache->m_deviceid == map->m_deviceid &&
	    cache->m_flags == map->m_flags &&
	    cache->m_algorithmformat ==
	    (unsigned char)map->m_algorithmformat);
}

static bool
z_erofs_extent_cache_copy(struct erofs_mount *em, struct erofs_node *en,
    struct erofs_map_blocks *map, uint64_t mapoff, size_t len, void *dst)
{
	bool matched;

	KASSERT(len <= MAXPHYS, ("erofs extent cache copy exceeds MAXPHYS"));
	if (!em->z_extent_cache_initialized)
		return (false);
	mtx_lock(&em->z_extent_cache_lock);
	matched = z_erofs_extent_cache_match(&em->z_extent_cache, en, map);
	if (matched)
		memcpy(dst, (char *)em->z_extent_cache.data + (size_t)mapoff, len);
	mtx_unlock(&em->z_extent_cache_lock);
	return (matched);
}

static void
z_erofs_extent_cache_publish(struct erofs_mount *em, struct erofs_node *en,
    struct erofs_map_blocks *map, uint64_t mapoff, size_t len, void *decoded,
    void *dst)
{
	void *old;

	KASSERT(len <= MAXPHYS, ("erofs extent cache publish exceeds MAXPHYS"));
	mtx_lock(&em->z_extent_cache_lock);
	if (z_erofs_extent_cache_match(&em->z_extent_cache, en, map)) {
		memcpy(dst, (char *)em->z_extent_cache.data + (size_t)mapoff, len);
		mtx_unlock(&em->z_extent_cache_lock);
		free(decoded, M_EROFS);
		return;
	}
	old = em->z_extent_cache.data;
	em->z_extent_cache.data = decoded;
	em->z_extent_cache.m_nid = en->nid;
	em->z_extent_cache.m_pa = map->m_pa;
	em->z_extent_cache.m_la = map->m_la;
	em->z_extent_cache.m_plen = map->m_plen;
	em->z_extent_cache.m_llen = map->m_llen;
	em->z_extent_cache.m_deviceid = map->m_deviceid;
	em->z_extent_cache.m_flags = map->m_flags;
	em->z_extent_cache.m_algorithmformat =
	    (unsigned char)map->m_algorithmformat;
	memcpy(dst, (char *)decoded + (size_t)mapoff, len);
	mtx_unlock(&em->z_extent_cache_lock);
	free(old, M_EROFS);
}

void
z_erofs_extent_cache_init(struct erofs_mount *em)
{

	mtx_init(&em->z_extent_cache_lock, "erofs zextent", NULL, MTX_DEF);
	em->z_extent_cache_initialized = true;
}

void
z_erofs_extent_cache_fini(struct erofs_mount *em)
{
	void *data;

	if (!em->z_extent_cache_initialized)
		return;
	mtx_lock(&em->z_extent_cache_lock);
	data = em->z_extent_cache.data;
	em->z_extent_cache.data = NULL;
	mtx_unlock(&em->z_extent_cache_lock);
	free(data, M_EROFS);
	mtx_destroy(&em->z_extent_cache_lock);
	em->z_extent_cache_initialized = false;
}

static bool
z_erofs_extent_cache_eligible(const struct erofs_mount *em,
    const struct erofs_node *en, const struct erofs_map_blocks *map,
    size_t len)
{

	return (len <= MAXPHYS && (map->m_flags & (EROFS_MAP_META |
	    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
	    EROFS_MAP_FRAGMENT)) == 0 &&
	    map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA &&
	    em->z_extent_cache_initialized && en != em->packed_inode &&
	    en != em->metabox_en);
}

static int
z_erofs_read_extent(struct erofs_mount *em, struct erofs_node *en,
    struct erofs_map_blocks *map, size_t decoded_len, void **bufp)
{
	void *compressed, *decoded;
	bool partial;
	int error;

	*bufp = NULL;
	if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0)
		return (EINTEGRITY);
	if ((map->m_flags & EROFS_MAP_MAPPED) == 0)
		return (EINTEGRITY);
#if SIZE_MAX < UINT64_MAX
	if (map->m_plen > SIZE_MAX || map->m_llen > SIZE_MAX)
		return (EOVERFLOW);
#endif
	if (decoded_len == 0 || decoded_len > map->m_llen)
		return (EINTEGRITY);
	partial = (map->m_flags & EROFS_MAP_PARTIAL_REF) != 0;
	if (!partial && decoded_len != map->m_llen)
		return (EINTEGRITY);

	if ((map->m_flags & EROFS_MAP_META) != 0)
		error = erofs_read_metadata(em, en->nid, map->m_pa,
		    (size_t)map->m_plen, &compressed);
	else
		error = erofs_read_physical(em, map->m_deviceid, map->m_pa,
		    (size_t)map->m_plen, &compressed);
	if (error != 0)
		return (error);

	decoded = malloc(decoded_len, M_EROFS, M_WAITOK | M_ZERO);
	error = z_erofs_decompress(em, map, compressed, (size_t)map->m_plen,
	    decoded, decoded_len, partial);
	erofs_brelse(compressed);
	if (error != 0) {
		free(decoded, M_EROFS);
		return (error);
	}
	*bufp = decoded;
	return (0);
}

static int
z_erofs_do_read(struct erofs_mount *em, struct erofs_node *en,
    uint64_t loff, size_t len, char *out)
{
	struct erofs_map_blocks map;
	void *decoded, *fragment;
	uint64_t mapoff;
	size_t decoded_len, done, want;
	int error;

	done = 0;
	while (done < len) {
		bzero(&map, sizeof(map));
		map.m_la = loff + done;
		error = z_erofs_map_blocks_iter(em, en, &map,
		    EROFS_GET_BLOCKS_FIEMAP);
		if (error != 0)
			return (error);
		if (map.m_llen == 0 || map.m_la > loff + done ||
		    loff + done - map.m_la >= map.m_llen)
			return (EINTEGRITY);
		mapoff = loff + done - map.m_la;
#if SIZE_MAX < UINT64_MAX
		if (mapoff > SIZE_MAX)
			return (EOVERFLOW);
		if (map.m_llen - mapoff > SIZE_MAX)
			return (EOVERFLOW);
#endif
		want = MIN((size_t)(map.m_llen - mapoff), len - done);
		if (want == 0)
			return (EINTEGRITY);

		if ((map.m_flags & EROFS_MAP_FRAGMENT) != 0) {
			if (em->packed_inode == NULL ||
			    em->packed_inode->nid == en->nid ||
			    en->z_fragmentoff > UINT64_MAX - mapoff)
				return (EINTEGRITY);
			error = erofs_read_data(em, em->packed_inode,
			    en->z_fragmentoff + mapoff, want, &fragment);
			if (error != 0)
				return (error);
			memcpy(out + done, fragment, want);
			erofs_brelse(fragment);
		} else if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
			bzero(out + done, want);
		} else {
			decoded_len = (size_t)map.m_llen;
			if ((map.m_flags & EROFS_MAP_PARTIAL_REF) != 0) {
				if (mapoff > SIZE_MAX - want)
					return (EOVERFLOW);
				decoded_len = (size_t)mapoff + want;
			}
			if (z_erofs_extent_cache_eligible(em, en, &map, want) &&
			    z_erofs_extent_cache_copy(em, en, &map, mapoff, want,
			    out + done)) {
				done += want;
				continue;
			}
			error = z_erofs_read_extent(em, en, &map, decoded_len,
			    &decoded);
			if (error != 0)
				return (error);
			if (z_erofs_extent_cache_eligible(em, en, &map, want))
				z_erofs_extent_cache_publish(em, en, &map, mapoff, want,
				    decoded, out + done);
			else {
				memcpy(out + done, (char *)decoded + mapoff, want);
				free(decoded, M_EROFS);
			}
		}
		done += want;
	}
	return (0);
}

int
z_erofs_read_data(struct erofs_mount *em, struct erofs_node *en,
    uint64_t loff, size_t len, void **bufp)
{
	char *out;
	int error;

	if (bufp == NULL)
		return (EINVAL);
	*bufp = NULL;
	if (len == 0)
		return (0);
	if (loff > UINT64_MAX - len)
		return (EOVERFLOW);
	if (loff > en->size || len > en->size - loff)
		return (EINTEGRITY);

	out = malloc(len, M_EROFS, M_WAITOK);
	error = z_erofs_do_read(em, en, loff, len, out);
	if (error != 0) {
		free(out, M_EROFS);
		return (error);
	}
	*bufp = out;
	return (0);
}

int
z_erofs_read_uio(struct erofs_mount *em, struct erofs_node *en,
    struct uio *uio)
{
	char *buf;
	size_t want;
	int error;

	if (uio->uio_offset < 0)
		return (EINVAL);
	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < en->size) {
		want = MIN((size_t)uio->uio_resid,
		    (size_t)MIN((uint64_t)MAXPHYS,
		    en->size - (uint64_t)uio->uio_offset));
		buf = malloc(want, M_EROFS, M_WAITOK);
		error = z_erofs_do_read(em, en, (uint64_t)uio->uio_offset,
		    want, buf);
		if (error == 0)
			error = uiomove(buf, want, uio);
		free(buf, M_EROFS);
		if (error != 0)
			return (error);
	}
	return (0);
}