erofs-freebsd-out-tree/data.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/_maxphys.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/vnode.h>
#include "internal.h"

#define EROFS_DIR_READAHEAD_BYTES	(1024 * 1024)
#define EROFS_DIR_READAHEAD_SLOTS	(EROFS_DIR_READAHEAD_BYTES / PAGE_SIZE)

/*
 * For flat inline files, compute the "inline tail" start offset.
 * Linux EROFS semantics: the last logical block may be tailpacked
 * into the inode metadata area.
 */
static erofs_off_t
erofs_inline_tail_start(const struct erofs_sb_info *sbi,
    const struct erofs_inode *vi)
{
	if (vi->size == 0)
		return (0);
	return (roundup2(vi->size, (uint64_t)sbi->block_size) - sbi->block_size);
}

static int
erofs_check_device_range(const struct erofs_sb_info *sbi,
    const struct erofs_device_info *dif, erofs_off_t off, uint64_t len)
{
	erofs_off_t end, limit;

	if (dif->blocks > (UINT64_MAX >> sbi->blkszbits) ||
	    __builtin_add_overflow(off, len, &end))
		return (EINTEGRITY);
	limit = dif->blocks << sbi->blkszbits;
	return (end > limit ? EINTEGRITY : 0);
}

static void
erofs_fill_from_devinfo(struct erofs_map_dev *map,
    struct erofs_device_info *dif, erofs_off_t pa)
{
	map->m_dif = dif;
	map->m_pa = pa;
}

static int
erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map)
{
	struct erofs_device_info *dif;
	uint64_t start;
	unsigned int id;
	int error;

	erofs_fill_from_devinfo(map, &sbi->dif0, map->m_pa);
	if (map->m_deviceid != 0) {
		if (map->m_deviceid > sbi->extra_devices || sbi->devs == NULL)
			return (ENODEV);
		dif = &sbi->devs[map->m_deviceid - 1];
		error = erofs_check_device_range(sbi, dif, map->m_pa,
		    map->m_plen);
		if (error != 0)
			return (error);
		if (sbi->flatdev) {
			if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
				return (EINTEGRITY);
			start = dif->uniaddr << sbi->blkszbits;
			if (__builtin_add_overflow(map->m_pa, start, &map->m_pa))
				return (EINTEGRITY);
			return (0);
		}
		if (dif->devvp == NULL || dif->cp == NULL)
			return (ENODEV);
		erofs_fill_from_devinfo(map, dif, map->m_pa);
		return (0);
	}

	if (sbi->extra_devices == 0)
		return (0);
	if (sbi->flatdev) {
		error = erofs_check_device_range(sbi, &sbi->dif0, map->m_pa,
		    map->m_plen);
		if (error == 0)
			return (0);
		for (id = 0; id < sbi->extra_devices; ++id) {
			dif = &sbi->devs[id];
			if (dif->uniaddr == 0 ||
			    dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
				continue;
			start = dif->uniaddr << sbi->blkszbits;
			if (map->m_pa < start)
				continue;
			error = erofs_check_device_range(sbi, dif,
			    map->m_pa - start, map->m_plen);
			if (error == 0)
				return (0);
			if (map->m_pa - start <
			    (dif->blocks << sbi->blkszbits))
				return (error);
		}
		return (EINTEGRITY);
	}
	for (id = 0; id < sbi->extra_devices; ++id) {
		dif = &sbi->devs[id];
		if (dif->uniaddr == 0)
			continue;
		if (dif->uniaddr > (UINT64_MAX >> sbi->blkszbits))
			return (EINTEGRITY);
		start = dif->uniaddr << sbi->blkszbits;
		if (map->m_pa >= start &&
		    map->m_pa - start < (dif->blocks << sbi->blkszbits)) {
			error = erofs_check_device_range(sbi, dif,
			    map->m_pa - start, map->m_plen);
			if (error != 0)
				return (error);
			if (dif->devvp == NULL || dif->cp == NULL)
				return (ENODEV);
			erofs_fill_from_devinfo(map, dif, map->m_pa - start);
			break;
		}
	}
	return (0);
}

/* Map chunk-based file to physical extent */
static int
erofs_map_blocks_chunk(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map)
{
	struct erofs_inode_chunk_index *idx;
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	uint64_t chunk_idx, chunk_size, entry_size;
	erofs_off_t loff, idx_off, chunk_off, idx_base;
	uint64_t image_size, addrmask;
	uint64_t blkaddr;
	uint16_t raw_device_id;
	int error;

	loff = map->m_la;
	chunk_size = 1ULL << vi->chunkbits;
	chunk_idx = loff >> vi->chunkbits;
	chunk_off = loff & (chunk_size - 1);

	if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0)
		entry_size = sizeof(struct erofs_inode_chunk_index);
	else
		entry_size = EROFS_BLOCK_MAP_ENTRY_SIZE;
	if (vi->inode_off > UINT64_MAX - vi->inode_isize ||
	    vi->inode_off + vi->inode_isize > UINT64_MAX - vi->xattr_isize)
		return (EOVERFLOW);
	idx_base = vi->inode_off + vi->inode_isize + vi->xattr_isize;
	if (idx_base > UINT64_MAX - (entry_size - 1))
		return (EOVERFLOW);
	idx_base = roundup2(idx_base, entry_size);
	if (chunk_idx > (UINT64_MAX - idx_base) / entry_size)
		return (EOVERFLOW);
	idx_off = idx_base + chunk_idx * entry_size;
	if (erofs_nid_in_metabox(vi->nid)) {
		if (sbi->metabox_en == NULL || idx_off > sbi->metabox_en->size ||
		    entry_size > sbi->metabox_en->size - idx_off)
			return (EINTEGRITY);
	} else {
		if (sbi->blocks > (UINT64_MAX >> sbi->blkszbits))
			return (EOVERFLOW);
		image_size = sbi->blocks << sbi->blkszbits;
		if (idx_off > image_size || entry_size > image_size - idx_off)
			return (EINTEGRITY);
	}

	error = erofs_read_metadata(sbi, vi->nid, idx_off, entry_size, &buf);
	if (error != 0)
		return (error);

	idx = buf.data;
	if ((vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES) != 0) {
		blkaddr = le32toh(idx->startblk_lo);
		if ((vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0)
			blkaddr |= (uint64_t)le16toh(idx->startblk_hi) << 32;
		raw_device_id = le16toh(idx->device_id);
		addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) != 0 ?
		    ((1ULL << 48) - 1) : UINT32_MAX;
	} else {
		blkaddr = le32dec(idx);
		raw_device_id = 0;
		addrmask = UINT32_MAX;
	}
	erofs_put_metabuf(&buf);

	if (!((blkaddr ^ EROFS_NULL_ADDR) & addrmask)) {
		map->m_pa = 0;
		map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff);
		map->m_plen = map->m_llen;
		return (0);
	}

	map->m_llen = MIN(chunk_size - chunk_off, vi->size - loff);
	map->m_plen = map->m_llen;
	map->m_deviceid = raw_device_id & sbi->device_id_mask;
	if (blkaddr > (UINT64_MAX >> sbi->blkszbits))
		return (EOVERFLOW);
	map->m_pa = blkaddr << sbi->blkszbits;
	if (chunk_off > UINT64_MAX - map->m_pa)
		return (EOVERFLOW);
	map->m_pa += chunk_off;
	map->m_flags |= EROFS_MAP_MAPPED;
	return (0);
}

static int
erofs_bread_device(struct erofs_sb_info *sbi, struct erofs_device_info *dif,
    erofs_blk_t blocks, erofs_off_t off, size_t len, daddr_t *rablkno,
    int *rabsize, int racnt, void **bufp)
{
	struct buf *bp;
	erofs_off_t end, limit;
	off_t blkoff, current;
	size_t blklen, done, iosize;
	char *out;
	int error;

	if (bufp == NULL)
		return (EINVAL);
	*bufp = NULL;
	if (len == 0) {
		return (0);
	}
	if (dif == NULL || dif->devvp == NULL || dif->cp == NULL)
		return (ENODEV);
	if (__builtin_add_overflow(off, (uint64_t)len, &end))
		return (EINTEGRITY);
	if (blocks != 0) {
		if (blocks > (UINT64_MAX >> sbi->blkszbits))
			return (EINTEGRITY);
		limit = blocks << sbi->blkszbits;
		if (end > limit)
			return (EINTEGRITY);
	}
	if (end > dif->mediasize)
		return (ENXIO);
	if (off > INT64_MAX || end > (uint64_t)INT64_MAX + 1)
		return (EOVERFLOW);

	iosize = sbi->block_size != 0 ? sbi->block_size : dif->sectorsize;
	if (iosize == 0 || (iosize & (iosize - 1)) != 0)
		return (EINVAL);
	out = malloc(len, M_EROFS, M_WAITOK);
	done = 0;
	while (done < len) {
		current = (off_t)(off + done);
		blkoff = rounddown2(current, (off_t)iosize);
		blklen = MIN(iosize - (size_t)(current - blkoff), len - done);
		if (done == 0 && racnt != 0)
			error = breadn(dif->devvp, btodb(blkoff), iosize,
			    rablkno, rabsize, racnt, NOCRED, &bp);
		else
			error = bread(dif->devvp, btodb(blkoff), iosize, NOCRED, &bp);
		if (error != 0) {
			free(out, M_EROFS);
			return (error);
		}
		if (bp->b_data == NULL) {
			brelse(bp);
			free(out, M_EROFS);
			return (EIO);
		}
		memcpy(out + done, (char *)bp->b_data + (current - blkoff),
		    blklen);
		brelse(bp);
		done += blklen;
	}
	*bufp = out;
	return (0);
}

int
erofs_bread(struct erofs_sb_info *sbi, erofs_off_t off, size_t len, void **bufp)
{
	return (erofs_bread_device(sbi, &sbi->dif0, sbi->dif0.blocks, off, len,
	    NULL, NULL, 0, bufp));
}

int
erofs_read_physical(struct erofs_sb_info *sbi, unsigned int device_id,
    erofs_off_t off, size_t len, void **bufp)
{
	struct erofs_map_dev map;
	erofs_blk_t blocks;
	int error;

	map = (struct erofs_map_dev) {
		.m_pa = off,
		.m_deviceid = device_id,
		.m_plen = len,
	};
	error = erofs_map_dev(sbi, &map);
	if (error != 0)
		return (error);
	blocks = map.m_dif->blocks;
	if (map.m_dif == &sbi->dif0 && sbi->flatdev)
		blocks = sbi->flatdev_blocks;
	return (erofs_bread_device(sbi, map.m_dif, blocks, map.m_pa, len,
	    NULL, NULL, 0, bufp));
}

static int
erofs_read_physical_readahead(struct erofs_sb_info *sbi,
    unsigned int device_id, erofs_off_t off, size_t len,
    unsigned int rablocks, void **bufp)
{
	struct erofs_map_dev current, future;
	daddr_t rablkno[EROFS_DIR_READAHEAD_SLOTS];
	int rabsize[EROFS_DIR_READAHEAD_SLOTS];
	erofs_off_t step;
	erofs_blk_t blocks;
	unsigned int count;
	int error;

	current = (struct erofs_map_dev) {
		.m_pa = off,
		.m_deviceid = device_id,
		.m_plen = len,
	};
	error = erofs_map_dev(sbi, &current);
	if (error != 0)
		return (error);
	blocks = current.m_dif->blocks;
	if (current.m_dif == &sbi->dif0 && sbi->flatdev)
		blocks = sbi->flatdev_blocks;
	rablocks = MIN(rablocks, (unsigned int)nitems(rablkno));
	for (count = 0; count < rablocks; count++) {
		step = (erofs_off_t)(count + 1) * sbi->block_size;
		if (off > UINT64_MAX - step || current.m_pa > UINT64_MAX - step)
			break;
		future = (struct erofs_map_dev) {
			.m_pa = off + step,
			.m_deviceid = device_id,
			.m_plen = sbi->block_size,
		};
		if (erofs_map_dev(sbi, &future) != 0 ||
		    future.m_dif != current.m_dif ||
		    future.m_pa != current.m_pa + step)
			break;
		rablkno[count] = btodb(future.m_pa);
		rabsize[count] = sbi->block_size;
	}
	return (erofs_bread_device(sbi, current.m_dif, blocks, current.m_pa, len,
	    rablkno, rabsize, count, bufp));
}

/* Release a contiguous buffer returned by erofs_bread(). */
void
erofs_brelse(void *buf)
{
	free(buf, M_EROFS);
}

void
erofs_put_metabuf(struct erofs_buf *buf)
{
	void (*release)(void *);
	void *data;

	if (buf == NULL)
		return;
	data = buf->data;
	release = buf->release;
	buf->data = NULL;
	buf->release = NULL;
	if (data != NULL && release != NULL)
		release(data);
}

/* Read inode metadata from either the primary image or the metabox file. */
int
erofs_read_metadata(struct erofs_sb_info *sbi, erofs_nid_t nid, erofs_off_t off,
    size_t len, struct erofs_buf *buf)
{
	void *data;
	int error;

	if (!erofs_nid_in_metabox(nid)) {
		if (off > INT64_MAX)
			return (EOVERFLOW);
		error = erofs_bread(sbi, (off_t)off, len, &data);
	} else {
		if (!erofs_sb_has_metabox(sbi) || sbi->metabox_en == NULL)
			return (EINTEGRITY);
		error = erofs_read_data(sbi, sbi->metabox_en, off, len, &data);
	}
	if (error != 0)
		return (error);
	buf->data = data;
	buf->release = erofs_brelse;
	return (0);
}

static int
erofs_map_blocks_flatmode(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map)
{
	erofs_off_t loff, tail_start;
	uint64_t remain, block_rem;

	loff = map->m_la;
	if (loff >= vi->size)
		return (0);

	remain = vi->size - loff;
	switch (vi->datalayout) {
	case EROFS_INODE_CHUNK_BASED:
		return (erofs_map_blocks_chunk(sbi, vi, map));
	case EROFS_INODE_FLAT_PLAIN:
		map->m_llen = remain;
		map->m_plen = map->m_llen;
		if (vi->startblk == EROFS_NULL_ADDR)
			return (0);
		if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) ||
		    __builtin_add_overflow(vi->startblk << sbi->blkszbits, loff,
		    &map->m_pa))
			return (EINTEGRITY);
		map->m_flags |= EROFS_MAP_MAPPED;
		return (0);
	case EROFS_INODE_FLAT_INLINE:
		tail_start = erofs_inline_tail_start(sbi, vi);
		if (loff < tail_start) {
			map->m_llen = MIN(remain, tail_start - loff);
			map->m_plen = map->m_llen;
			if (vi->startblk == EROFS_NULL_ADDR)
				return (0);
			if (vi->startblk > (UINT64_MAX >> sbi->blkszbits) ||
			    __builtin_add_overflow(vi->startblk << sbi->blkszbits,
			    loff, &map->m_pa))
				return (EINTEGRITY);
			map->m_flags |= EROFS_MAP_MAPPED;
			return (0);
		}
		block_rem = sbi->block_size -
		    ((loff - tail_start) & (sbi->block_size - 1));
		map->m_llen = MIN(remain, block_rem);
		map->m_plen = map->m_llen;
		if (__builtin_add_overflow(vi->inode_off, vi->inode_isize,
		    &map->m_pa) || __builtin_add_overflow(map->m_pa, vi->xattr_isize,
		    &map->m_pa) || __builtin_add_overflow(map->m_pa, loff - tail_start,
		    &map->m_pa))
			return (EINTEGRITY);
		map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_META;
		return (0);
	case EROFS_INODE_COMPRESSED_FULL:
	case EROFS_INODE_COMPRESSED_COMPACT:
		return (EOPNOTSUPP);
	default:
		return (EOPNOTSUPP);
	}
}

int
erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map)
{
	struct erofs_map_blocks next = { .m_la = map->m_la };
	int error;

	if (erofs_inode_is_data_compressed(vi->datalayout)) {
		error = z_erofs_map_blocks(sbi, vi, &next);
	} else {
		error = erofs_map_blocks_flatmode(sbi, vi, &next);
	}
	*map = next;
	return (error);
}
/*
 * Read a small range at a logical file offset into a contiguous buffer.
 * Primarily used for directory block reads, lookup, and symlink fragment
 * parsing.
 */
static int
erofs_read_data_impl(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t loff, size_t len, unsigned int rablocks, void **bufp)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct erofs_map_blocks map;
	char *out;
	void *blk;
	size_t done, want;
	int error;

	if (bufp == NULL)
		return (EINVAL);
	*bufp = NULL;
	if (len == 0) {
		return (0);
	}
	if (loff > UINT64_MAX - (uint64_t)len)
		return (EOVERFLOW);
	if (loff > vi->size || (uint64_t)len > vi->size - loff)
		return (EINTEGRITY);

	/* Compressed file path */
	if (erofs_inode_is_data_compressed(vi->datalayout))
		return (z_erofs_read_data(sbi, vi, loff, len, bufp));

	/* Uncompressed file path */
	out = malloc(len, M_EROFS, M_WAITOK);
	done = 0;
	while (done < len) {
		map = (struct erofs_map_blocks) { .m_la = loff + done };
		error = erofs_map_blocks(sbi, vi, &map);
		if (error != 0) {
			free(out, M_EROFS);
			return (error);
		}
		if (map.m_llen == 0) {
			free(out, M_EROFS);
			return (EINTEGRITY);
		}
		want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS),
		    len - done);
		if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
			bzero(out + done, want);
		} else {
			if ((map.m_flags & EROFS_MAP_META) != 0) {
				error = erofs_read_metadata(sbi, vi->nid, map.m_pa,
				    want, &buf);
			} else if (done == 0 && rablocks != 0 &&
			    map.m_flags == EROFS_MAP_MAPPED) {
				error = erofs_read_physical_readahead(sbi,
				    map.m_deviceid, map.m_pa, want, rablocks, &blk);
			} else {
				error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa,
				    want, &blk);
			}
			if (error != 0) {
				free(out, M_EROFS);
				return (error);
			}
			if ((map.m_flags & EROFS_MAP_META) != 0) {
				memcpy(out + done, buf.data, want);
				erofs_put_metabuf(&buf);
			} else {
				memcpy(out + done, blk, want);
				erofs_brelse(blk);
			}
		}
		done += want;
	}
	*bufp = out;
	return (0);
}

int
erofs_read_data(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t loff, size_t len, void **bufp)
{
	return (erofs_read_data_impl(sbi, vi, loff, len, 0, bufp));
}

int
erofs_read_data_readahead(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t loff, size_t len, bool sequential, void **bufp)
{
	uint64_t remaining;
	unsigned int rablocks;

	rablocks = 0;
	if (sequential && vi->datalayout == EROFS_INODE_FLAT_PLAIN &&
	    sbi->block_size != 0 && (loff & (sbi->block_size - 1)) == 0 &&
	    len <= sbi->block_size && loff <= vi->size && len <= vi->size - loff) {
		remaining = vi->size - loff - len;
		rablocks = MIN(howmany(remaining, sbi->block_size),
		    (uint64_t)EROFS_DIR_READAHEAD_SLOTS);
	}
	return (erofs_read_data_impl(sbi, vi, loff, len, rablocks, bufp));
}

/*
 * Transfer the logical content of an inode directly into a uio.
 * Regular files and symlinks both use this read path.
 */
static int
erofs_read_uio(struct erofs_sb_info *sbi, struct erofs_inode *vi, struct uio *uio)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct erofs_map_blocks map;
	char zerobuf[PAGE_SIZE];
	void *blk;
	size_t want, chunk;
	int error;

	if (uio->uio_offset < 0)
		return (EINVAL);
	if ((uint64_t)uio->uio_offset >= vi->size)
		return (0);

	/* Compressed file path */
	if (erofs_inode_is_data_compressed(vi->datalayout))
		return (z_erofs_read_uio(sbi, vi, uio));

	/* Uncompressed file path */
	bzero(zerobuf, sizeof(zerobuf));
	while (uio->uio_resid > 0 && (uint64_t)uio->uio_offset < vi->size) {
		map = (struct erofs_map_blocks) { .m_la = uio->uio_offset };
		error = erofs_map_blocks(sbi, vi, &map);
		if (error != 0)
			return (error);
		if (map.m_llen == 0)
			break;
		want = MIN((size_t)MIN(map.m_llen, (uint64_t)MAXPHYS),
		    (size_t)uio->uio_resid);
		if ((map.m_flags & EROFS_MAP_MAPPED) == 0) {
			chunk = want;
			while (chunk > 0) {
				size_t zlen = MIN(chunk, sizeof(zerobuf));

				error = uiomove(zerobuf, zlen, uio);
				if (error != 0)
					return (error);
				chunk -= zlen;
			}
			continue;
		}
		if ((map.m_flags & EROFS_MAP_META) != 0) {
			error = erofs_read_metadata(sbi, vi->nid, map.m_pa, want,
			    &buf);
		} else {
			error = erofs_read_physical(sbi, map.m_deviceid, map.m_pa,
			    want, &blk);
		}
		if (error != 0)
			return (error);
		if ((map.m_flags & EROFS_MAP_META) != 0) {
			error = uiomove(buf.data, want, uio);
			erofs_put_metabuf(&buf);
		} else {
			error = uiomove(blk, want, uio);
			erofs_brelse(blk);
		}
		if (error != 0)
			return (error);
	}
	return (0);
}

/* Read regular file data. */
int
erofs_read_file(struct vnode *vp, struct uio *uio, int ioflag)
{
	(void)ioflag;
	return (erofs_read_uio(MTOE(vp->v_mount), VTOE(vp), uio));
}

int
erofs_validate_symlink_target(struct erofs_sb_info *sbi,
    struct erofs_inode *vi)
{
	void *target;
	int error;

	if (vi->vtype != VLNK)
		return (EINVAL);
	if (vi->size == 0)
		return (EINTEGRITY);
	if (vi->size > MAXPATHLEN)
		return (ENAMETOOLONG);
	error = erofs_read_data(sbi, vi, 0, (size_t)vi->size, &target);
	if (error != 0)
		return (error);
	if (memchr(target, '\0', (size_t)vi->size) != NULL)
		error = EINTEGRITY;
	erofs_brelse(target);
	return (error);
}

/* Read symlink target string. */
int
erofs_readlink_target(struct vnode *vp, struct uio *uio)
{
	struct erofs_inode *vi;

	vi = VTOE(vp);
	if (vi->size == 0)
		return (EINTEGRITY);
	if (vi->size > MAXPATHLEN)
		return (ENAMETOOLONG);
	return (erofs_read_uio(MTOE(vp->v_mount), vi, uio));
}