erofs-freebsd-out-tree/zmap.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2018-2019 HUAWEI, Inc.
 *             https://www.huawei.com/
 */

#include <sys/param.h>
#include <sys/endian.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/systm.h>

#include "internal.h"

#define Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE (64 * 1024)

struct z_erofs_maprecorder {
	struct erofs_sb_info *sbi;
	struct erofs_inode *vi;
	struct erofs_map_blocks *map;
	uint64_t lcn;
	uint8_t type;
	uint8_t headtype;
	unsigned int clusterofs;
	uint16_t delta[2];
	erofs_blk_t pblk;
	erofs_blk_t compressedblks;
	erofs_off_t nextpackoff;
	bool partialref;
};

_Static_assert((EROFS_MAP_MAPPED | EROFS_MAP_META |
    EROFS_MAP_PARTIAL_MAPPED | EROFS_MAP_PARTIAL_REF |
    EROFS_MAP_FRAGMENT) == 0x1f, "map adapter must preserve every map flag");

static int
z_erofs_index_base(const struct erofs_inode *vi, erofs_off_t *base)
{
	erofs_off_t end;

	if (__builtin_add_overflow(vi->inode_off,
	    (erofs_off_t)vi->inode_isize, &end) ||
	    __builtin_add_overflow(end, (erofs_off_t)vi->xattr_isize, &end) ||
	    __builtin_add_overflow(end, (erofs_off_t)7, &end))
		return (EINTEGRITY);
	end = rounddown2(end, 8);
	if (__builtin_add_overflow(end,
	    (erofs_off_t)sizeof(struct z_erofs_map_header), base))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_index_advance(erofs_off_t *pos, uint64_t count, uint64_t unit)
{
	erofs_off_t delta, next;

	if (__builtin_mul_overflow(count, unit, &delta) ||
	    __builtin_add_overflow(*pos, delta, &next))
		return (EINTEGRITY);
	*pos = next;
	return (0);
}

static int
z_erofs_lcluster_count(uint64_t size, unsigned int lclusterbits,
    uint64_t *count)
{
	uint64_t mask;

	if (lclusterbits >= 64)
		return (EINTEGRITY);
	mask = (1ULL << lclusterbits) - 1;
	*count = size >> lclusterbits;
	if ((size & mask) != 0 && __builtin_add_overflow(*count, 1, count))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_lcluster_pos(uint64_t lcn, unsigned int lclusterbits,
    uint64_t clusterofs, uint64_t *pos)
{
	uint64_t base, clustersize;

	if (lclusterbits >= 64)
		return (EINTEGRITY);
	clustersize = 1ULL << lclusterbits;
	if (__builtin_mul_overflow(lcn, clustersize, &base) ||
	    __builtin_add_overflow(base, clusterofs, pos))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_lcn_advance(uint64_t *lcn, uint64_t delta)
{
	uint64_t next;

	if (__builtin_add_overflow(*lcn, delta, &next))
		return (EINTEGRITY);
	*lcn = next;
	return (0);
}

static int
z_erofs_compact_pblk(uint32_t base, unsigned int nblk, erofs_blk_t *pblk)
{
	if (__builtin_add_overflow((erofs_blk_t)base, (erofs_blk_t)nblk,
	    pblk))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_fragment_offset(uint64_t low, erofs_blk_t high, erofs_off_t *offset)
{
	if (low > UINT32_MAX || high > UINT32_MAX)
		return (EINTEGRITY);
	*offset = low | (high << 32);
	return (0);
}

static int
z_erofs_post_eof_len(uint64_t la, uint64_t size, uint64_t *len)
{
	if (la < size)
		return (EINTEGRITY);
	*len = la - size;
	if (__builtin_add_overflow(*len, 1, len))
		*len = UINT64_MAX;
	return (0);
}

static int
z_erofs_physical_end(uint64_t pa, uint64_t plen, uint64_t block_size)
{
	uint64_t limit, pend;

	if (__builtin_add_overflow(pa, plen, &pend))
		return (EINTEGRITY);
	if (__builtin_mul_overflow(1ULL << 48, block_size, &limit))
		return (0);
	if (pend > limit)
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_read_index(struct z_erofs_maprecorder *m, erofs_off_t pos, size_t len,
    struct erofs_buf *buf)
{
	return (erofs_read_metadata(m->sbi, m->vi->nid, pos, len, buf));
}

static int
z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct erofs_inode *vi;
	struct z_erofs_lcluster_index *di;
	erofs_off_t base, pos;
	unsigned int advise;
	int error;

	vi = m->vi;
	if (z_erofs_index_base(vi, &base) != 0 ||
	    z_erofs_index_advance(&base, 1, 8) != 0)
		return (EINTEGRITY);
	pos = base;
	if (z_erofs_index_advance(&pos, lcn, sizeof(*di)) != 0)
		return (EINTEGRITY);
	m->nextpackoff = pos;
	if (z_erofs_index_advance(&m->nextpackoff, 1, sizeof(*di)) != 0)
		return (EINTEGRITY);
	error = z_erofs_read_index(m, pos, sizeof(*di), &buf);
	if (error != 0)
		return (error);

	di = buf.data;
	m->lcn = lcn;
	advise = le16toh(di->di_advise);
	m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK;
	if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
		m->clusterofs = 1U << vi->z_lclusterbits;
		m->delta[0] = le16toh(di->di_u.delta[0]);
		if ((m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) != 0) {
			if ((vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
			    Z_EROFS_ADVISE_BIG_PCLUSTER_2)) == 0) {
				erofs_put_metabuf(&buf);
				return (EINTEGRITY);
			}
			m->compressedblks =
			    m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT;
			m->delta[0] = 1;
		}
		m->delta[1] = le16toh(di->di_u.delta[1]);
	} else {
		m->partialref = (advise & Z_EROFS_LI_PARTIAL_REF) != 0;
		m->clusterofs = le16toh(di->di_clusterofs);
		m->pblk = le32toh(di->di_u.blkaddr);
	}
	erofs_put_metabuf(&buf);
	return (0);
}

static unsigned int
decode_compactedbits(unsigned int lobits, const uint8_t *in,
    unsigned int pos, uint8_t *type)
{
	uint32_t value;
	unsigned int lo;

	value = le32dec(in + pos / 8) >> (pos & 7);
	lo = value & ((1U << lobits) - 1);
	*type = (value >> lobits) & 3;
	return (lo);
}

static int
get_compacted_la_distance(unsigned int lobits, unsigned int encodebits,
    unsigned int vcnt, const uint8_t *in, int i)
{
	unsigned int lo, distance;
	uint8_t type;

	distance = 0;
	do {
		lo = decode_compactedbits(lobits, in, encodebits * i, &type);
		if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
			return (distance);
		++distance;
	} while (++i < (int)vcnt);

	if ((lo & Z_EROFS_LI_D0_CBLKCNT) == 0) {
		if (lo == 0)
			return (-1);
		distance += lo - 1;
	}
	return ((int)distance);
}

static int
z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, uint64_t lcn,
    bool lookahead)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct erofs_inode *vi;
	erofs_off_t ebase, pos;
	uint64_t compacted_2b, totalidx, original_lcn;
	unsigned int compacted_4b_initial, amortizedshift;
	unsigned int vcnt, lo, lobits, encodebits, nblk, bytes, packsize;
	bool big_pcluster;
	uint8_t *in, type;
	int distance, error, i;

	vi = m->vi;
	if (vi->z_lclusterbits > 14 ||
	    z_erofs_index_base(vi, &ebase) != 0 ||
	    z_erofs_lcluster_count(vi->size, vi->z_lclusterbits,
	    &totalidx) != 0)
		return (EINTEGRITY);
	if (lcn >= totalidx)
		return (EINVAL);

	original_lcn = lcn;
	m->lcn = lcn;
	compacted_4b_initial = ((32 - ebase % 32) / 4) & 7;
	compacted_2b = 0;
	if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) != 0 &&
	    compacted_4b_initial < totalidx)
		compacted_2b = rounddown2(totalidx - compacted_4b_initial, 16);

	pos = ebase;
	amortizedshift = 2;
	if (lcn >= compacted_4b_initial) {
		if (z_erofs_index_advance(&pos, compacted_4b_initial, 4) != 0)
			return (EINTEGRITY);
		lcn -= compacted_4b_initial;
		if (lcn < compacted_2b) {
			amortizedshift = 1;
		} else {
			if (z_erofs_index_advance(&pos, compacted_2b, 2) != 0)
				return (EINTEGRITY);
			lcn -= compacted_2b;
		}
	}
	if (z_erofs_index_advance(&pos, lcn, 1U << amortizedshift) != 0)
		return (EINTEGRITY);

	if (amortizedshift == 2 && vi->z_lclusterbits <= 14)
		vcnt = 2;
	else if (amortizedshift == 1 && vi->z_lclusterbits <= 12)
		vcnt = 16;
	else
		return (EOPNOTSUPP);

	packsize = vcnt << amortizedshift;
	bytes = pos & (packsize - 1);
	pos -= bytes;
	m->nextpackoff = pos;
	if (z_erofs_index_advance(&m->nextpackoff, 1, packsize) != 0)
		return (EINTEGRITY);
	error = z_erofs_read_index(m, pos, packsize, &buf);
	if (error != 0)
		return (error);
	in = buf.data;
	lobits = MAX(vi->z_lclusterbits, fls(Z_EROFS_LI_D0_CBLKCNT));
	encodebits = (packsize - sizeof(uint32_t)) * 8 / vcnt;
	i = bytes >> amortizedshift;

	lo = decode_compactedbits(lobits, in, encodebits * i, &type);
	m->type = type;
	if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
		m->clusterofs = 1U << vi->z_lclusterbits;
		if (lookahead) {
			distance = get_compacted_la_distance(lobits, encodebits,
			    vcnt, in, i);
			if (distance < 0) {
				erofs_put_metabuf(&buf);
				return (EINTEGRITY);
			}
			m->delta[1] = distance;
		}
		big_pcluster =
		    (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
		if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) {
			if (!big_pcluster) {
				erofs_put_metabuf(&buf);
				return (EINTEGRITY);
			}
			m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT;
			m->delta[0] = 1;
		} else if (i + 1 != (int)vcnt) {
			m->delta[0] = lo;
		} else {
			if (i == 0) {
				erofs_put_metabuf(&buf);
				return (EINTEGRITY);
			}
			lo = decode_compactedbits(lobits, in,
			    encodebits * (i - 1), &type);
			if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
				lo = 0;
			else if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0)
				lo = 1;
			m->delta[0] = lo + 1;
		}
		erofs_put_metabuf(&buf);
		return (m->delta[0] == 0 ? EINTEGRITY : 0);
	}

	m->clusterofs = lo;
	m->delta[0] = 0;
	big_pcluster =
	    (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
	if (!big_pcluster) {
		nblk = 1;
		while (i > 0) {
			--i;
			lo = decode_compactedbits(lobits, in,
			    encodebits * i, &type);
			if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD)
				i -= lo;
			if (i >= 0)
				++nblk;
		}
	} else {
		nblk = 0;
		while (i > 0) {
			--i;
			lo = decode_compactedbits(lobits, in,
			    encodebits * i, &type);
			if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
				if ((lo & Z_EROFS_LI_D0_CBLKCNT) != 0) {
					--i;
					nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT;
					continue;
				}
				if (lo <= 1) {
					erofs_put_metabuf(&buf);
					return (EINTEGRITY);
				}
				i -= lo - 2;
				continue;
			}
			++nblk;
		}
	}
	error = z_erofs_compact_pblk(
	    le32dec(in + packsize - sizeof(uint32_t)), nblk, &m->pblk);
	erofs_put_metabuf(&buf);
	if (error != 0)
		return (error);
	m->lcn = original_lcn;
	return (0);
}

static int
z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, uint64_t lcn,
    bool lookahead)
{
	int error;

	if (m->vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT)
		error = z_erofs_load_compact_lcluster(m, lcn, lookahead);
	else if (m->vi->datalayout == EROFS_INODE_COMPRESSED_FULL)
		error = z_erofs_load_full_lcluster(m, lcn);
	else
		return (EINTEGRITY);
	if (error != 0)
		return (error);
	if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX)
		return (EOPNOTSUPP);
	if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
	    m->clusterofs >= (1U << m->vi->z_lclusterbits))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
    unsigned int lookback_distance)
{
	uint64_t lcn;
	int error;

	while (lookback_distance != 0 && m->lcn >= lookback_distance) {
		lcn = m->lcn - lookback_distance;
		error = z_erofs_load_lcluster_from_disk(m, lcn, false);
		if (error != 0)
			return (error);
		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
			lookback_distance = m->delta[0];
			continue;
		}
		m->headtype = m->type;
		error = z_erofs_lcluster_pos(lcn, m->vi->z_lclusterbits,
		    m->clusterofs, &m->map->m_la);
		if (error != 0)
			return (error);
		return (0);
	}
	return (EINTEGRITY);
}

static int
z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
    uint64_t initial_lcn)
{
	struct erofs_inode *vi;
	bool bigpcl1, bigpcl2;
	uint64_t lcn, lcnpos;
	int error;

	vi = m->vi;
	bigpcl1 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0;
	bigpcl2 = (vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0;
	lcn = m->lcn;
	if (z_erofs_lcn_advance(&lcn, 1) != 0 ||
	    z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0, &lcnpos) != 0)
		return (EINTEGRITY);
	if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) ||
	    ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN ||
	    m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) ||
	    lcnpos >= vi->size)
		m->compressedblks = 1;
	if (m->compressedblks == 0) {
		error = z_erofs_load_lcluster_from_disk(m, lcn, false);
		if (error != 0)
			return (error);
		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
		    m->delta[0] != 1)
			return (EINTEGRITY);
		if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD ||
		    m->compressedblks == 0)
			m->compressedblks = 1;
	}
	if (__builtin_mul_overflow(m->compressedblks,
	    (uint64_t)m->sbi->block_size, &m->map->m_plen))
		return (EINTEGRITY);
	(void)initial_lcn;
	return (0);
}

static int
z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
{
	struct erofs_inode *vi;
	struct erofs_map_blocks *map;
	uint64_t lcn, headlcn, lend, lcnpos;
	int error;

	vi = m->vi;
	map = m->map;
	lcn = m->lcn;
	headlcn = map->m_la >> vi->z_lclusterbits;
	for (;;) {
		if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, 0,
		    &lcnpos) != 0)
			return (EINTEGRITY);
		if (lcnpos >= vi->size) {
			if (map->m_la > vi->size)
				return (EINTEGRITY);
			map->m_llen = vi->size - map->m_la;
			return (0);
		}
		error = z_erofs_load_lcluster_from_disk(m, lcn, true);
		if (error != 0)
			return (error);
		if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
			if (m->delta[1] == 0)
				m->delta[1] = 1;
		} else {
			if (lcn != headlcn)
				break;
			m->delta[1] = 1;
		}
		if (z_erofs_lcn_advance(&lcn, m->delta[1]) != 0)
			return (EINTEGRITY);
	}
	if (z_erofs_lcluster_pos(lcn, vi->z_lclusterbits, m->clusterofs,
	    &lend) != 0 || lend < map->m_la)
		return (EINTEGRITY);
	map->m_llen = lend - map->m_la;
	return (0);
}

static int
z_erofs_extent_add(uint64_t left, uint64_t right, uint64_t *result)
{
	if (__builtin_add_overflow(left, right, result))
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_extent_roundup(uint64_t value, unsigned int alignment,
    uint64_t *result)
{
	uint64_t rounded;

	if (z_erofs_extent_add(value, alignment - 1, &rounded) != 0)
		return (EINTEGRITY);
	*result = rounddown2(rounded, alignment);
	return (0);
}

static int
z_erofs_extent_table_pos(const struct erofs_inode *vi, unsigned int recsz,
    erofs_off_t *result)
{
	erofs_off_t pos;

	if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 ||
	    z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 ||
	    z_erofs_extent_roundup(pos, 8, &pos) != 0 ||
	    z_erofs_extent_add(pos, sizeof(struct z_erofs_map_header), &pos) != 0 ||
	    z_erofs_extent_roundup(pos, recsz, result) != 0)
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_extent_record_pos(const struct erofs_inode *vi, erofs_off_t table_pos,
    unsigned int recsz, uint64_t index, erofs_off_t *result)
{
	uint64_t offset;

	if (index >= vi->z_extents ||
	    __builtin_mul_overflow(index, recsz, &offset) ||
	    z_erofs_extent_add(table_pos, offset, result) != 0)
		return (EINTEGRITY);
	return (0);
}

static int
z_erofs_read_extent(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    erofs_off_t pos, unsigned int recsz, struct z_erofs_extent *ext)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	int error;

	bzero(ext, sizeof(*ext));
	error = erofs_read_metadata(sbi, vi->nid, pos, recsz, &buf);
	if (error != 0)
		return (error);
	memcpy(ext, buf.data, recsz);
	erofs_put_metabuf(&buf);
	return (0);
}

static uint64_t
z_erofs_extent_lstart(const struct z_erofs_extent *ext, unsigned int recsz)
{
	uint64_t lstart;

	lstart = le32toh(ext->lstart_lo);
	if (recsz > offsetof(struct z_erofs_extent, lstart_hi))
		lstart |= (uint64_t)le32toh(ext->lstart_hi) << 32;
	return (lstart);
}

static int
z_erofs_validate_extent_table(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    unsigned int recsz)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct z_erofs_extent ext;
	erofs_off_t extent_pos, record_pos, scan_pos, table_end;
	uint64_t index, lstart, previous;
	size_t chunk_len, offset;
	int error;

	if (vi->z_extents == 0)
		return (vi->size == 0 ? 0 : EINTEGRITY);
	error = z_erofs_extent_table_pos(vi, recsz, &extent_pos);
	if (error != 0)
		return (error);
	if (recsz <= offsetof(struct z_erofs_extent, pstart_lo) &&
	    z_erofs_extent_add(extent_pos, sizeof(uint64_t), &extent_pos) != 0)
		return (EINTEGRITY);
	error = z_erofs_extent_record_pos(vi, extent_pos, recsz,
	    vi->z_extents - 1, &record_pos);
	if (error != 0 ||
	    z_erofs_extent_add(record_pos, recsz, &table_end) != 0)
		return (EINTEGRITY);
	if (recsz <= offsetof(struct z_erofs_extent, pstart_hi))
		return (z_erofs_read_extent(sbi, vi, record_pos, recsz, &ext));
	if (vi->size == 0)
		return (EINTEGRITY);

	scan_pos = extent_pos;
	index = 0;
	previous = 0;
	while (scan_pos < table_end) {
		chunk_len = (size_t)MIN(table_end - scan_pos,
		    (uint64_t)Z_EROFS_EXTENT_VALIDATE_CHUNK_SIZE);
		error = erofs_read_metadata(sbi, vi->nid, scan_pos, chunk_len, &buf);
		if (error != 0)
			return (error);
		for (offset = 0; offset < chunk_len; offset += recsz, index++) {
			bzero(&ext, sizeof(ext));
			memcpy(&ext, (const char *)buf.data + offset, recsz);
			lstart = z_erofs_extent_lstart(&ext, recsz);
			if (lstart >= vi->size ||
			    (index != 0 && lstart <= previous)) {
				error = EINTEGRITY;
				goto out;
			}
			previous = lstart;
		}
		erofs_put_metabuf(&buf);
		scan_pos += chunk_len;
	}
	return (index == vi->z_extents ? 0 : EINTEGRITY);
out:
	erofs_put_metabuf(&buf);
	return (error);
}

static int
z_erofs_map_blocks_fo(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map, int flags)
{
	bool fragment, ztailpacking;
	struct z_erofs_maprecorder m;
	uint64_t initial_lcn, ofs, end, end_lcn;
	erofs_off_t fragmentoff;
	unsigned int endoff;
	int error;

	fragment = (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0;
	ztailpacking = vi->z_idata_size != 0;
	bzero(&m, sizeof(m));
	m.sbi = sbi;
	m.vi = vi;
	m.map = map;

	if (vi->size == 0) {
		map->m_la = 0;
		map->m_llen = 0;
		map->m_flags = 0;
		return (0);
	}
	ofs = (flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 ?
	    vi->size - 1 : map->m_la;
	if (fragment && (flags & EROFS_GET_BLOCKS_FINDTAIL) == 0 &&
	    vi->z_tailextent_headlcn == 0) {
		map->m_la = 0;
		map->m_llen = vi->size;
		map->m_flags = EROFS_MAP_FRAGMENT;
		return (0);
	}
	initial_lcn = ofs >> vi->z_lclusterbits;
	endoff = ofs & ((1U << vi->z_lclusterbits) - 1);
	error = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false);
	if (error != 0)
		return (error);
	if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0 && ztailpacking)
		vi->z_fragmentoff = m.nextpackoff;

	map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED;
	end_lcn = m.lcn;
	if (z_erofs_lcn_advance(&end_lcn, 1) != 0 ||
	    z_erofs_lcluster_pos(end_lcn, vi->z_lclusterbits, 0, &end) != 0)
		return (EINTEGRITY);
	if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD &&
	    endoff >= m.clusterofs) {
		m.headtype = m.type;
		if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits,
		    m.clusterofs, &map->m_la) != 0)
			return (EINTEGRITY);
		if (ztailpacking && end > vi->size)
			end = vi->size;
	} else {
		if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
			if (z_erofs_lcluster_pos(m.lcn, vi->z_lclusterbits,
			    m.clusterofs, &end) != 0)
				return (EINTEGRITY);
			map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED;
			m.delta[0] = 1;
		}
		error = z_erofs_extent_lookback(&m, m.delta[0]);
		if (error != 0)
			return (error);
	}
	if (m.partialref)
		map->m_flags |= EROFS_MAP_PARTIAL_REF;
	if (end < map->m_la)
		return (EINTEGRITY);
	map->m_llen = end - map->m_la;

	if ((flags & EROFS_GET_BLOCKS_FINDTAIL) != 0) {
		vi->z_tailextent_headlcn = m.lcn;
		if (fragment &&
		    vi->datalayout == EROFS_INODE_COMPRESSED_FULL) {
			error = z_erofs_fragment_offset(vi->z_fragmentoff, m.pblk,
			    &fragmentoff);
			if (error != 0)
				return (error);
			vi->z_fragmentoff = fragmentoff;
		}
	}
	if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
		map->m_flags |= EROFS_MAP_META;
		map->m_pa = vi->z_fragmentoff;
		map->m_plen = vi->z_idata_size;
		if ((map->m_pa & (sbi->block_size - 1)) + map->m_plen >
		    sbi->block_size)
			return (EINTEGRITY);
	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
		map->m_flags = EROFS_MAP_FRAGMENT;
	} else {
		if (__builtin_mul_overflow(m.pblk, (erofs_blk_t)sbi->block_size,
		    &map->m_pa))
			return (EINTEGRITY);
		error = z_erofs_get_extent_compressedlen(&m, initial_lcn);
		if (error != 0)
			return (error);
	}

	if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) {
		map->m_algorithmformat =
		    (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0 ?
		    Z_EROFS_COMPRESSION_INTERLACED :
		    Z_EROFS_COMPRESSION_SHIFTED;
	} else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) {
		map->m_algorithmformat = vi->z_algorithmtype[1];
	} else {
		map->m_algorithmformat = vi->z_algorithmtype[0];
	}

	if ((flags & EROFS_GET_BLOCKS_FIEMAP) != 0 ||
	    ((flags & EROFS_GET_BLOCKS_READMORE) != 0 &&
	    (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA ||
	    map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE ||
	    map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) &&
	    map->m_llen >= sbi->block_size)) {
		error = z_erofs_get_extent_decompressedlen(&m);
		if (error == 0)
			map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED;
		return (error);
	}
	return (0);
}

static int
z_erofs_map_blocks_ext(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map, int flags)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct z_erofs_extent ext;
	unsigned int recsz, bmask;
	uint8_t fmt;
	uint64_t cluster_size, extent_idx, next, rounded_lend;
	erofs_off_t extent_pos, pos, table_pos;
	uint64_t fragmentoff, lend, l, r, mid, pa, la, lstart;
	bool interlaced, last;
	int error;

	(void)flags;
	interlaced =
	    (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) != 0;
	recsz = z_erofs_extent_recsize(vi->z_advise);
	error = z_erofs_extent_table_pos(vi, recsz, &table_pos);
	if (error != 0)
		return (error);
	pos = table_pos;
	bmask = sbi->block_size - 1;
	lend = vi->size;
	cluster_size = 1ULL << vi->z_lclusterbits;
	map->m_flags = 0;

	if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) {
		if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) {
			error = erofs_read_metadata(sbi, vi->nid, pos,
			    sizeof(uint64_t), &buf);
			if (error != 0)
				return (error);
			pa = le64dec(buf.data);
			erofs_put_metabuf(&buf);
			if (z_erofs_extent_add(pos, sizeof(uint64_t), &pos) != 0)
				return (EINTEGRITY);
			lstart = 0;
			extent_idx = 0;
		} else {
			lstart = rounddown2(map->m_la, cluster_size);
			extent_idx = lstart >> vi->z_lclusterbits;
			pa = EROFS_NULL_ADDR;
		}
		for (;;) {
			error = z_erofs_extent_record_pos(vi, pos, recsz,
			    extent_idx, &extent_pos);
			if (error != 0)
				return (error);
			error = z_erofs_read_extent(sbi, vi, extent_pos, recsz, &ext);
			if (error != 0)
				return (error);
			map->m_plen = le32toh(ext.plen);
			if (pa != EROFS_NULL_ADDR) {
				map->m_pa = pa;
				if (z_erofs_extent_add(pa,
				    map->m_plen & Z_EROFS_EXTENT_PLEN_MASK,
				    &next) != 0)
					return (EINTEGRITY);
				pa = next;
			} else {
				map->m_pa = le32toh(ext.pstart_lo);
			}
			if (extent_idx == UINT64_MAX)
				return (EINTEGRITY);
			extent_idx++;
			if (z_erofs_extent_add(lstart, cluster_size, &next) != 0)
				return (EINTEGRITY);
			lstart = next;
			if (lstart > map->m_la)
				break;
		}
		if (z_erofs_extent_roundup(lend, cluster_size, &rounded_lend) != 0)
			return (EINTEGRITY);
		last = lstart >= rounded_lend;
		lend = MIN(lstart, lend);
		lstart -= cluster_size;
	} else {
		lstart = lend;
		for (l = 0, r = vi->z_extents; l < r;) {
			mid = l + (r - l) / 2;
			error = z_erofs_extent_record_pos(vi, table_pos, recsz, mid,
			    &extent_pos);
			if (error != 0)
				return (error);
			error = z_erofs_read_extent(sbi, vi, extent_pos,
			    recsz, &ext);
			if (error != 0)
				return (error);
			la = z_erofs_extent_lstart(&ext, recsz);
			pa = le32toh(ext.pstart_lo) |
			    ((uint64_t)le32toh(ext.pstart_hi) << 32);
			if (la > map->m_la) {
				r = mid;
				if (la > lend)
					return (EINTEGRITY);
				lend = la;
			} else {
				l = mid + 1;
				if (map->m_la == la)
					r = MIN(l + 1, r);
				lstart = la;
				map->m_plen = le32toh(ext.plen);
				map->m_pa = pa;
			}
		}
		last = l >= vi->z_extents;
	}

	if (lstart < lend) {
		map->m_la = lstart;
		if (last &&
		    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) {
			map->m_flags = EROFS_MAP_FRAGMENT;
			if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) {
				error = z_erofs_fragment_offset(map->m_plen, map->m_pa,
				    &fragmentoff);
				if (error != 0)
					return (error);
				vi->z_fragmentoff = fragmentoff;
			} else {
				vi->z_fragmentoff = map->m_plen;
			}
		} else if ((map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) != 0) {
			map->m_flags = EROFS_MAP_MAPPED;
			if ((map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT) >
			    UINT8_MAX)
				return (EINTEGRITY);
			fmt = (uint8_t)(map->m_plen >>
			    Z_EROFS_EXTENT_PLEN_FMT_BIT);
			if ((map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) != 0)
				map->m_flags |= EROFS_MAP_PARTIAL_REF;
			map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK;
			if (fmt != 0)
				map->m_algorithmformat = (uint8_t)(fmt - 1);
			else if (interlaced &&
			    ((map->m_pa | map->m_plen) & bmask) == 0)
				map->m_algorithmformat =
				    Z_EROFS_COMPRESSION_INTERLACED;
			else
				map->m_algorithmformat =
				    Z_EROFS_COMPRESSION_SHIFTED;
		}
	}
	map->m_llen = lend - map->m_la;
	return (0);
}

int
z_erofs_fill_inode(struct erofs_sb_info *sbi, struct erofs_inode *vi)
{
	struct erofs_buf buf = EROFS_BUF_INITIALIZER;
	struct z_erofs_map_header *h;
	struct erofs_map_blocks map;
	uint64_t cluster_size, raw, rounded_size;
	erofs_off_t pos;
	unsigned int recsz;
	int error;

	if (vi->z_initialized)
		return (0);
	if (z_erofs_extent_add(vi->inode_off, vi->inode_isize, &pos) != 0 ||
	    z_erofs_extent_add(pos, vi->xattr_isize, &pos) != 0 ||
	    z_erofs_extent_roundup(pos, 8, &pos) != 0)
		return (EINTEGRITY);
	error = erofs_read_metadata(sbi, vi->nid, pos, sizeof(*h), &buf);
	if (error != 0)
		return (error);
	h = buf.data;
	if ((h->h_clusterbits & (1U << Z_EROFS_FRAGMENT_INODE_BIT)) != 0) {
		if (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0) {
			erofs_put_metabuf(&buf);
			return (EINTEGRITY);
		}
		raw = le64dec(h);
		vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
		vi->z_fragmentoff = raw ^ (1ULL << 63);
		vi->z_tailextent_headlcn = 0;
		vi->fragment = true;
		erofs_put_metabuf(&buf);
		vi->z_initialized = true;
		return (0);
	}

	vi->z_advise = le16toh(h->h_advise);
	vi->z_lclusterbits = sbi->blkszbits + (h->h_clusterbits & 15);
	if (vi->z_lclusterbits >= 31) {
		erofs_put_metabuf(&buf);
		return (EINTEGRITY);
	}
	if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL &&
	    (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0) {
		recsz = z_erofs_extent_recsize(vi->z_advise);
		if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) {
			cluster_size = 1ULL << vi->z_lclusterbits;
			if (z_erofs_extent_roundup(vi->size, cluster_size,
			    &rounded_size) != 0) {
				erofs_put_metabuf(&buf);
				return (EINTEGRITY);
			}
			vi->z_extents = rounded_size >> vi->z_lclusterbits;
		} else {
			vi->z_extents = le32toh(h->h_extents_lo) |
			    ((uint64_t)le16toh(h->h_extents_hi) << 32);
		}
		vi->fragment =
		    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0;
		erofs_put_metabuf(&buf);
		if (vi->fragment &&
		    (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0))
			return (EINTEGRITY);
		if (recsz > offsetof(struct z_erofs_extent, pstart_hi) &&
		    vi->z_extents == 0 && vi->size != 0)
			return (EINTEGRITY);
		error = z_erofs_validate_extent_table(sbi, vi, recsz);
		if (error != 0)
			return (error);
		vi->z_initialized = true;
		return (0);
	}
	vi->z_algorithmtype[0] = (uint8_t)(h->h_algorithmtype & 15);
	vi->z_algorithmtype[1] = (uint8_t)(h->h_algorithmtype >> 4);
	if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0)
		vi->z_fragmentoff = le32toh(h->h_fragmentoff);
	else if ((vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) != 0)
		vi->z_idata_size = le16toh(h->h_idata_size);
	erofs_put_metabuf(&buf);

	if (!erofs_sb_has_big_pcluster(sbi) &&
	    (vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
	    Z_EROFS_ADVISE_BIG_PCLUSTER_2)) != 0)
		return (EINTEGRITY);
	if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT &&
	    (((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) != 0) !=
	    ((vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2) != 0)))
		return (EINTEGRITY);
	if (vi->z_idata_size != 0 ||
	    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0) {
		if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0 &&
		    (!erofs_sb_has_fragments(sbi) || sbi->packed_nid == 0))
			return (EINTEGRITY);
		bzero(&map, sizeof(map));
		error = z_erofs_map_blocks_fo(sbi, vi, &map,
		    EROFS_GET_BLOCKS_FINDTAIL);
		if (error != 0)
			return (error);
	}
	vi->fragment =
	    (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) != 0;
	vi->z_initialized = true;
	return (0);
}

static int
z_erofs_map_sanity_check(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map)
{
	if ((map->m_flags & EROFS_MAP_FRAGMENT) != 0) {
		if ((map->m_flags & (EROFS_MAP_MAPPED | EROFS_MAP_META)) != 0 ||
		    sbi->packed_inode == NULL || sbi->packed_inode->nid == vi->nid ||
		    vi->z_fragmentoff > sbi->packed_inode->size ||
		    map->m_llen > sbi->packed_inode->size - vi->z_fragmentoff)
			return (EINTEGRITY);
		return (0);
	}
	if ((map->m_flags & EROFS_MAP_MAPPED) == 0)
		return (0);
	if (map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)
		return (EOPNOTSUPP);
	if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) {
		if ((sbi->available_compr_algs &
		    (1U << map->m_algorithmformat)) == 0)
			return (EINTEGRITY);
		if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen)
			return (EINTEGRITY);
	} else if (map->m_llen > map->m_plen) {
		return (EINTEGRITY);
	}
	if (map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE ||
	    map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE)
		return (EOPNOTSUPP);
	if ((map->m_flags & EROFS_MAP_META) != 0)
		return (0);
	if (z_erofs_physical_end(map->m_pa, map->m_plen,
	    sbi->block_size) != 0)
		return (EINTEGRITY);
	(void)vi;
	return (0);
}

int
z_erofs_map_blocks(struct erofs_sb_info *sbi, struct erofs_inode *vi,
    struct erofs_map_blocks *map)
{
	int error;

	if (map->m_la >= vi->size) {
		error = z_erofs_post_eof_len(map->m_la, vi->size,
		    &map->m_llen);
		if (error != 0)
			return (error);
		map->m_la = vi->size;
		map->m_flags = 0;
		return (0);
	}
	error = z_erofs_fill_inode(sbi, vi);
	if (error == 0) {
		if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL &&
		    (vi->z_advise & Z_EROFS_ADVISE_EXTENTS) != 0)
			error = z_erofs_map_blocks_ext(sbi, vi, map,
			    EROFS_GET_BLOCKS_FIEMAP);
		else
			error = z_erofs_map_blocks_fo(sbi, vi, map,
			    EROFS_GET_BLOCKS_FIEMAP);
	}
	if (error == 0)
		error = z_erofs_map_sanity_check(sbi, vi, map);
	if (error != 0)
		map->m_llen = 0;
	return (error);
}