From 6b8f798774a28a2c1b0447e1c48bb6b0b64cf7cc Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Thu, 25 Jun 2026 12:13:14 -0400 Subject: [PATCH] Avoid more abd_t allocations in RAIDZ/dRAID RAIDZ/dRAID allocate rc_abdstruct for every column, but used it only for data columns, while parity columns allocated additional abd_t's on top of that. This change introduces abd_alloc_linear_struct(), abd_alloc_gang_struct() and abd_get_zeros_struct() to the ABD API, mirroring the existing abd_get_offset_struct() pattern, and uses them to use rc_abdstruct in remaining column cases. Reviewed-by: Brian Behlendorf Reviewed-by: Rob Norris Signed-off-by: Alexander Motin Closes #18668 --- include/sys/abd.h | 3 +++ module/zfs/abd.c | 45 ++++++++++++++++++++++++++++++++++------- module/zfs/vdev_draid.c | 10 +++++---- module/zfs/vdev_raidz.c | 21 ++++++++++--------- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/include/sys/abd.h b/include/sys/abd.h index 41cdb2615427..62c8c0c4e539 100644 --- a/include/sys/abd.h +++ b/include/sys/abd.h @@ -81,8 +81,10 @@ __attribute__((malloc)) abd_t *abd_alloc(size_t, boolean_t); __attribute__((malloc)) abd_t *abd_alloc_linear(size_t, boolean_t); +abd_t *abd_alloc_linear_struct(abd_t *, size_t, boolean_t); __attribute__((malloc)) abd_t *abd_alloc_gang(void); +abd_t *abd_alloc_gang_struct(abd_t *); __attribute__((malloc)) abd_t *abd_alloc_for_io(size_t, boolean_t); __attribute__((malloc)) @@ -94,6 +96,7 @@ abd_t *abd_get_offset(abd_t *, size_t); abd_t *abd_get_offset_size(abd_t *, size_t, size_t); abd_t *abd_get_offset_struct(abd_t *, abd_t *, size_t, size_t); abd_t *abd_get_zeros(size_t); +abd_t *abd_get_zeros_struct(abd_t *, size_t); abd_t *abd_get_from_buf(void *, size_t); abd_t *abd_get_from_buf_struct(abd_t *, void *, size_t); void abd_cache_reap_now(void); diff --git a/module/zfs/abd.c b/module/zfs/abd.c index 7ea07c418300..cb575e21acd7 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -212,11 +212,9 @@ abd_alloc(size_t size, boolean_t is_metadata) * buffer. Only use this when it would be very annoying to write your ABD * consumer with a scattered ABD. */ -abd_t * -abd_alloc_linear(size_t size, boolean_t is_metadata) +static abd_t * +abd_alloc_linear_impl(abd_t *abd, size_t size, boolean_t is_metadata) { - abd_t *abd = abd_alloc_struct(0); - VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER; @@ -236,6 +234,19 @@ abd_alloc_linear(size_t size, boolean_t is_metadata) return (abd); } +abd_t * +abd_alloc_linear(size_t size, boolean_t is_metadata) +{ + return (abd_alloc_linear_impl(abd_alloc_struct(0), size, is_metadata)); +} + +abd_t * +abd_alloc_linear_struct(abd_t *abd, size_t size, boolean_t is_metadata) +{ + abd_init_struct(abd); + return (abd_alloc_linear_impl(abd, size, is_metadata)); +} + static void abd_free_linear(abd_t *abd) { @@ -349,16 +360,28 @@ abd_alloc_sametype(abd_t *sabd, size_t size) * to "chain" scatter/gather lists together when constructing aggregated * IO's. To free this abd, abd_free() must be called. */ -abd_t * -abd_alloc_gang(void) +static abd_t * +abd_alloc_gang_impl(abd_t *abd) { - abd_t *abd = abd_alloc_struct(0); abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER; list_create(&ABD_GANG(abd).abd_gang_chain, sizeof (abd_t), offsetof(abd_t, abd_gang_link)); return (abd); } +abd_t * +abd_alloc_gang(void) +{ + return (abd_alloc_gang_impl(abd_alloc_struct(0))); +} + +abd_t * +abd_alloc_gang_struct(abd_t *abd) +{ + abd_init_struct(abd); + return (abd_alloc_gang_impl(abd)); +} + /* * Add a child gang ABD to a parent gang ABDs chained list. */ @@ -624,6 +647,14 @@ abd_get_zeros(size_t size) return (abd_get_offset_size(abd_zero_scatter, 0, size)); } +abd_t * +abd_get_zeros_struct(abd_t *abd, size_t size) +{ + ASSERT3P(abd_zero_scatter, !=, NULL); + ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); + return (abd_get_offset_struct(abd, abd_zero_scatter, 0, size)); +} + /* * Create a linear ABD for an existing buf. */ diff --git a/module/zfs/vdev_draid.c b/module/zfs/vdev_draid.c index 8f556b868784..019c20f3f99d 100644 --- a/module/zfs/vdev_draid.c +++ b/module/zfs/vdev_draid.c @@ -798,7 +798,8 @@ vdev_draid_map_alloc_write(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) if (rc->rc_size == 0) { /* empty data column (small write), add a skip sector */ ASSERT3U(skip_size, ==, parity_size); - rc->rc_abd = abd_get_zeros(skip_size); + rc->rc_abd = abd_get_zeros_struct(&rc->rc_abdstruct, + skip_size); } else if (rc->rc_size == parity_size) { /* this is a "big column" */ rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct, @@ -806,7 +807,7 @@ vdev_draid_map_alloc_write(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) } else { /* short data column, add a skip sector */ ASSERT3U(rc->rc_size + skip_size, ==, parity_size); - rc->rc_abd = abd_alloc_gang(); + rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); abd_gang_add(rc->rc_abd, abd_get_offset_size( zio->io_abd, abd_off, rc->rc_size), B_TRUE); abd_gang_add(rc->rc_abd, abd_get_zeros(skip_size), @@ -863,7 +864,7 @@ vdev_draid_map_alloc_scrub(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) /* short data column, add a skip sector */ ASSERT3U(rc->rc_size + skip_size, ==, parity_size); ASSERT3U(rr->rr_nempty, !=, 0); - rc->rc_abd = abd_alloc_gang(); + rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); abd_gang_add(rc->rc_abd, abd_get_offset_size( zio->io_abd, abd_off, rc->rc_size), B_TRUE); abd_gang_add(rc->rc_abd, abd_get_offset_size( @@ -1240,7 +1241,8 @@ vdev_draid_map_alloc_row(zio_t *zio, raidz_row_t **rrp, uint64_t io_offset, /* Allocate buffers for the parity columns */ for (uint64_t c = 0; c < rr->rr_firstdatacol; c++) { raidz_col_t *rc = &rr->rr_col[c]; - rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); } /* diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index d126cdb71b2b..ab9f62c3c2d0 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -571,12 +571,13 @@ vdev_raidz_map_alloc_write(zio_t *zio, raidz_map_t *rm, uint64_t ashift) * VDEV queue locks (vq_lock). */ if (c < nwrapped) { - rc->rc_abd = abd_alloc_linear( + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, rc->rc_size + (1ULL << ashift), B_FALSE); abd_zero_off(rc->rc_abd, rc->rc_size, 1ULL << ashift); skipped++; } else { - rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); } } @@ -624,9 +625,11 @@ vdev_raidz_map_alloc_read(zio_t *zio, raidz_map_t *rm) ASSERT3U(rm->rm_nrows, ==, 1); /* Allocate buffers for the parity columns */ - for (c = 0; c < rr->rr_firstdatacol; c++) - rr->rr_col[c].rc_abd = - abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE); + for (c = 0; c < rr->rr_firstdatacol; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); + } for (uint64_t off = 0; c < rr->rr_cols; c++) { raidz_col_t *rc = &rr->rr_col[c]; @@ -1071,8 +1074,8 @@ vdev_raidz_map_alloc_expanded(zio_t *zio, continue; prc->rc_abd = - abd_alloc_linear(rm->rm_phys_col[i].rc_size, - B_FALSE); + abd_alloc_linear_struct(&prc->rc_abdstruct, + prc->rc_size, B_FALSE); } /* @@ -1100,8 +1103,8 @@ vdev_raidz_map_alloc_expanded(zio_t *zio, for (int c = 0; c < rr->rr_firstdatacol; c++) { raidz_col_t *rc = &rr->rr_col[c]; rc->rc_abd = - abd_alloc_linear(rc->rc_size, - B_TRUE); + abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_TRUE); } } }