Skip to content

Commit 80a99b8

Browse files
robnrobn
committed
zio: make io_bp_orig a separate allocation
We only need a copy if we would modify the original and might need to revert, which only applies to non-vdev writes. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris <[email protected]>
1 parent 96ce0b6 commit 80a99b8

File tree

6 files changed

+31
-24
lines changed

6 files changed

+31
-24
lines changed

‎include/sys/zio.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ struct zio {
498498
zio_done_func_t *io_done;
499499
void *io_private;
500500
int64_t io_prev_space_delta; /* DMU private */
501-
blkptr_t io_bp_orig;
501+
blkptr_t *io_bp_orig;
502502
/* io_lsize != io_orig_size iff this is a raw write */
503503
uint64_t io_lsize;
504504

‎module/zfs/arc.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6985,7 +6985,7 @@ arc_write_done(zio_t *zio)
69856985
* buffers from the hash table when we arc_free().
69866986
*/
69876987
if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
6988-
if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6988+
if (!BP_EQUAL(zio->io_bp_orig, zio->io_bp))
69896989
panic("bad overwrite, hdr=%p exists=%p",
69906990
(void *)hdr, (void *)exists);
69916991
ASSERT(zfs_refcount_is_zero(
@@ -6998,7 +6998,7 @@ arc_write_done(zio_t *zio)
69986998
} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
69996999
/* nopwrite */
70007000
ASSERT(zio->io_prop.zp_nopwrite);
7001-
if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
7001+
if (!BP_EQUAL(zio->io_bp_orig, zio->io_bp))
70027002
panic("bad nopwrite, hdr=%p exists=%p",
70037003
(void *)hdr, (void *)exists);
70047004
} else {

‎module/zfs/dbuf.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4667,10 +4667,10 @@ dbuf_lightweight_done(zio_t *zio)
46674667
dmu_tx_t *tx = os->os_synctx;
46684668

46694669
if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) {
4670-
ASSERT(BP_EQUAL(zio->io_bp, &zio->io_bp_orig));
4670+
ASSERT(BP_EQUAL(zio->io_bp, zio->io_bp_orig));
46714671
} else {
46724672
dsl_dataset_t *ds = os->os_dsl_dataset;
4673-
(void) dsl_dataset_block_kill(ds, &zio->io_bp_orig, tx, B_TRUE);
4673+
(void) dsl_dataset_block_kill(ds, zio->io_bp_orig, tx, B_TRUE);
46744674
dsl_dataset_block_born(ds, zio->io_bp, tx);
46754675
}
46764676

@@ -4929,7 +4929,7 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
49294929
dmu_buf_impl_t *db = vdb;
49304930
dnode_t *dn;
49314931
blkptr_t *bp = zio->io_bp;
4932-
blkptr_t *bp_orig = &zio->io_bp_orig;
4932+
blkptr_t *bp_orig = zio->io_bp_orig;
49334933
spa_t *spa = zio->io_spa;
49344934
int64_t delta;
49354935
uint64_t fill = 0;
@@ -5077,7 +5077,7 @@ dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
50775077
{
50785078
(void) buf;
50795079
dmu_buf_impl_t *db = vdb;
5080-
blkptr_t *bp_orig = &zio->io_bp_orig;
5080+
blkptr_t *bp_orig = zio->io_bp_orig;
50815081
blkptr_t *bp = db->db_blkptr;
50825082
objset_t *os = db->db_objset;
50835083
dmu_tx_t *tx = os->os_synctx;

‎module/zfs/dmu.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1908,7 +1908,7 @@ dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
19081908
dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
19091909
if (dr->dt.dl.dr_nopwrite) {
19101910
blkptr_t *bp = zio->io_bp;
1911-
blkptr_t *bp_orig = &zio->io_bp_orig;
1911+
blkptr_t *bp_orig = zio->io_bp_orig;
19121912
uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
19131913

19141914
ASSERT(BP_EQUAL(bp, bp_orig));
@@ -1963,7 +1963,7 @@ dmu_sync_late_arrival_done(zio_t *zio)
19631963
zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
19641964

19651965
if (!BP_IS_HOLE(bp)) {
1966-
blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig;
1966+
blkptr_t *bp_orig __maybe_unused = zio->io_bp_orig;
19671967
ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE));
19681968
ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
19691969
ASSERT(BP_GET_BIRTH(zio->io_bp) == zio->io_txg);

‎module/zfs/dmu_objset.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1528,7 +1528,7 @@ dmu_objset_write_done(zio_t *zio, arc_buf_t *abuf, void *arg)
15281528
{
15291529
(void) abuf;
15301530
blkptr_t *bp = zio->io_bp;
1531-
blkptr_t *bp_orig = &zio->io_bp_orig;
1531+
blkptr_t *bp_orig = zio->io_bp_orig;
15321532
objset_t *os = arg;
15331533

15341534
if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {

‎module/zfs/zio.c‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -999,7 +999,12 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
999999
} else {
10001000
zio->io_bp = (blkptr_t *)bp;
10011001
}
1002-
zio->io_bp_orig = *bp;
1002+
if (type == ZIO_TYPE_WRITE &&
1003+
zio->io_child_type != ZIO_CHILD_VDEV) {
1004+
zio->io_bp_orig =
1005+
kmem_cache_alloc(zio_bp_cache, KM_SLEEP);
1006+
*zio->io_bp_orig = *bp;
1007+
}
10031008
if (zio->io_child_type == ZIO_CHILD_LOGICAL)
10041009
zio->io_logical = zio;
10051010
if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
@@ -1058,6 +1063,8 @@ zio_destroy(zio_t *zio)
10581063
cv_destroy(&zio->io_cv);
10591064
if (zio->io_bp_copy != NULL)
10601065
kmem_cache_free(zio_bp_cache, zio->io_bp_copy);
1066+
if (zio->io_bp_orig != NULL)
1067+
kmem_cache_free(zio_bp_cache, zio->io_bp_orig);
10611068
kmem_cache_free(zio_cache, zio);
10621069
}
10631070

@@ -1927,7 +1934,7 @@ zio_write_bp_init(zio_t *zio)
19271934
* it as a regular write I/O.
19281935
*/
19291936
zio->io_bp_override = NULL;
1930-
*bp = zio->io_bp_orig;
1937+
*bp = *zio->io_bp_orig;
19311938
zio->io_pipeline = zio->io_orig_pipeline;
19321939
}
19331940

@@ -2059,7 +2066,7 @@ zio_write_compress(zio_t *zio)
20592066
* it as a regular write I/O.
20602067
*/
20612068
zio->io_bp_override = NULL;
2062-
*bp = zio->io_bp_orig;
2069+
*bp = *zio->io_bp_orig;
20632070
zio->io_pipeline = zio->io_orig_pipeline;
20642071

20652072
} else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 &&
@@ -2121,7 +2128,7 @@ zio_write_compress(zio_t *zio)
21212128
}
21222129

21232130
if (psize == 0) {
2124-
if (BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig) != 0 &&
2131+
if (BP_GET_LOGICAL_BIRTH(zio->io_bp_orig) != 0 &&
21252132
spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
21262133
BP_SET_LSIZE(bp, lsize);
21272134
BP_SET_TYPE(bp, zp->zp_type);
@@ -3136,7 +3143,7 @@ zio_write_gang_member_ready(zio_t *zio)
31363143
* If we're getting direct-invoked from zio_write_gang_block(),
31373144
* the bp_orig will be set.
31383145
*/
3139-
ASSERT(BP_IS_HOLE(&zio->io_bp_orig) ||
3146+
ASSERT(BP_IS_HOLE(zio->io_bp_orig) ||
31403147
zio->io_flags & ZIO_FLAG_PREALLOCATED);
31413148

31423149
ASSERT(zio->io_child_type == ZIO_CHILD_GANG);
@@ -3316,7 +3323,7 @@ zio_write_gang_block(zio_t *pio, metaslab_class_t *mc)
33163323
if (allocated) {
33173324
metaslab_trace_move(&cio_list, &cio->io_alloc_list);
33183325
metaslab_group_alloc_increment_all(spa,
3319-
&cio->io_bp_orig, zio->io_allocator, flags, psize,
3326+
cio->io_bp_orig, zio->io_allocator, flags, psize,
33203327
cio);
33213328
}
33223329
/*
@@ -3385,7 +3392,7 @@ static zio_t *
33853392
zio_nop_write(zio_t *zio)
33863393
{
33873394
blkptr_t *bp = zio->io_bp;
3388-
blkptr_t *bp_orig = &zio->io_bp_orig;
3395+
blkptr_t *bp_orig = zio->io_bp_orig;
33893396
zio_prop_t *zp = &zio->io_prop;
33903397

33913398
ASSERT(BP_IS_HOLE(bp));
@@ -3957,9 +3964,9 @@ zio_ddt_write(zio_t *zio)
39573964
*/
39583965
if (zp->zp_rewrite) {
39593966
uint64_t orig_logical_birth =
3960-
BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig);
3967+
BP_GET_LOGICAL_BIRTH(zio->io_bp_orig);
39613968
ddt_bp_fill(ddp, v, bp, orig_logical_birth);
3962-
if (BP_EQUAL(bp, &zio->io_bp_orig)) {
3969+
if (BP_EQUAL(bp, zio->io_bp_orig)) {
39633970
/* We can skip accounting. */
39643971
zio->io_flags |= ZIO_FLAG_NOPWRITE;
39653972
ddt_exit(ddt);
@@ -4267,12 +4274,12 @@ zio_dva_allocate(zio_t *zio)
42674274
}
42684275
if (zio->io_flags & ZIO_FLAG_PREALLOCATED) {
42694276
ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_GANG);
4270-
memcpy(zio->io_bp->blk_dva, zio->io_bp_orig.blk_dva,
4277+
memcpy(zio->io_bp->blk_dva, zio->io_bp_orig->blk_dva,
42714278
3 * sizeof (dva_t));
42724279
BP_SET_LOGICAL_BIRTH(zio->io_bp,
4273-
BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig));
4280+
BP_GET_LOGICAL_BIRTH(zio->io_bp_orig));
42744281
BP_SET_PHYSICAL_BIRTH(zio->io_bp,
4275-
BP_GET_RAW_PHYSICAL_BIRTH(&zio->io_bp_orig));
4282+
BP_GET_RAW_PHYSICAL_BIRTH(zio->io_bp_orig));
42764283
return (zio);
42774284
}
42784285

@@ -4404,7 +4411,7 @@ zio_dva_allocate(zio_t *zio)
44044411
* For rewrite operations, preserve the logical birth time
44054412
* but set the physical birth time to the current txg.
44064413
*/
4407-
uint64_t logical_birth = BP_GET_LOGICAL_BIRTH(&zio->io_bp_orig);
4414+
uint64_t logical_birth = BP_GET_LOGICAL_BIRTH(zio->io_bp_orig);
44084415
ASSERT3U(logical_birth, <=, zio->io_txg);
44094416
BP_SET_BIRTH(zio->io_bp, logical_birth, zio->io_txg);
44104417
BP_SET_REWRITE(zio->io_bp, 1);
@@ -5497,7 +5504,7 @@ zio_done(zio_t *zio)
54975504
BP_GET_NDVAS(zio->io_bp)));
54985505
}
54995506
if (zio->io_flags & ZIO_FLAG_NOPWRITE)
5500-
VERIFY(BP_EQUAL(zio->io_bp, &zio->io_bp_orig));
5507+
VERIFY(BP_EQUAL(zio->io_bp, zio->io_bp_orig));
55015508
}
55025509

55035510
/*

0 commit comments

Comments
 (0)