Skip to content

Commit 1737faa

Browse files
snajpasnajpa
committed
Add dbuf locking for remaining db_data sites
Signed-off-by: Pavel Snajdr <[email protected]>
1 parent 1af41fd commit 1737faa

20 files changed

+168
-10
lines changed

‎module/zfs/bpobj.c

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
#include <sys/bpobj.h>
2929
#include <sys/zfs_context.h>
30+
#include <sys/dbuf.h>
3031
#include <sys/zfs_refcount.h>
3132
#include <sys/dsl_pool.h>
3233
#include <sys/zfeature.h>
@@ -131,8 +132,10 @@ bpobj_free(objset_t *os, uint64_t obj, dmu_tx_t *tx)
131132
ASSERT3U(offset, >=, dbuf->db_offset);
132133
ASSERT3U(offset, <, dbuf->db_offset + dbuf->db_size);
133134

135+
mutex_enter(&((dmu_buf_impl_t *)dbuf)->db_mtx);
134136
objarray = dbuf->db_data;
135137
bpobj_free(os, objarray[blkoff], tx);
138+
mutex_exit(&((dmu_buf_impl_t *)dbuf)->db_mtx);
136139
}
137140
if (dbuf) {
138141
dmu_buf_rele(dbuf, FTAG);
@@ -176,7 +179,9 @@ bpobj_open(bpobj_t *bpo, objset_t *os, uint64_t object)
176179
bpo->bpo_havecomp = (doi.doi_bonus_size > BPOBJ_SIZE_V0);
177180
bpo->bpo_havesubobj = (doi.doi_bonus_size > BPOBJ_SIZE_V1);
178181
bpo->bpo_havefreed = (doi.doi_bonus_size > BPOBJ_SIZE_V2);
182+
mutex_enter(&((dmu_buf_impl_t *)bpo->bpo_dbuf)->db_mtx);
179183
bpo->bpo_phys = bpo->bpo_dbuf->db_data;
184+
mutex_exit(&((dmu_buf_impl_t *)bpo->bpo_dbuf)->db_mtx);
180185
return (0);
181186
}
182187

@@ -318,13 +323,16 @@ bpobj_iterate_blkptrs(bpobj_info_t *bpi, bpobj_itor_t func, void *arg,
318323
ASSERT3U(offset, >=, dbuf->db_offset);
319324
ASSERT3U(offset, <, dbuf->db_offset + dbuf->db_size);
320325

326+
mutex_enter(&((dmu_buf_impl_t *)dbuf)->db_mtx);
321327
blkptr_t *bparray = dbuf->db_data;
322328
blkptr_t *bp = &bparray[blkoff];
323329

324330
boolean_t bp_freed = BP_GET_FREE(bp);
325331
err = func(arg, bp, bp_freed, tx);
326-
if (err)
332+
if (err) {
333+
mutex_exit(&((dmu_buf_impl_t *)dbuf)->db_mtx);
327334
break;
335+
}
328336

329337
if (free) {
330338
int sign = bp_freed ? -1 : +1;
@@ -340,6 +348,7 @@ bpobj_iterate_blkptrs(bpobj_info_t *bpi, bpobj_itor_t func, void *arg,
340348
bpo->bpo_phys->bpo_num_freed--;
341349
ASSERT3S(bpo->bpo_phys->bpo_num_freed, >=, 0);
342350
}
351+
mutex_exit(&((dmu_buf_impl_t *)dbuf)->db_mtx);
343352
}
344353
}
345354
if (free) {
@@ -750,9 +759,11 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)
750759
DMU_OT_BPOBJ_SUBOBJ, SPA_OLD_MAXBLOCKSIZE,
751760
DMU_OT_NONE, 0, tx);
752761
}
762+
mutex_enter(&((dmu_buf_impl_t *)subdb)->db_mtx);
753763
dmu_write(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs,
754764
bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj),
755765
numsubsub * sizeof (subobj), subdb->db_data, tx);
766+
mutex_exit(&((dmu_buf_impl_t *)subdb)->db_mtx);
756767
dmu_buf_rele(subdb, FTAG);
757768
bpo->bpo_phys->bpo_num_subobjs += numsubsub;
758769

@@ -774,10 +785,12 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)
774785
* to write more data than we have in our buffer.
775786
*/
776787
VERIFY3U(bps->db_size, >=, numbps * sizeof (blkptr_t));
788+
mutex_enter(&((dmu_buf_impl_t *)bps)->db_mtx);
777789
dmu_write(bpo->bpo_os, bpo->bpo_object,
778790
bpo->bpo_phys->bpo_num_blkptrs * sizeof (blkptr_t),
779791
numbps * sizeof (blkptr_t),
780792
bps->db_data, tx);
793+
mutex_exit(&((dmu_buf_impl_t *)bps)->db_mtx);
781794
dmu_buf_rele(bps, FTAG);
782795
bpo->bpo_phys->bpo_num_blkptrs += numbps;
783796

@@ -920,8 +933,10 @@ bpobj_enqueue(bpobj_t *bpo, const blkptr_t *bp, boolean_t bp_freed,
920933
}
921934

922935
dmu_buf_will_dirty(bpo->bpo_cached_dbuf, tx);
936+
mutex_enter(&((dmu_buf_impl_t *)bpo->bpo_cached_dbuf)->db_mtx);
923937
bparray = bpo->bpo_cached_dbuf->db_data;
924938
bparray[blkoff] = stored_bp;
939+
mutex_exit(&((dmu_buf_impl_t *)bpo->bpo_cached_dbuf)->db_mtx);
925940

926941
dmu_buf_will_dirty(bpo->bpo_dbuf, tx);
927942
bpo->bpo_phys->bpo_num_blkptrs++;

‎module/zfs/bptree.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <sys/arc.h>
2828
#include <sys/bptree.h>
2929
#include <sys/dmu.h>
30+
#include <sys/dbuf.h>
3031
#include <sys/dmu_objset.h>
3132
#include <sys/dmu_tx.h>
3233
#include <sys/dmu_traverse.h>
@@ -74,12 +75,14 @@ bptree_alloc(objset_t *os, dmu_tx_t *tx)
7475
*/
7576
VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
7677
dmu_buf_will_dirty(db, tx);
78+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
7779
bt = db->db_data;
7880
bt->bt_begin = 0;
7981
bt->bt_end = 0;
8082
bt->bt_bytes = 0;
8183
bt->bt_comp = 0;
8284
bt->bt_uncomp = 0;
85+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
8386
dmu_buf_rele(db, FTAG);
8487

8588
return (obj);
@@ -92,11 +95,13 @@ bptree_free(objset_t *os, uint64_t obj, dmu_tx_t *tx)
9295
bptree_phys_t *bt;
9396

9497
VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
98+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
9599
bt = db->db_data;
96100
ASSERT3U(bt->bt_begin, ==, bt->bt_end);
97101
ASSERT0(bt->bt_bytes);
98102
ASSERT0(bt->bt_comp);
99103
ASSERT0(bt->bt_uncomp);
104+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
100105
dmu_buf_rele(db, FTAG);
101106

102107
return (dmu_object_free(os, obj, tx));
@@ -110,8 +115,10 @@ bptree_is_empty(objset_t *os, uint64_t obj)
110115
boolean_t rv;
111116

112117
VERIFY0(dmu_bonus_hold(os, obj, FTAG, &db));
118+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
113119
bt = db->db_data;
114120
rv = (bt->bt_begin == bt->bt_end);
121+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
115122
dmu_buf_rele(db, FTAG);
116123
return (rv);
117124
}
@@ -132,6 +139,7 @@ bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
132139
ASSERT(dmu_tx_is_syncing(tx));
133140

134141
VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
142+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
135143
bt = db->db_data;
136144

137145
bte = kmem_zalloc(sizeof (*bte), KM_SLEEP);
@@ -145,6 +153,7 @@ bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg,
145153
bt->bt_bytes += bytes;
146154
bt->bt_comp += comp;
147155
bt->bt_uncomp += uncomp;
156+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
148157
dmu_buf_rele(db, FTAG);
149158
}
150159

@@ -204,6 +213,7 @@ bptree_iterate(objset_t *os, uint64_t obj, boolean_t free, bptree_itor_t func,
204213
if (free)
205214
dmu_buf_will_dirty(db, tx);
206215

216+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
207217
ba.ba_phys = db->db_data;
208218
ba.ba_free = free;
209219
ba.ba_func = func;

‎module/zfs/brt.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include <sys/ddt.h>
3434
#include <sys/bitmap.h>
3535
#include <sys/zap.h>
36+
#include <sys/dbuf.h>
3637
#include <sys/dmu_tx.h>
3738
#include <sys/arc.h>
3839
#include <sys/dsl_pool.h>
@@ -552,12 +553,14 @@ brt_vdev_load(spa_t *spa, brt_vdev_t *brtvd)
552553
if (error != 0)
553554
return (error);
554555

556+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
555557
bvphys = db->db_data;
556558
if (spa->spa_brt_rangesize == 0) {
557559
spa->spa_brt_rangesize = bvphys->bvp_rangesize;
558560
} else {
559561
ASSERT3U(spa->spa_brt_rangesize, ==, bvphys->bvp_rangesize);
560562
}
563+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
561564

562565
brt_vdev_realloc(spa, brtvd);
563566

@@ -802,6 +805,7 @@ brt_vdev_sync(spa_t *spa, brt_vdev_t *brtvd, dmu_tx_t *tx)
802805
}
803806

804807
dmu_buf_will_dirty(db, tx);
808+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
805809
bvphys = db->db_data;
806810
bvphys->bvp_mos_entries = brtvd->bv_mos_entries;
807811
bvphys->bvp_size = brtvd->bv_size;
@@ -814,6 +818,7 @@ brt_vdev_sync(spa_t *spa, brt_vdev_t *brtvd, dmu_tx_t *tx)
814818
bvphys->bvp_rangesize = spa->spa_brt_rangesize;
815819
bvphys->bvp_usedspace = brtvd->bv_usedspace;
816820
bvphys->bvp_savedspace = brtvd->bv_savedspace;
821+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
817822
dmu_buf_rele(db, FTAG);
818823

819824
brtvd->bv_meta_dirty = FALSE;

‎module/zfs/ddt_log.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,14 @@ ddt_log_update_header(ddt_t *ddt, ddt_log_t *ddl, dmu_tx_t *tx)
102102
VERIFY0(dmu_bonus_hold(ddt->ddt_os, ddl->ddl_object, FTAG, &db));
103103
dmu_buf_will_dirty(db, tx);
104104

105+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
105106
ddt_log_header_t *hdr = (ddt_log_header_t *)db->db_data;
106107
DLH_SET_VERSION(hdr, 1);
107108
DLH_SET_FLAGS(hdr, ddl->ddl_flags);
108109
hdr->dlh_length = ddl->ddl_length;
109110
hdr->dlh_first_txg = ddl->ddl_first_txg;
110111
hdr->dlh_checkpoint = ddl->ddl_checkpoint;
112+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
111113

112114
dmu_buf_rele(db, FTAG);
113115
}
@@ -290,10 +292,13 @@ ddt_log_entry(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, ddt_log_update_t *dlu)
290292
*/
291293
if (dlu->dlu_offset == 0) {
292294
dmu_buf_will_fill(db, dlu->dlu_tx, B_FALSE);
295+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
293296
memset(db->db_data, 0, db->db_size);
297+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
294298
}
295299

296300
/* Create the log record directly in the buffer */
301+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
297302
ddt_log_record_t *dlr = (db->db_data + dlu->dlu_offset);
298303
DLR_SET_TYPE(dlr, DLR_ENTRY);
299304
DLR_SET_RECLEN(dlr, dlu->dlu_reclen);
@@ -307,6 +312,7 @@ ddt_log_entry(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, ddt_log_update_t *dlu)
307312

308313
/* Advance offset for next record. */
309314
dlu->dlu_offset += dlu->dlu_reclen;
315+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
310316
}
311317

312318
void
@@ -563,7 +569,9 @@ ddt_log_load_one(ddt_t *ddt, uint_t n)
563569
dnode_rele(dn, FTAG);
564570
return (err);
565571
}
572+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
566573
memcpy(&hdr, db->db_data, sizeof (ddt_log_header_t));
574+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
567575
dmu_buf_rele(db, FTAG);
568576

569577
if (DLH_GET_VERSION(&hdr) != 1) {
@@ -599,9 +607,10 @@ ddt_log_load_one(ddt_t *ddt, uint_t n)
599607
}
600608

601609
uint64_t boffset = 0;
610+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
602611
while (boffset < db->db_size) {
603612
ddt_log_record_t *dlr =
604-
(ddt_log_record_t *)(db->db_data + boffset);
613+
(ddt_log_record_t *)(db->db_data + boffset);
605614

606615
/* Partially-filled block, skip the rest */
607616
if (DLR_GET_TYPE(dlr) == DLR_INVALID)
@@ -610,20 +619,21 @@ ddt_log_load_one(ddt_t *ddt, uint_t n)
610619
switch (DLR_GET_TYPE(dlr)) {
611620
case DLR_ENTRY:
612621
ddt_log_load_entry(ddt, ddl, dlr,
613-
checkpoint);
622+
checkpoint);
614623
break;
615624

616625
default:
626+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
617627
dmu_buf_rele(db, FTAG);
618628
dnode_rele(dn, FTAG);
619629
ddt_log_empty(ddt, ddl);
620630
return (SET_ERROR(EINVAL));
621631
}
622632

623633
boffset += DLR_GET_RECLEN(dlr);
634+
dmu_buf_rele(db, FTAG);
624635
}
625-
626-
dmu_buf_rele(db, FTAG);
636+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
627637
}
628638
}
629639

‎module/zfs/dmu.c

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,8 +1222,11 @@ dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size,
12221222
bufoff = offset - db->db_offset;
12231223
tocpy = MIN(db->db_size - bufoff, size);
12241224

1225+
/* Ensure stable copy of db_data */
1226+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
12251227
ASSERT(db->db_data != NULL);
1226-
(void) memcpy(buf, (char *)db->db_data + bufoff, tocpy);
1228+
memcpy(buf, (char *)db->db_data + bufoff, tocpy);
1229+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
12271230

12281231
offset += tocpy;
12291232
size -= tocpy;
@@ -1280,8 +1283,11 @@ dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size,
12801283
else
12811284
dmu_buf_will_dirty(db, tx);
12821285

1286+
/* Serialise modifications to db_data */
1287+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
12831288
ASSERT(db->db_data != NULL);
1284-
(void) memcpy((char *)db->db_data + bufoff, buf, tocpy);
1289+
memcpy((char *)db->db_data + bufoff, buf, tocpy);
1290+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
12851291

12861292
if (tocpy == db->db_size)
12871293
dmu_buf_fill_done(db, tx, B_FALSE);
@@ -1429,9 +1435,12 @@ dmu_read_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size)
14291435
bufoff = zfs_uio_offset(uio) - db->db_offset;
14301436
tocpy = MIN(db->db_size - bufoff, size);
14311437

1438+
/* Hold db_mtx during fault move */
1439+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
14321440
ASSERT(db->db_data != NULL);
14331441
err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy,
14341442
UIO_READ, uio);
1443+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
14351444

14361445
if (err)
14371446
break;
@@ -1554,9 +1563,12 @@ dmu_write_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size, dmu_tx_t *tx)
15541563
else
15551564
dmu_buf_will_dirty(db, tx);
15561565

1566+
/* Protect db_data during write fault move */
1567+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
15571568
ASSERT(db->db_data != NULL);
15581569
err = zfs_uio_fault_move((char *)db->db_data + bufoff,
15591570
tocpy, UIO_WRITE, uio);
1571+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
15601572

15611573
if (tocpy == db->db_size && dmu_buf_fill_done(db, tx, err)) {
15621574
/* The fill was reverted. Undo any uio progress. */
@@ -2038,11 +2050,13 @@ dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
20382050
*/
20392051
zp->zp_nopwrite = B_FALSE;
20402052

2053+
mutex_enter(&((dmu_buf_impl_t *)zgd->zgd_db)->db_mtx);
20412054
zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
20422055
abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size),
20432056
zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp,
2044-
dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done,
2057+
dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done,
20452058
dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
2059+
mutex_exit(&((dmu_buf_impl_t *)zgd->zgd_db)->db_mtx);
20462060

20472061
return (0);
20482062
}

‎module/zfs/dmu_recv.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1664,8 +1664,10 @@ receive_object_is_same_generation(objset_t *os, uint64_t object,
16641664
err = dmu_bonus_hold(os, object, FTAG, &old_bonus_dbuf);
16651665
if (err != 0)
16661666
return (err);
1667+
mutex_enter(&((dmu_buf_impl_t *)old_bonus_dbuf)->db_mtx);
16671668
err = dmu_get_file_info(os, old_bonus_type, old_bonus_dbuf->db_data,
16681669
&zoi);
1670+
mutex_exit(&((dmu_buf_impl_t *)old_bonus_dbuf)->db_mtx);
16691671
dmu_buf_rele(old_bonus_dbuf, FTAG);
16701672
if (err != 0)
16711673
return (err);
@@ -2146,6 +2148,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
21462148
dmu_buf_will_dirty(db, tx);
21472149

21482150
ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
2151+
2152+
mutex_enter(&((dmu_buf_impl_t *)db)->db_mtx);
21492153
memcpy(db->db_data, data, DRR_OBJECT_PAYLOAD_SIZE(drro));
21502154

21512155
/*
@@ -2158,6 +2162,7 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
21582162
dmu_ot_byteswap[byteswap].ob_func(db->db_data,
21592163
DRR_OBJECT_PAYLOAD_SIZE(drro));
21602164
}
2165+
mutex_exit(&((dmu_buf_impl_t *)db)->db_mtx);
21612166
dmu_buf_rele(db, FTAG);
21622167
dnode_rele(dn, FTAG);
21632168
}

0 commit comments

Comments
 (0)