Skip to content

Commit ba5ee33

Browse files
oshogbooshogbo
committed
Add TXG timestamp database
This feature enables tracking of when TXGs are committed to disk, providing an estimated timestamp for each TXG. With this information, it becomes possible to perform scrubs based on specific date ranges, improving the granularity of data management and recovery operations. Signed-off-by: Mariusz Zaborski <[email protected]>
1 parent e0039c7 commit ba5ee33

File tree

21 files changed

+609
-13
lines changed

21 files changed

+609
-13
lines changed

‎cmd/zpool/zpool_main.c

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8367,6 +8367,8 @@ zpool_do_reopen(int argc, char **argv)
83678367
typedef struct scrub_cbdata {
83688368
int cb_type;
83698369
pool_scrub_cmd_t cb_scrub_cmd;
8370+
time_t cb_date_start;
8371+
time_t cb_date_end;
83708372
} scrub_cbdata_t;
83718373

83728374
static boolean_t
@@ -8410,7 +8412,8 @@ scrub_callback(zpool_handle_t *zhp, void *data)
84108412
return (1);
84118413
}
84128414

8413-
err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
8415+
err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd, cb->cb_date_start,
8416+
cb->cb_date_end);
84148417

84158418
if (err == 0 && zpool_has_checkpoint(zhp) &&
84168419
cb->cb_type == POOL_SCAN_SCRUB) {
@@ -8429,10 +8432,32 @@ wait_callback(zpool_handle_t *zhp, void *data)
84298432
return (zpool_wait(zhp, *act));
84308433
}
84318434

8435+
static time_t
8436+
date_string_to_sec(const char *timestr)
8437+
{
8438+
int ret;
8439+
struct tm tm = {0};
8440+
8441+
ret = sscanf(timestr, "%4d-%2d-%2d %2d:%2d", &tm.tm_year, &tm.tm_mon,
8442+
&tm.tm_mday, &tm.tm_hour, &tm.tm_min);
8443+
if (ret < 3) {
8444+
fprintf(stderr, gettext("Failed to parse the date.\n"));
8445+
usage(B_FALSE);
8446+
}
8447+
8448+
// Adjust struct
8449+
tm.tm_year -= 1900;
8450+
tm.tm_mon -= 1;
8451+
8452+
return (timegm(&tm));
8453+
}
8454+
84328455
/*
84338456
* zpool scrub [-e | -s | -p | -C] [-w] <pool> ...
84348457
*
84358458
* -e Only scrub blocks in the error log.
8459+
* -E End date of scrub.
8460+
* -S Start date of scrub.
84368461
* -s Stop. Stops any in-progress scrub.
84378462
* -p Pause. Pause in-progress scrub.
84388463
* -w Wait. Blocks until scrub has completed.
@@ -8448,21 +8473,28 @@ zpool_do_scrub(int argc, char **argv)
84488473

84498474
cb.cb_type = POOL_SCAN_SCRUB;
84508475
cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
8476+
cb.cb_date_start = cb.cb_date_end = 0;
84518477

84528478
boolean_t is_error_scrub = B_FALSE;
84538479
boolean_t is_pause = B_FALSE;
84548480
boolean_t is_stop = B_FALSE;
84558481
boolean_t is_txg_continue = B_FALSE;
84568482

84578483
/* check options */
8458-
while ((c = getopt(argc, argv, "spweC")) != -1) {
8484+
while ((c = getopt(argc, argv, "spweCE:S:")) != -1) {
84598485
switch (c) {
84608486
case 'e':
84618487
is_error_scrub = B_TRUE;
84628488
break;
8489+
case 'E':
8490+
cb.cb_date_end = date_string_to_sec(optarg);
8491+
break;
84638492
case 's':
84648493
is_stop = B_TRUE;
84658494
break;
8495+
case 'S':
8496+
cb.cb_date_start = date_string_to_sec(optarg);
8497+
break;
84668498
case 'p':
84678499
is_pause = B_TRUE;
84688500
break;
@@ -8510,6 +8542,13 @@ zpool_do_scrub(int argc, char **argv)
85108542
}
85118543
}
85128544

8545+
if ((cb.cb_date_start != 0 || cb.cb_date_end != 0) &&
8546+
cb.cb_scrub_cmd != POOL_SCRUB_NORMAL) {
8547+
(void) fprintf(stderr, gettext("invalid option combination: "
8548+
"start/end date is avlilable only with normal scrub\n"));
8549+
usage(B_FALSE);
8550+
}
8551+
85138552
if (wait && (cb.cb_type == POOL_SCAN_NONE ||
85148553
cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
85158554
(void) fprintf(stderr, gettext("invalid option combination: "

‎include/Makefile.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ COMMON_H = \
1010
cityhash.h \
1111
zfeature_common.h \
1212
zfs_comutil.h \
13+
zfs_crrd.h \
1314
zfs_deleg.h \
1415
zfs_fletcher.h \
1516
zfs_namecheck.h \

‎include/libzfs.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,8 @@ typedef struct trimflags {
290290
/*
291291
* Functions to manipulate pool and vdev state
292292
*/
293-
_LIBZFS_H int zpool_scan(zpool_handle_t *, pool_scan_func_t, pool_scrub_cmd_t);
293+
_LIBZFS_H int zpool_scan(zpool_handle_t *, pool_scan_func_t, pool_scrub_cmd_t,
294+
time_t, time_t);
294295
_LIBZFS_H int zpool_initialize(zpool_handle_t *, pool_initialize_func_t,
295296
nvlist_t *);
296297
_LIBZFS_H int zpool_initialize_wait(zpool_handle_t *, pool_initialize_func_t,

‎include/sys/dmu.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,9 @@ typedef struct dmu_buf {
393393
#define DMU_POOL_ZPOOL_CHECKPOINT "com.delphix:zpool_checkpoint"
394394
#define DMU_POOL_LOG_SPACEMAP_ZAP "com.delphix:log_spacemap_zap"
395395
#define DMU_POOL_DELETED_CLONES "com.delphix:deleted_clones"
396+
#define DMU_POOL_TXG_LOG_TIME_MINUTES "com.klaraystems:txg_log_time:minutes"
397+
#define DMU_POOL_TXG_LOG_TIME_DAYS "com.klaraystems:txg_log_time:days"
398+
#define DMU_POOL_TXG_LOG_TIME_MONTHS "com.klaraystems:txg_log_time:months"
396399

397400
/*
398401
* Allocate an object from this objset. The range of object numbers

‎include/sys/spa_impl.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@
5454
#include <sys/dsl_deadlist.h>
5555
#include <zfeature_common.h>
5656

57+
#include "zfs_crrd.h"
58+
5759
#ifdef __cplusplus
5860
extern "C" {
5961
#endif
@@ -353,6 +355,12 @@ struct spa {
353355
spa_checkpoint_info_t spa_checkpoint_info; /* checkpoint accounting */
354356
zthr_t *spa_checkpoint_discard_zthr;
355357

358+
kmutex_t spa_txg_log_time_lock; /* for spa_txg_log_time */
359+
dbrrd_t spa_txg_log_time;
360+
uint64_t spa_last_noted_txg;
361+
uint64_t spa_last_noted_txg_time;
362+
uint64_t spa_last_flush_txg_time;
363+
356364
space_map_t *spa_syncing_log_sm; /* current log space map */
357365
avl_tree_t spa_sm_logs_by_txg;
358366
kmutex_t spa_flushed_ms_lock; /* for metaslabs_by_flushed */

‎include/zfs_crrd.h

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9+
* or https://opensource.org/licenses/CDDL-1.0.
10+
* See the License for the specific language governing permissions
11+
* and limitations under the License.
12+
*
13+
* When distributing Covered Code, include this CDDL HEADER in each
14+
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15+
* If applicable, add the following below this CDDL HEADER, with the
16+
* fields enclosed by brackets "[]" replaced with your own identifying
17+
* information: Portions Copyright [yyyy] [name of copyright owner]
18+
*
19+
* CDDL HEADER END
20+
*/
21+
/*
22+
* Copyright (c) 2024 Klara Inc.
23+
*
24+
* This software was developed by
25+
* Fred Weigel <[email protected]>
26+
* Mariusz Zaborski <[email protected]>
27+
* under sponsorship from Wasabi Technology, Inc. and Klara Inc.
28+
*/
29+
30+
#ifndef _CRRD_H_
31+
#define _CRRD_H_
32+
33+
#define RRD_MAX_ENTRIES 256
34+
35+
typedef enum {
36+
DBRRD_FLOOR,
37+
DBRRD_CEILING
38+
} dbrrd_rounding_t;
39+
40+
typedef struct {
41+
hrtime_t rrdd_time;
42+
uint64_t rrdd_txg;
43+
} rrd_data_t;
44+
45+
typedef struct {
46+
int rrd_head; /* head (beginning) */
47+
int rrd_tail; /* tail (end) */
48+
size_t rrd_length;
49+
50+
rrd_data_t rrd_entries[RRD_MAX_ENTRIES];
51+
} rrd_t;
52+
53+
typedef struct {
54+
rrd_t dbr_minutes;
55+
rrd_t dbr_days;
56+
rrd_t dbr_months;
57+
} dbrrd_t;
58+
59+
rrd_t *rrd_create(void);
60+
size_t rrd_len(rrd_t *rrd);
61+
62+
const rrd_data_t *rrd_entry(rrd_t *r, size_t i);
63+
const rrd_data_t *rrd_tail_entry(rrd_t *rrd);
64+
uint64_t rrd_tail(rrd_t *rrd);
65+
uint64_t rrd_get(rrd_t *rrd, size_t i);
66+
67+
void rrd_add(rrd_t *rrd, hrtime_t time, uint64_t txg);
68+
void rrd_pack(rrd_t *rrd, uint8_t *buffer);
69+
void rrd_ntoh(rrd_t *rrd);
70+
71+
void dbrrd_add(dbrrd_t *db, hrtime_t time, uint64_t txg);
72+
uint64_t dbrrd_query(dbrrd_t *r, hrtime_t tv, dbrrd_rounding_t rouding);
73+
74+
#endif

‎lib/libnvpair/libnvpair.abi

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2194,6 +2194,7 @@
21942194
</data-member>
21952195
</class-decl>
21962196
<typedef-decl name='stack_t' type-id='380f9954' id='ac5e685f'/>
2197+
<typedef-decl name='unw_regnum_t' type-id='95e97e5e' id='c53620f0'/>
21972198
<class-decl name='unw_cursor' size-in-bits='8128' is-struct='yes' visibility='default' id='384a1f22'>
21982199
<data-member access='public' layout-offset-in-bits='0'>
21992200
<var-decl name='opaque' type-id='dc70ec0b' visibility='default'/>
@@ -2306,6 +2307,10 @@
23062307
<parameter type-id='b59d7dce'/>
23072308
<return type-id='79a0948f'/>
23082309
</function-decl>
2310+
<function-decl name='_Ux86_64_regname' visibility='default' binding='global' size-in-bits='64'>
2311+
<parameter type-id='c53620f0'/>
2312+
<return type-id='80f4b756'/>
2313+
</function-decl>
23092314
<function-decl name='_ULx86_64_init_local' visibility='default' binding='global' size-in-bits='64'>
23102315
<parameter type-id='3946e4d1'/>
23112316
<parameter type-id='2e408b96'/>

‎lib/libuutil/libuutil.abi

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,7 @@
652652
</data-member>
653653
</class-decl>
654654
<typedef-decl name='stack_t' type-id='380f9954' id='ac5e685f'/>
655+
<typedef-decl name='unw_regnum_t' type-id='95e97e5e' id='c53620f0'/>
655656
<class-decl name='unw_cursor' size-in-bits='8128' is-struct='yes' visibility='default' id='384a1f22'>
656657
<data-member access='public' layout-offset-in-bits='0'>
657658
<var-decl name='opaque' type-id='dc70ec0b' visibility='default'/>
@@ -763,6 +764,10 @@
763764
<parameter type-id='b59d7dce'/>
764765
<return type-id='79a0948f'/>
765766
</function-decl>
767+
<function-decl name='_Ux86_64_regname' visibility='default' binding='global' size-in-bits='64'>
768+
<parameter type-id='c53620f0'/>
769+
<return type-id='80f4b756'/>
770+
</function-decl>
766771
<function-decl name='_ULx86_64_init_local' visibility='default' binding='global' size-in-bits='64'>
767772
<parameter type-id='3946e4d1'/>
768773
<parameter type-id='2e408b96'/>

‎lib/libzfs/libzfs.abi

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@
629629
<elf-symbol name='fletcher_4_superscalar_ops' size='128' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
630630
<elf-symbol name='libzfs_config_ops' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
631631
<elf-symbol name='sa_protocol_names' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
632-
<elf-symbol name='spa_feature_table' size='2464' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
632+
<elf-symbol name='spa_feature_table' size='2520' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
633633
<elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
634634
<elf-symbol name='zfs_deleg_perm_tab' size='512' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
635635
<elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1170,6 +1170,7 @@
11701170
</data-member>
11711171
</class-decl>
11721172
<typedef-decl name='stack_t' type-id='380f9954' id='ac5e685f'/>
1173+
<typedef-decl name='unw_regnum_t' type-id='95e97e5e' id='c53620f0'/>
11731174
<class-decl name='unw_cursor' size-in-bits='8128' is-struct='yes' visibility='default' id='384a1f22'>
11741175
<data-member access='public' layout-offset-in-bits='0'>
11751176
<var-decl name='opaque' type-id='dc70ec0b' visibility='default'/>
@@ -1275,6 +1276,10 @@
12751276
<pointer-type-def type-id='1203d35c' size-in-bits='64' id='3946e4d1'/>
12761277
<pointer-type-def type-id='190d09ef' size-in-bits='64' id='3e0601f0'/>
12771278
<pointer-type-def type-id='73d941c6' size-in-bits='64' id='42f5faab'/>
1279+
<function-decl name='_Ux86_64_regname' visibility='default' binding='global' size-in-bits='64'>
1280+
<parameter type-id='c53620f0'/>
1281+
<return type-id='80f4b756'/>
1282+
</function-decl>
12781283
<function-decl name='_ULx86_64_init_local' visibility='default' binding='global' size-in-bits='64'>
12791284
<parameter type-id='3946e4d1'/>
12801285
<parameter type-id='2e408b96'/>
@@ -6197,7 +6202,8 @@
61976202
<enumerator name='SPA_FEATURE_FAST_DEDUP' value='41'/>
61986203
<enumerator name='SPA_FEATURE_LONGNAME' value='42'/>
61996204
<enumerator name='SPA_FEATURE_LARGE_MICROZAP' value='43'/>
6200-
<enumerator name='SPA_FEATURES' value='44'/>
6205+
<enumerator name='SPA_FEATURE_TXG_TIMELOG' value='44'/>
6206+
<enumerator name='SPA_FEATURES' value='45'/>
62016207
</enum-decl>
62026208
<typedef-decl name='spa_feature_t' type-id='33ecb627' id='d6618c78'/>
62036209
<qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
@@ -6720,6 +6726,8 @@
67206726
<parameter type-id='4c81de99' name='zhp'/>
67216727
<parameter type-id='7313fbe2' name='func'/>
67226728
<parameter type-id='b51cf3c2' name='cmd'/>
6729+
<parameter type-id='c9d12d66' name='date_start'/>
6730+
<parameter type-id='c9d12d66' name='date_end'/>
67236731
<return type-id='95e97e5e'/>
67246732
</function-decl>
67256733
<function-decl name='zpool_find_vdev_by_physpath' mangled-name='zpool_find_vdev_by_physpath' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_find_vdev_by_physpath'>
@@ -8368,6 +8376,11 @@
83688376
<parameter type-id='95e97e5e'/>
83698377
<return type-id='48b5725f'/>
83708378
</function-decl>
8379+
<function-decl name='setpgid' visibility='default' binding='global' size-in-bits='64'>
8380+
<parameter type-id='3629bad8'/>
8381+
<parameter type-id='3629bad8'/>
8382+
<return type-id='95e97e5e'/>
8383+
</function-decl>
83718384
<function-decl name='fork' visibility='default' binding='global' size-in-bits='64'>
83728385
<return type-id='3629bad8'/>
83738386
</function-decl>
@@ -9376,8 +9389,8 @@
93769389
</function-decl>
93779390
</abi-instr>
93789391
<abi-instr address-size='64' path='module/zcommon/zfeature_common.c' language='LANG_C99'>
9379-
<array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='19712' id='fd4573e5'>
9380-
<subrange length='44' type-id='7359adad' id='cf8ba455'/>
9392+
<array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='20160' id='b948da70'>
9393+
<subrange length='45' type-id='7359adad' id='cb8ddca0'/>
93819394
</array-type-def>
93829395
<enum-decl name='zfeature_flags' id='6db816a4'>
93839396
<underlying-type type-id='9cac1fee'/>
@@ -9454,7 +9467,7 @@
94549467
<pointer-type-def type-id='611586a1' size-in-bits='64' id='2e243169'/>
94559468
<qualified-type-def type-id='eaa32e2f' const='yes' id='83be723c'/>
94569469
<pointer-type-def type-id='83be723c' size-in-bits='64' id='7acd98a2'/>
9457-
<var-decl name='spa_feature_table' type-id='fd4573e5' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
9470+
<var-decl name='spa_feature_table' type-id='b948da70' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
94589471
<var-decl name='zfeature_checks_disable' type-id='c19b74c3' mangled-name='zfeature_checks_disable' visibility='default' elf-symbol-id='zfeature_checks_disable'/>
94599472
<function-decl name='opendir' visibility='default' binding='global' size-in-bits='64'>
94609473
<parameter type-id='80f4b756'/>

‎lib/libzfs/libzfs_pool.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2728,7 +2728,8 @@ zpool_trim(zpool_handle_t *zhp, pool_trim_func_t cmd_type, nvlist_t *vds,
27282728
* Scan the pool.
27292729
*/
27302730
int
2731-
zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func, pool_scrub_cmd_t cmd)
2731+
zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func, pool_scrub_cmd_t cmd,
2732+
time_t date_start, time_t date_end)
27322733
{
27332734
char errbuf[ERRBUFLEN];
27342735
int err;
@@ -2737,6 +2738,8 @@ zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func, pool_scrub_cmd_t cmd)
27372738
nvlist_t *args = fnvlist_alloc();
27382739
fnvlist_add_uint64(args, "scan_type", (uint64_t)func);
27392740
fnvlist_add_uint64(args, "scan_command", (uint64_t)cmd);
2741+
fnvlist_add_uint64(args, "scan_date_start", (uint64_t)date_start);
2742+
fnvlist_add_uint64(args, "scan_date_end", (uint64_t)date_end);
27402743

27412744
err = lzc_scrub(ZFS_IOC_POOL_SCRUB, zhp->zpool_name, args, NULL);
27422745
fnvlist_free(args);

‎lib/libzfs_core/libzfs_core.abi

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,6 +651,7 @@
651651
</data-member>
652652
</class-decl>
653653
<typedef-decl name='stack_t' type-id='380f9954' id='ac5e685f'/>
654+
<typedef-decl name='unw_regnum_t' type-id='95e97e5e' id='c53620f0'/>
654655
<class-decl name='unw_cursor' size-in-bits='8128' is-struct='yes' visibility='default' id='384a1f22'>
655656
<data-member access='public' layout-offset-in-bits='0'>
656657
<var-decl name='opaque' type-id='dc70ec0b' visibility='default'/>
@@ -762,6 +763,10 @@
762763
<parameter type-id='b59d7dce'/>
763764
<return type-id='79a0948f'/>
764765
</function-decl>
766+
<function-decl name='_Ux86_64_regname' visibility='default' binding='global' size-in-bits='64'>
767+
<parameter type-id='c53620f0'/>
768+
<return type-id='80f4b756'/>
769+
</function-decl>
765770
<function-decl name='_ULx86_64_init_local' visibility='default' binding='global' size-in-bits='64'>
766771
<parameter type-id='3946e4d1'/>
767772
<parameter type-id='2e408b96'/>

‎lib/libzfsbootenv/libzfsbootenv.abi

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<abi-corpus version='2.0' architecture='elf-amd-x86_64' soname='libzfsbootenv.so.1'>
22
<elf-needed>
3-
<dependency name='libzfs.so.4'/>
3+
<dependency name='libzfs.so.6'/>
44
<dependency name='libnvpair.so.3'/>
55
<dependency name='libc.so.6'/>
66
</elf-needed>

‎lib/libzpool/Makefile.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ nodist_libzpool_la_SOURCES = \
178178
module/zfs/zfeature.c \
179179
module/zfs/zfs_byteswap.c \
180180
module/zfs/zfs_chksum.c \
181+
module/zfs/zfs_crrd.c \
181182
module/zfs/zfs_fm.c \
182183
module/zfs/zfs_fuid.c \
183184
module/zfs/zfs_ratelimit.c \

0 commit comments

Comments
 (0)