Skip to content

Commit daf7860

Browse files
antekresicantekresic
committed
Add compression batch size limit GUC
This GUC can be used to set the limit of rows batched together during compression. Valid values are 1-999 with 0 disabling limiting and using the default 1000 row limit.
1 parent 672f3ea commit daf7860

File tree

6 files changed

+134
-2
lines changed

6 files changed

+134
-2
lines changed

‎.unreleased/pr_7923

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implements: #7923 Add compression batch size limit GUC

‎src/guc.c

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ bool ts_guc_enable_chunk_skipping = false;
154154
TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression = true;
155155
TSDLLEXPORT bool ts_guc_enable_exclusive_locking_recompression = false;
156156
TSDLLEXPORT bool ts_guc_enable_bool_compression = false;
157+
TSDLLEXPORT int ts_guc_compression_batch_size_limit = 1000;
157158

158159
/* Only settable in debug mode for testing */
159160
TSDLLEXPORT bool ts_guc_enable_null_compression = true;
@@ -791,6 +792,23 @@ _guc_init(void)
791792
NULL,
792793
NULL,
793794
NULL);
795+
DefineCustomIntVariable(MAKE_EXTOPTION("compression_batch_size_limit"),
796+
"The max number of tuples that can be batched together during "
797+
"compression",
798+
"Setting this option to a number between 1 and 999 will force "
799+
"compression "
800+
"to limit the size of compressed batches to that amount of "
801+
"uncompressed tuples."
802+
"Setting this to 0 defaults to the max batch size of 1000.",
803+
&ts_guc_compression_batch_size_limit,
804+
1000,
805+
1,
806+
1000,
807+
PGC_USERSET,
808+
0,
809+
NULL,
810+
NULL,
811+
NULL);
794812

795813
#ifdef TS_DEBUG
796814
DefineCustomBoolVariable(MAKE_EXTOPTION("enable_null_compression"),

‎src/guc.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ extern bool ts_guc_enable_chunk_skipping;
7171
extern TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression;
7272
extern TSDLLEXPORT bool ts_guc_enable_exclusive_locking_recompression;
7373
extern TSDLLEXPORT bool ts_guc_enable_bool_compression;
74+
extern TSDLLEXPORT int ts_guc_compression_batch_size_limit;
7475
#if PG16_GE
7576
extern TSDLLEXPORT bool ts_guc_enable_skip_scan_for_distinct_aggregates;
7677
#endif

‎tsl/src/compression/compression.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -894,8 +894,8 @@ row_compressor_process_ordered_slot(RowCompressor *row_compressor, TupleTableSlo
894894
row_compressor->first_iteration = false;
895895
}
896896
bool changed_groups = row_compressor_new_row_is_in_new_group(row_compressor, slot);
897-
bool compressed_row_is_full =
898-
row_compressor->rows_compressed_into_current_value >= TARGET_COMPRESSED_BATCH_SIZE;
897+
bool compressed_row_is_full = row_compressor->rows_compressed_into_current_value >=
898+
(uint32) ts_guc_compression_batch_size_limit;
899899
if (compressed_row_is_full || changed_groups)
900900
{
901901
if (row_compressor->rows_compressed_into_current_value > 0)

‎tsl/test/expected/compression.out

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2905,3 +2905,74 @@ SELECT count(*) FROM :CHUNK;
29052905
(1 row)
29062906

29072907
RESET timescaledb.enable_delete_after_compression;
2908+
-- Test batch size limiting GUC
2909+
CREATE TABLE hyper_85 (time timestamptz, device int8, location int8, temp float8);
2910+
SELECT create_hypertable('hyper_85', 'time', create_default_indexes => false);
2911+
NOTICE: adding not-null constraint to column "time"
2912+
create_hypertable
2913+
------------------------
2914+
(55,public,hyper_85,t)
2915+
(1 row)
2916+
2917+
INSERT INTO hyper_85
2918+
SELECT t, 1, 1, 1.0
2919+
FROM generate_series('2024-01-01'::timestamptz, '2024-01-02'::timestamptz, '20 sec'::interval) t;
2920+
ALTER TABLE hyper_85 SET (timescaledb.compress, timescaledb.compress_segmentby='device');
2921+
NOTICE: default order by for hypertable "hyper_85" is set to ""time" DESC"
2922+
-- first without the limit
2923+
BEGIN;
2924+
SELECT compress_chunk(ch) FROM show_chunks('hyper_85') ch;
2925+
compress_chunk
2926+
-------------------------------------------
2927+
_timescaledb_internal._hyper_55_113_chunk
2928+
(1 row)
2929+
2930+
SELECT ch1.id "CHUNK_ID"
2931+
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'hyper_85'
2932+
ORDER BY ch1.id
2933+
LIMIT 1 \gset
2934+
select compressed.schema_name|| '.' || compressed.table_name as "COMPRESSED_CHUNK_NAME"
2935+
from _timescaledb_catalog.chunk uncompressed, _timescaledb_catalog.chunk compressed
2936+
where uncompressed.compressed_chunk_id = compressed.id AND uncompressed.id = :'CHUNK_ID' \gset
2937+
SELECT _ts_meta_count FROM :COMPRESSED_CHUNK_NAME ORDER BY device, _ts_meta_min_1 DESC;
2938+
_ts_meta_count
2939+
----------------
2940+
1000
2941+
1000
2942+
1000
2943+
1000
2944+
321
2945+
(5 rows)
2946+
2947+
ROLLBACK;
2948+
-- now lets set the limit
2949+
SET timescaledb.compression_batch_size_limit = 505;
2950+
BEGIN;
2951+
SELECT compress_chunk(ch) FROM show_chunks('hyper_85') ch;
2952+
compress_chunk
2953+
-------------------------------------------
2954+
_timescaledb_internal._hyper_55_113_chunk
2955+
(1 row)
2956+
2957+
SELECT ch1.id "CHUNK_ID"
2958+
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'hyper_85'
2959+
ORDER BY ch1.id
2960+
LIMIT 1 \gset
2961+
select compressed.schema_name|| '.' || compressed.table_name as "COMPRESSED_CHUNK_NAME"
2962+
from _timescaledb_catalog.chunk uncompressed, _timescaledb_catalog.chunk compressed
2963+
where uncompressed.compressed_chunk_id = compressed.id AND uncompressed.id = :'CHUNK_ID' \gset
2964+
SELECT _ts_meta_count FROM :COMPRESSED_CHUNK_NAME ORDER BY device, _ts_meta_min_1 DESC;
2965+
_ts_meta_count
2966+
----------------
2967+
505
2968+
505
2969+
505
2970+
505
2971+
505
2972+
505
2973+
505
2974+
505
2975+
281
2976+
(9 rows)
2977+
2978+
ROLLBACK;

‎tsl/test/sql/compression.sql

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,3 +1198,44 @@ SELECT relpages, reltuples::int AS reltuples FROM pg_catalog.pg_class WHERE oid
11981198
SELECT count(*) FROM :CHUNK;
11991199

12001200
RESET timescaledb.enable_delete_after_compression;
1201+
1202+
-- Test batch size limiting GUC
1203+
CREATE TABLE hyper_85 (time timestamptz, device int8, location int8, temp float8);
1204+
SELECT create_hypertable('hyper_85', 'time', create_default_indexes => false);
1205+
INSERT INTO hyper_85
1206+
SELECT t, 1, 1, 1.0
1207+
FROM generate_series('2024-01-01'::timestamptz, '2024-01-02'::timestamptz, '20 sec'::interval) t;
1208+
ALTER TABLE hyper_85 SET (timescaledb.compress, timescaledb.compress_segmentby='device');
1209+
1210+
-- first without the limit
1211+
BEGIN;
1212+
SELECT compress_chunk(ch) FROM show_chunks('hyper_85') ch;
1213+
1214+
SELECT ch1.id "CHUNK_ID"
1215+
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'hyper_85'
1216+
ORDER BY ch1.id
1217+
LIMIT 1 \gset
1218+
1219+
select compressed.schema_name|| '.' || compressed.table_name as "COMPRESSED_CHUNK_NAME"
1220+
from _timescaledb_catalog.chunk uncompressed, _timescaledb_catalog.chunk compressed
1221+
where uncompressed.compressed_chunk_id = compressed.id AND uncompressed.id = :'CHUNK_ID' \gset
1222+
1223+
SELECT _ts_meta_count FROM :COMPRESSED_CHUNK_NAME ORDER BY device, _ts_meta_min_1 DESC;
1224+
ROLLBACK;
1225+
1226+
-- now lets set the limit
1227+
SET timescaledb.compression_batch_size_limit = 505;
1228+
BEGIN;
1229+
SELECT compress_chunk(ch) FROM show_chunks('hyper_85') ch;
1230+
1231+
SELECT ch1.id "CHUNK_ID"
1232+
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'hyper_85'
1233+
ORDER BY ch1.id
1234+
LIMIT 1 \gset
1235+
1236+
select compressed.schema_name|| '.' || compressed.table_name as "COMPRESSED_CHUNK_NAME"
1237+
from _timescaledb_catalog.chunk uncompressed, _timescaledb_catalog.chunk compressed
1238+
where uncompressed.compressed_chunk_id = compressed.id AND uncompressed.id = :'CHUNK_ID' \gset
1239+
1240+
SELECT _ts_meta_count FROM :COMPRESSED_CHUNK_NAME ORDER BY device, _ts_meta_min_1 DESC;
1241+
ROLLBACK;

0 commit comments

Comments
 (0)