Repair numeric partial state on the fly

The numeric format changed between PG13 and PG14 to include infinities.
As a result the serialized partial state of numeric aggregates also
changed format.

If a user that has stored partials (e.g. by using Continuous
Aggregates) upgrades to PG14 then the partial state deserialization
will lead to errors due to the mismatch with the PG14 code.

Repair the deserialization process on the fly by appending zeroed
plus-infinity and minus-infinity counts for the numeric aggregate state
to use.

Fixes #4427

Author: Markos Fountoulakis

tsl/src/partialize_finalize.c

@@ -14,6 +14,7 @@
 #include <parser/parse_coerce.h>
 #include <utils/builtins.h>
 #include <utils/datum.h>
+#include <utils/fmgroids.h>
 #include <utils/syscache.h>
 
 #include "partialize_finalize.h"
@@ -147,6 +148,99 @@ collation_oid_from_name(char *schema_name, char *collation_name)
 	namel = lappend(namel, makeString(collation_name));
 	return get_collation_oid(namel, false);
 }
+
+#if PG14_GE
+/*
+ * =======================================
+ * Record serialized partials errata here:
+ * =======================================
+ *
+ * ================================================================================================
+ * The numeric format changed between PG13 and PG14 to include infinities. Consequently the
+ * internal aggregate for the combine functions changed as well, which lead to the serialized
+ * partial state of numeric aggregates also changing format.
+ *
+ * If a user that has stored partials (by using Continuous Aggregates or calling
+ * _timescaledb_internal.finalize_agg()) upgrades to PG14 then the partial state deserialization
+ * will lead to errors due to the mismatch with the PG14 code.
+ *
+ * For F_NUMERIC_AVG_DESERIALIZE and F_NUMERIC_DESERIALIZE the length of the serialized aggregate
+ * state is 16 bytes longer than the previous versions. It suffices to zero out the extra bytes of
+ * pInfcount and nInfcount in the numeric aggregate combine state. Those fields are only there to
+ * support infinity and -infinity in the numeric type, which wasn't supported in older PostgreSQL
+ * versions.
+ *
+ * PostgreSQL versions < 14
+ * F_NUMERIC_AVG_DESERIALIZE serialized partial length = X
+ * F_NUMERIC_DESERIALIZE serialized partial length = X + 10
+ *
+ * PostgreSQL versions >= 14
+ * F_NUMERIC_AVG_DESERIALIZE serialized partial length = X + 16
+ * F_NUMERIC_DESERIALIZE serialized partial length = X + 10 + 16
+ *
+ * For more information see: https://www.postgresql.org/message-id/[email protected]
+ *
+ * Non-numeric aggregate functions affected:
+ * 1. var_pop(int8)
+ * 2. var_samp(int8)
+ * 3. variance(int8)
+ * 4. stddev_pop(int8)
+ * 5. stddev_samp(int8)
+ * 6. stddev(int8)
+ * ================================================================================================
+ * F_NUMERIC_POLY_DESERIALIZE depends on compiler support for HAVE_INT128. As a result, recompiling
+ * the same versions of PostgreSQL and TimescaleDB with a different compiler can lead to corruption
+ * if the same database is reused.
+ *
+ * Non-numeric aggregate functions affected:
+ * 1. var_pop(int4)
+ * 2. var_pop(int2)
+ * 3. var_samp(int4)
+ * 4. var_samp(int2)
+ * 5. variance(int4)
+ * 6. variance(int2)
+ * 7. stddev_pop(int4)
+ * 8. stddev_pop(int2)
+ * 9. stddev_samp(int4)
+ * 10. stddev_samp(int2)
+ * 11. stddev(int4)
+ * 12. stddev(int2)
+ */
+#define NUMERIC_PARTIAL_MISSING_LENGTH (16)
+
+static bytea *
+zero_fill_bytearray(bytea *serialized_partial, size_t missing_length)
+{
+	size_t original_length = VARSIZE_ANY_EXHDR(serialized_partial);
+	size_t desired_length = original_length + missing_length;
+
+	bytea *new_bytea = repalloc(serialized_partial, desired_length + VARHDRSZ);
+
+	SET_VARSIZE(new_bytea, desired_length + VARHDRSZ);
+	void *end_of_serialized_partial = VARDATA(new_bytea) + original_length;
+	memset(end_of_serialized_partial, 0, missing_length);
+
+	return new_bytea;
+}
+#endif
+
+/* Only call this function if the partial is known to be problematic. */
+static bytea *
+sanitize_serialized_partial(Oid deserialfnoid, bytea *serialized_partial)
+{
+#if PG14_GE
+	if ((deserialfnoid == F_NUMERIC_DESERIALIZE) || (deserialfnoid == F_NUMERIC_AVG_DESERIALIZE))
+		/*
+		 * Always add NUMERIC_PARTIAL_MISSING_LENGTH extra bytes because the length is not fixed.
+		 * This is only safe to do when the partial state is known to be short, otherwise an
+		 * exception is thrown if the serialized_partial is not fully consumed by deserialfn().
+		 */
+		return zero_fill_bytearray(serialized_partial, NUMERIC_PARTIAL_MISSING_LENGTH);
+#endif
+
+	return serialized_partial;
+}
+
 /*
  * deserialize from the internal format in which data is stored in bytea
  * parameter. Callers need to check deserialized_isnull . Only if this is set to false,
@@ -169,8 +263,35 @@ inner_agg_deserialize(FACombineFnMeta *combine_meta, bytea *serialized_partial,
 
 		FC_ARG(deser_fcinfo, 0) = PointerGetDatum(serialized_partial);
 		FC_NULL(deser_fcinfo, 0) = serialized_isnull;
-		combine_meta->deserialfn_fcinfo->isnull = false;
-		deserialized = FunctionCallInvoke(deser_fcinfo);
+		deser_fcinfo->isnull = false;
+
+		/*
+		 * When an exception is thrown and longjmp() is called, CurrentMemoryContext is potentially
+		 * different than what it was inside the PG_TRY() block below.
+		 *
+		 * Restore it to the old value so that the code in the subsequent PG_CATCH() block does not
+		 * corrupt the memory.
+		 *
+		 * No need for volatile variables since we don't modify any of this function's stack frame
+		 * inside the PG_TRY() block.
+		 */
+		MemoryContext oldcontext = CurrentMemoryContext;
+		PG_TRY();
+		{
+			deserialized = FunctionCallInvoke(deser_fcinfo);
+		}
+		PG_CATCH();
+		{
+			CurrentMemoryContext = oldcontext;
+			FlushErrorState();
+			/* attempt to repair the serialized partial */
+			serialized_partial =
+				sanitize_serialized_partial(combine_meta->deserialfnoid, serialized_partial);
+			FC_ARG(deser_fcinfo, 0) = PointerGetDatum(serialized_partial);
+			deser_fcinfo->isnull = false;
+			deserialized = FunctionCallInvoke(deser_fcinfo);
+		}
+		PG_END_TRY();
 		*deserialized_isnull = deser_fcinfo->isnull;
 	}
 	else if (!serialized_isnull)

tsl/test/expected/partialize_finalize.out

@@ -142,44 +142,66 @@ select a, _timescaledb_internal.finalize_agg( 'sum(double precision)', null, nul
 drop table t1;
 drop view v1;
 drop table foo;
-create table foo (a integer, b numeric , c text, d timestamptz);
-insert into foo values( 1 , 10 , 'hello', '2010-01-01 09:00:00-08');
-insert into foo values( 1 , 20 , 'abc', '2010-01-02 09:00:00-08');
-insert into foo values( 1 , 30 , 'abcd',  '2010-01-03 09:00:00-08');
-insert into foo values( 1 , 40 , 'abcde', NULL );
-insert into foo values( 1 , 50 , NULL,  '2010-01-01 09:00:00-08');
+create table foo (a integer, b numeric , c text, d timestamptz, e bigint);
+insert into foo values( 1 , 10 , 'hello', '2010-01-01 09:00:00-08', 10);
+insert into foo values( 1 , 20 , 'abc', '2010-01-02 09:00:00-08', 20);
+insert into foo values( 1 , 30 , 'abcd',  '2010-01-03 09:00:00-08', 30);
+insert into foo values( 1 , 40 , 'abcde', NULL, 40);
+insert into foo values( 1 , 50 , NULL,  '2010-01-01 09:00:00-08', 50);
 --group with all values for c and d same
-insert into foo values( 2 , 10 ,  'hello', '2010-01-01 09:00:00-08');
-insert into foo values( 2 , 20 , 'hello', '2010-01-01 09:00:00-08');
-insert into foo values( 2 , 30 , 'hello', '2010-01-01 09:00:00-08');
+insert into foo values( 2 , 10 ,  'hello', '2010-01-01 09:00:00-08', 10);
+insert into foo values( 2 , 20 , 'hello', '2010-01-01 09:00:00-08', 20);
+insert into foo values( 2 , 30 , 'hello', '2010-01-01 09:00:00-08', 30);
 --group with all values for c and d NULL
-insert into foo values( 3 , 40 , NULL, NULL);
-insert into foo values( 3 , 50 , NULL, NULL);
-insert into foo values(11, NULL, NULL, NULL);
-insert into foo values(11, NULL, 'hello', '2010-01-02 09:00:00-05');
+insert into foo values( 3 , 40 , NULL, NULL, 40);
+insert into foo values( 3 , 50 , NULL, NULL, 50);
+insert into foo values(11, NULL, NULL, NULL, NULL);
+insert into foo values(11, NULL, 'hello', '2010-01-02 09:00:00-05', NULL);
 --group with all values for c and d NULL and later add non-null.
-insert into foo values(12, NULL, NULL, NULL);
-create or replace view v1(a , b, partialb, partialc, partiald)
+insert into foo values(12, NULL, NULL, NULL, NULL);
+create or replace view v1(a , b, partialb, partialc, partiald, partiale, partialf)
 as
- SELECT a, b, _timescaledb_internal.partialize_agg( sum(b)) , _timescaledb_internal.partialize_agg( min(c)) , _timescaledb_internal.partialize_agg(max(d)) from foo group by a, b ;
+ SELECT a, b, _timescaledb_internal.partialize_agg(sum(b))
+ , _timescaledb_internal.partialize_agg(min(c))
+ , _timescaledb_internal.partialize_agg(max(d))
+ , _timescaledb_internal.partialize_agg(stddev(b))
+ , _timescaledb_internal.partialize_agg(stddev(e)) from foo group by a, b ;
 create table t1 as select * from v1;
 --sum 2114, collid 0, min(text) 2145, collid 100, max(ts) 2127
-insert into foo values(12, 10, 'hello', '2010-01-02 09:00:00-05');
+insert into foo values(12, 10, 'hello', '2010-01-02 09:00:00-05', 10);
 insert into t1 select * from v1 where  (a = 12 and b = 10) ;
---select a, sum(b), min(c) , max(d) from foo group by a order by a;
+--select a, sum(b), min(c) , max(d), stddev(b), stddev(e) from foo group by a order by a;
 --results should match above query
+CREATE OR REPLACE VIEW vfinal(a , sumb, minc, maxd, stddevb, stddeve)
+AS
 select a, _timescaledb_internal.finalize_agg( 'sum(numeric)', null, null, null, partialb, null::numeric ) sumb
 , _timescaledb_internal.finalize_agg( 'min(text)', 'pg_catalog', 'default', null, partialc, null::text ) minc
-, _timescaledb_internal.finalize_agg( 'max(timestamp with time zone)', null, null, null, partiald, null::timestamptz ) maxd from t1 group by a order by a ;
- a  | sumb | minc  |             maxd             
-----+------+-------+------------------------------
-  1 |  150 | abc   | Sun Jan 03 09:00:00 2010 PST
-  2 |   60 | hello | Fri Jan 01 09:00:00 2010 PST
-  3 |   90 |       | 
- 11 |      | hello | Sat Jan 02 06:00:00 2010 PST
- 12 |   10 | hello | Sat Jan 02 06:00:00 2010 PST
+, _timescaledb_internal.finalize_agg( 'max(timestamp with time zone)', null, null, null, partiald, null::timestamptz ) maxd
+, _timescaledb_internal.finalize_agg( 'stddev(numeric)', null, null, null, partiale, null::numeric ) stddevb
+, _timescaledb_internal.finalize_agg( 'stddev(int8)', null, null, null, partialf, null::numeric ) stddeve
+from t1 group by a order by a ;
+SELECT * FROM vfinal;
+ a  | sumb | minc  |             maxd             |       stddevb       |       stddeve       
+----+------+-------+------------------------------+---------------------+---------------------
+  1 |  150 | abc   | Sun Jan 03 09:00:00 2010 PST | 15.8113883008418967 | 15.8113883008418967
+  2 |   60 | hello | Fri Jan 01 09:00:00 2010 PST | 10.0000000000000000 | 10.0000000000000000
+  3 |   90 |       |                              |  7.0710678118654752 |  7.0710678118654752
+ 11 |      | hello | Sat Jan 02 06:00:00 2010 PST |                     |                    
+ 12 |   10 | hello | Sat Jan 02 06:00:00 2010 PST |                     |                    
 (5 rows)
 
+CREATE TABLE vfinal_res AS SELECT * FROM vfinal;
+-- overwrite partials with dumped binary values from PostrgeSQL 13 --
+TRUNCATE TABLE t1;
+\COPY t1 FROM data/partialize_finalize_data.csv WITH CSV HEADER
+--repeat query to verify partial serialization sanitization works for versions PG >= 14
+CREATE TABLE vfinal_dump_res AS SELECT * FROM vfinal;
+-- compare results to verify there is no difference
+(SELECT * FROM vfinal_res) EXCEPT (SELECT * FROM vfinal_dump_res);
+ a | sumb | minc | maxd | stddevb | stddeve 
+---+------+------+------+---------+---------
+(0 rows)
+
 --with having clause --
 select a, b ,  _timescaledb_internal.finalize_agg( 'min(text)', 'pg_catalog', 'default', null, partialc, null::text ) minc, _timescaledb_internal.finalize_agg( 'max(timestamp with time zone)', null, null, null, partiald, null::timestamptz ) maxd from t1  where b is not null group by a, b having _timescaledb_internal.finalize_agg( 'max(timestamp with time zone)', null, null, null, partiald, null::timestamptz ) is not null order by a, b;
  a  | b  | minc  |             maxd             
@@ -195,6 +217,7 @@ select a, b ,  _timescaledb_internal.finalize_agg( 'min(text)', 'pg_catalog', 'd
 (8 rows)
 
 --TEST5 test with TOAST data
+drop view vfinal;
 drop table t1;
 drop view v1;
 drop table foo;

tsl/test/sql/data/partialize_finalize_data.csv

@@ -0,0 +1,14 @@
+a,b,partialb,partialc,partiald,partiale,partialf
+1,10,\x00000000000000010001000000000000000a0000000000000000000000010000000000000000,\x68656c6c6f,\x00011f1c121ac400,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000
+2,10,\x00000000000000010001000000000000000a0000000000000000000000010000000000000000,\x68656c6c6f,\x00011f1c121ac400,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000
+1,30,\x00000000000000010001000000000000001e0000000000000000000000010000000000000000,\x61626364,\x00011f444dc98400,\x00000000000000010001000000000000001e000100000000000003840000000000000000000000010000000000000000,\x00000000000000010001000000000000001e000100000000000003840000000000000000000000010000000000000000
+12,,\x000000000000000000000000000000000000000000000000000000000000000000000000,,,\x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,\x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+1,40,\x0000000000000001000100000000000000280000000000000000000000010000000000000000,\x6162636465,,\x000000000000000100010000000000000028000100000000000006400000000000000000000000010000000000000000,\x000000000000000100010000000000000028000100000000000006400000000000000000000000010000000000000000
+1,50,\x0000000000000001000100000000000000320000000000000000000000010000000000000000,,\x00011f1c121ac400,\x000000000000000100010000000000000032000100000000000009c40000000000000000000000010000000000000000,\x000000000000000100010000000000000032000100000000000009c40000000000000000000000010000000000000000
+11,,\x000000000000000000000000000000000000000000000000000000000000000000000000,\x68656c6c6f,\x00011f2dac373800,\x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,\x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+2,20,\x0000000000000001000100000000000000140000000000000000000000010000000000000000,\x68656c6c6f,\x00011f1c121ac400,\x000000000000000100010000000000000014000100000000000001900000000000000000000000010000000000000000,\x000000000000000100010000000000000014000100000000000001900000000000000000000000010000000000000000
+3,40,\x0000000000000001000100000000000000280000000000000000000000010000000000000000,,,\x000000000000000100010000000000000028000100000000000006400000000000000000000000010000000000000000,\x000000000000000100010000000000000028000100000000000006400000000000000000000000010000000000000000
+3,50,\x0000000000000001000100000000000000320000000000000000000000010000000000000000,,,\x000000000000000100010000000000000032000100000000000009c40000000000000000000000010000000000000000,\x000000000000000100010000000000000032000100000000000009c40000000000000000000000010000000000000000
+2,30,\x00000000000000010001000000000000001e0000000000000000000000010000000000000000,\x68656c6c6f,\x00011f1c121ac400,\x00000000000000010001000000000000001e000100000000000003840000000000000000000000010000000000000000,\x00000000000000010001000000000000001e000100000000000003840000000000000000000000010000000000000000
+1,20,\x0000000000000001000100000000000000140000000000000000000000010000000000000000,\x616263,\x00011f302ff22400,\x000000000000000100010000000000000014000100000000000001900000000000000000000000010000000000000000,\x000000000000000100010000000000000014000100000000000001900000000000000000000000010000000000000000
+12,10,\x00000000000000010001000000000000000a0000000000000000000000010000000000000000,\x68656c6c6f,\x00011f2dac373800,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000,\x00000000000000010001000000000000000a000100000000000000640000000000000000000000010000000000000000