Skip to content

Commit be059d6

Browse files
davidhcoedavidhcoe
authored
feat(go/adbc/driver/snowflake): New setting to set the maximum timestamp precision to microseconds (#2917)
Introduces a new setting to set the maximum timestamp precision to Microsecond. Setting this value will convert the default Nanosecond value to Microsecond to avoid the overflow that occurs when a date is before the year 1678 or after 2262. Provides a fix for #2811 by creating a workaround that can be set by the caller. --------- Co-authored-by: David Coe <>
1 parent 95391d0 commit be059d6

File tree

13 files changed

+372
-60
lines changed

13 files changed

+372
-60
lines changed

‎csharp/test/Apache.Arrow.Adbc.Tests/IArrowArrayExtensionsTests.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,27 @@
1717

1818
using System;
1919
using Apache.Arrow.Adbc.Extensions;
20+
using Apache.Arrow.Types;
2021
using Xunit;
2122

2223
namespace Apache.Arrow.Adbc.Tests
2324
{
2425
public class IArrowArrayExtensionsTests
2526
{
27+
[Fact]
28+
public void ValidateTimestamp()
29+
{
30+
DateTimeOffset theFuture = new DateTimeOffset(new DateTime(9999, 12, 31, 0, 0, 0), TimeSpan.Zero);
31+
TimestampArray.Builder theFutureBuilder = new TimestampArray.Builder(TimestampType.Default);
32+
theFutureBuilder.Append(theFuture);
33+
TimestampArray tsFutureArray = theFutureBuilder.Build();
34+
35+
Assert.Equal(theFuture, tsFutureArray.GetTimestamp(0));
36+
37+
Assert.Equal(theFuture, tsFutureArray.ValueAt(0));
38+
Assert.Equal(theFuture, tsFutureArray.Data.DataType.GetValueConverter().Invoke(tsFutureArray, 0));
39+
}
40+
2641
[Fact]
2742
public void ValidateTime32()
2843
{

‎csharp/test/Drivers/Interop/Snowflake/ClientTests.cs

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
using System.Data;
2121
using System.Data.Common;
2222
using System.Data.SqlTypes;
23-
using System.IO;
2423
using Apache.Arrow.Adbc.Client;
2524
using Apache.Arrow.Adbc.Tests.Xunit;
25+
using Apache.Arrow.Types;
2626
using Xunit;
2727

2828
namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
@@ -243,6 +243,62 @@ public void VerifyTypesAndValues()
243243
}
244244
}
245245

246+
[SkippableFact, Order(6)]
247+
public void VerifyTimestampPrecision()
248+
{
249+
string query = "SELECT " +
250+
"TO_TIMESTAMP('9999-12-31 00:00:00') December31_9999, " +
251+
"TO_TIMESTAMP('2001-09-11 13:46:00') As September11_2001, " +
252+
"TO_TIMESTAMP('33-04-03 15:00:00') as April3_0033";
253+
254+
List<ColumnNetTypeArrowTypeValue> expectedMicrosecondValues = new List<ColumnNetTypeArrowTypeValue>()
255+
{
256+
new ColumnNetTypeArrowTypeValue("DECEMBER31_9999", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(9999, 12, 31, 0, 0, 0), TimeSpan.Zero)),
257+
new ColumnNetTypeArrowTypeValue("SEPTEMBER11_2001", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2001, 9, 11, 13, 46, 0), TimeSpan.Zero)),
258+
new ColumnNetTypeArrowTypeValue("APRIL3_0033", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(0033, 4, 3, 15, 0, 0), TimeSpan.Zero)),
259+
};
260+
261+
// if using microseconds as the max precision, everything returns correctly
262+
ValidateTimestampPrecision(SnowflakeConstants.OptionValueMicroseconds, query, expectedMicrosecondValues);
263+
264+
List<ColumnNetTypeArrowTypeValue> expectedNanoseconddValues = new List<ColumnNetTypeArrowTypeValue>()
265+
{
266+
// 572833941680662774 ticks = 3/29/1816 5:56:08 AM +00:00 and not what we asked for :/
267+
new ColumnNetTypeArrowTypeValue("DECEMBER31_9999", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(572833941680662774, TimeSpan.Zero)),
268+
269+
// within normal range, so the values return as expected
270+
new ColumnNetTypeArrowTypeValue("SEPTEMBER11_2001", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2001, 9, 11, 13, 46, 0), TimeSpan.Zero)),
271+
272+
// 563580782211286549 ticks = 12/1/1786 1:43:41 PM +00:00 and not what we asked for :/
273+
new ColumnNetTypeArrowTypeValue("APRIL3_0033", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(563580782211286549, TimeSpan.Zero)),
274+
};
275+
276+
// if you use the default (nanoseconds) precision, the values are incorrect
277+
ValidateTimestampPrecision(SnowflakeConstants.OptionValueNanoseconds, query, expectedNanoseconddValues);
278+
279+
// if `error on overflow` is enforced, then an error is thrown
280+
Assert.Throws<Exception>(() => ValidateTimestampPrecision(SnowflakeConstants.OptionValueNanosecondsNoOverflow, query, expectedNanoseconddValues));
281+
}
282+
283+
private void ValidateTimestampPrecision(string precision, string query, List<ColumnNetTypeArrowTypeValue> expectedValues)
284+
{
285+
SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
286+
testConfiguration.MaxTimestampPrecision = precision;
287+
288+
using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration))
289+
{
290+
SampleDataBuilder sampleDataBuilder = new SampleDataBuilder();
291+
sampleDataBuilder.Samples.Add(
292+
new SampleData()
293+
{
294+
Query = query,
295+
ExpectedValues = expectedValues
296+
});
297+
298+
Tests.ClientTests.VerifyTypesAndValues(adbcConnection, sampleDataBuilder);
299+
}
300+
}
301+
246302
[SkippableFact]
247303
public void VerifySchemaTables()
248304
{
@@ -297,6 +353,8 @@ private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnectionUsingConnectionStri
297353
builder[SnowflakeParameters.HOST] = testConfiguration.Host;
298354
builder[SnowflakeParameters.DATABASE] = testConfiguration.Database;
299355
builder[SnowflakeParameters.USERNAME] = testConfiguration.User;
356+
builder[SnowflakeParameters.MAX_TIMESTAMP_PRECISION] = testConfiguration.MaxTimestampPrecision;
357+
300358
if (authType == SnowflakeAuthentication.AuthJwt)
301359
{
302360
string privateKey = testConfiguration.Authentication.SnowflakeJwt!.PrivateKey;

‎csharp/test/Drivers/Interop/Snowflake/DriverTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,8 @@ public void CanGetObjectsAll()
364364
/// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a Special Character.
365365
/// </summary>
366366
[SkippableTheory, Order(3)]
367-
[InlineData(@"ADBCDEMO_DB",@"PUBLIC","MyIdentifier")]
368-
[InlineData(@"ADBCDEMO'DB", @"PUBLIC'SCHEMA","my.identifier")]
367+
[InlineData(@"ADBCDEMO_DB", @"PUBLIC", "MyIdentifier")]
368+
[InlineData(@"ADBCDEMO'DB", @"PUBLIC'SCHEMA", "my.identifier")]
369369
[InlineData(@"ADBCDEM""DB", @"PUBLIC""SCHEMA", "my.identifier")]
370370
[InlineData(@"ADBCDEMO_DB", @"PUBLIC", "my identifier")]
371371
[InlineData(@"ADBCDEMO_DB", @"PUBLIC", "My 'Identifier'")]

‎csharp/test/Drivers/Interop/Snowflake/SnowflakeData.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public static SampleDataBuilder GetSampleData()
107107
new ColumnNetTypeArrowTypeValue("TIMESTAMPNTZTYPE", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023,7,28, 12,34,56), TimeSpan.Zero)),
108108
new ColumnNetTypeArrowTypeValue("TIMESTAMPTZTYPE", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023,7,28, 19,34,56), TimeSpan.Zero)),
109109
}
110-
});
110+
});
111111

112112
// null data
113113
sampleDataBuilder.Samples.Add(

‎csharp/test/Drivers/Interop/Snowflake/SnowflakeTestConfiguration.cs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,17 @@ internal class SnowflakeTestConfiguration : TestConfiguration
7373
public string Warehouse { get; set; } = string.Empty;
7474

7575
/// <summary>
76-
/// The Snowflake use high precision
76+
/// The Snowflake setting to use high precision
7777
/// </summary>
7878
[JsonPropertyName("useHighPrecision")]
7979
public bool UseHighPrecision { get; set; } = true;
8080

81+
/// <summary>
82+
/// The Snowflake setting indicate the maximum timestamp precision.
83+
/// </summary>
84+
[JsonPropertyName("maxTimestampPrecision")]
85+
public string MaxTimestampPrecision { get; set; } = "nanoseconds";
86+
8187
/// <summary>
8288
/// The snowflake Authentication
8389
/// </summary>

‎csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ internal class SnowflakeParameters
4141
public const string PKCS8_VALUE = "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_value";
4242
public const string PKCS8_PASS = "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_password";
4343
public const string USE_HIGH_PRECISION = "adbc.snowflake.sql.client_option.use_high_precision";
44+
public const string MAX_TIMESTAMP_PRECISION = "adbc.snowflake.sql.client_option.max_timestamp_precision";
45+
}
46+
47+
public class SnowflakeConstants
48+
{
49+
public const string OptionValueNanoseconds = "nanoseconds";
50+
public const string OptionValueNanosecondsNoOverflow = "nanoseconds_error_on_overflow";
51+
public const string OptionValueMicroseconds = "microseconds";
4452
}
4553

4654
internal class SnowflakeTestingUtils
@@ -205,7 +213,8 @@ internal static string[] GetQueries(SnowflakeTestConfiguration testConfiguration
205213
/// <param name="value"></param>
206214
internal static void AssertContainsAll(string[]? expectedTexts, string value)
207215
{
208-
if (expectedTexts == null) { return; };
216+
if (expectedTexts == null) { return; }
217+
209218
foreach (string text in expectedTexts)
210219
{
211220
Assert.Contains(text, value);

‎docs/source/driver/snowflake.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,14 @@ These options map 1:1 with the Snowflake `Config object <https://pkg.go.dev/gith
469469
non-zero scaled columns will be returned as ``Float64`` typed Arrow columns.
470470
The default is ``true``.
471471

472+
``adbc.snowflake.sql.client_option.max_timestamp_precision``
473+
Controls the behavior of Timestamp values with Nanosecond precision. Native Go behavior
474+
is these values will overflow to an unpredictable value when the year is before year 1677 or after 2262.
475+
This option can control the behavior of the `timestamp_ltz`, `timestamp_ntz`, and `timestamp_tz` types.
476+
Valid values are
477+
- ``nanoseconds``: Use default behavior for nanoseconds.
478+
- ``nanoseconds_error_on_overflow``: Throws an error when the value will overflow to enforce integrity of the data.
479+
- ``microseconds``: Limits the max Timestamp precision to microseconds, which is safe for all values.
472480

473481
Metadata
474482
--------

‎go/adbc/driver/snowflake/connection.go

Lines changed: 43 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,9 @@ type connectionImpl struct {
7373
db *databaseImpl
7474
ctor driver.Connector
7575

76-
activeTransaction bool
77-
useHighPrecision bool
76+
activeTransaction bool
77+
useHighPrecision bool
78+
maxTimestampPrecision MaxTimestampPrecision
7879
}
7980

8081
func escapeSingleQuoteForLike(arg string) string {
@@ -483,11 +484,23 @@ func (c *connectionImpl) toArrowField(columnInfo driverbase.ColumnInfo) arrow.Fi
483484
case "DATETIME":
484485
fallthrough
485486
case "TIMESTAMP", "TIMESTAMP_NTZ":
486-
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond}
487+
if c.maxTimestampPrecision == Microseconds {
488+
field.Type = &arrow.TimestampType{Unit: arrow.Microsecond}
489+
} else {
490+
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond}
491+
}
487492
case "TIMESTAMP_LTZ":
488-
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()}
493+
if c.maxTimestampPrecision == Microseconds {
494+
field.Type = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: loc.String()}
495+
} else {
496+
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()}
497+
}
489498
case "TIMESTAMP_TZ":
490-
field.Type = arrow.FixedWidthTypes.Timestamp_ns
499+
if c.maxTimestampPrecision == Microseconds {
500+
field.Type = arrow.FixedWidthTypes.Timestamp_us
501+
} else {
502+
field.Type = arrow.FixedWidthTypes.Timestamp_ns
503+
}
491504
case "GEOGRAPHY":
492505
fallthrough
493506
case "GEOMETRY":
@@ -502,7 +515,7 @@ func (c *connectionImpl) toArrowField(columnInfo driverbase.ColumnInfo) arrow.Fi
502515
return field
503516
}
504517

505-
func descToField(name, typ, isnull, primary string, comment sql.NullString) (field arrow.Field, err error) {
518+
func descToField(name, typ, isnull, primary string, comment sql.NullString, maxTimestampPrecision MaxTimestampPrecision) (field arrow.Field, err error) {
506519
field.Name = strings.ToLower(name)
507520
if isnull == "Y" {
508521
field.Nullable = true
@@ -573,11 +586,23 @@ func descToField(name, typ, isnull, primary string, comment sql.NullString) (fie
573586
case "DATETIME":
574587
fallthrough
575588
case "TIMESTAMP", "TIMESTAMP_NTZ":
576-
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond}
589+
if maxTimestampPrecision == Microseconds {
590+
field.Type = &arrow.TimestampType{Unit: arrow.Microsecond}
591+
} else {
592+
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond}
593+
}
577594
case "TIMESTAMP_LTZ":
578-
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()}
595+
if maxTimestampPrecision == Microseconds {
596+
field.Type = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: loc.String()}
597+
} else {
598+
field.Type = &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: loc.String()}
599+
}
579600
case "TIMESTAMP_TZ":
580-
field.Type = arrow.FixedWidthTypes.Timestamp_ns
601+
if maxTimestampPrecision == Microseconds {
602+
field.Type = arrow.FixedWidthTypes.Timestamp_us
603+
} else {
604+
field.Type = arrow.FixedWidthTypes.Timestamp_ns
605+
}
581606
default:
582607
err = adbc.Error{
583608
Msg: fmt.Sprintf("Snowflake Data Type %s not implemented", typ),
@@ -669,7 +694,7 @@ func (c *connectionImpl) GetTableSchema(ctx context.Context, catalog *string, db
669694
return nil, errToAdbcErr(adbc.StatusIO, err)
670695
}
671696

672-
f, err := descToField(name, typ, isnull, primary, comment)
697+
f, err := descToField(name, typ, isnull, primary, comment, c.maxTimestampPrecision)
673698
if err != nil {
674699
return nil, err
675700
}
@@ -713,13 +738,14 @@ func (c *connectionImpl) NewStatement() (adbc.Statement, error) {
713738
defaultIngestOptions := DefaultIngestOptions()
714739
stmtBase := driverbase.NewStatementImplBase(c.Base(), c.ErrorHelper)
715740
stmt := &statement{
716-
StatementImplBase: stmtBase,
717-
alloc: c.db.Alloc,
718-
cnxn: c,
719-
queueSize: defaultStatementQueueSize,
720-
prefetchConcurrency: defaultPrefetchConcurrency,
721-
useHighPrecision: c.useHighPrecision,
722-
ingestOptions: defaultIngestOptions,
741+
StatementImplBase: stmtBase,
742+
alloc: c.db.Alloc,
743+
cnxn: c,
744+
queueSize: defaultStatementQueueSize,
745+
prefetchConcurrency: defaultPrefetchConcurrency,
746+
useHighPrecision: c.useHighPrecision,
747+
maxTimestampPrecision: c.maxTimestampPrecision,
748+
ingestOptions: defaultIngestOptions,
723749
}
724750
return driverbase.NewStatement(stmt), nil
725751
}

‎go/adbc/driver/snowflake/driver.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,16 @@ const (
7373
// with a scale of 0 will be returned as Int64 columns, and a non-zero
7474
// scale will return a Float64 column.
7575
OptionUseHighPrecision = "adbc.snowflake.sql.client_option.use_high_precision"
76+
// OptionMaxTimestampPrecision controls the behavior of Timestamp values with
77+
// Nanosecond precision. Native Go behavior is these values will overflow to an
78+
// unpredictable value when the year is before year 1677 or after 2262. This option
79+
// can control the behavior of the `timestamp_ltz`, `timestamp_ntz`, and `timestamp_tz` types.
80+
//
81+
// Valid values are
82+
// `nanoseconds`: Use default behavior for nanoseconds.
83+
// `nanoseconds_error_on_overflow`: Throws an error when the value will overflow to enforce integrity of the data.
84+
// `microseconds`: Limits the max Timestamp precision to microseconds, which is safe for all values.
85+
OptionMaxTimestampPrecision = "adbc.snowflake.sql.client_option.max_timestamp_precision"
7686

7787
OptionApplicationName = "adbc.snowflake.sql.client_option.app_name"
7888
OptionSSLSkipVerify = "adbc.snowflake.sql.client_option.tls_skip_verify"
@@ -117,6 +127,13 @@ const (
117127
OptionValueAuthJwt = "auth_jwt"
118128
// use a username and password with mfa
119129
OptionValueAuthUserPassMFA = "auth_mfa"
130+
131+
// Use default behavior for nanoseconds.
132+
OptionValueNanoseconds = "nanoseconds"
133+
// throws an error when the value will overflow to enforce integrity of the data.
134+
OptionValueNanosecondsNoOverflow = "nanoseconds_error_on_overflow"
135+
// use a max of microseconds precision for timestamps
136+
OptionValueMicroseconds = "microseconds"
120137
)
121138

122139
var (
@@ -255,9 +272,10 @@ func (d *driverImpl) NewDatabaseWithOptionsContext(
255272
defaultAppName := "[ADBC][Go-" + driverVersion + "]"
256273

257274
db := &databaseImpl{
258-
DatabaseImplBase: dbBase,
259-
useHighPrecision: true,
260-
defaultAppName: defaultAppName,
275+
DatabaseImplBase: dbBase,
276+
useHighPrecision: true,
277+
defaultAppName: defaultAppName,
278+
maxTimestampPrecision: Nanoseconds,
261279
}
262280
if err := db.SetOptions(opts); err != nil {
263281
return nil, err

0 commit comments

Comments
 (0)