Skip to content

Commit f33421b

Browse files
Yury-FridlyandMitchellGale
Yury-Fridlyand
authored and
MitchellGale
committed
Add datetime functions FROM_UNIXTIME and UNIX_TIMESTAMP (opensearch-project#835)
* Add datetime functions `FROM_UNIXTIME` and `UNIX_TIMESTAMP` (#114) * Add implementation for `FROM_UNIXTIME` and `UNIX_TIMESTAMP` functions, UT and IT. Signed-off-by: Yury-Fridlyand <[email protected]> * Collent all DateTime formatters into one place. Signed-off-by: Yury-Fridlyand <[email protected]> * Rename `DateFormatters` -> `DateTimeFormatters`. Signed-off-by: Yury-Fridlyand <[email protected]> Signed-off-by: Yury-Fridlyand <[email protected]>
1 parent 705ad13 commit f33421b

File tree

21 files changed

+1064
-154
lines changed

21 files changed

+1064
-154
lines changed

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66

77
package org.opensearch.sql.data.model;
88

9+
import static org.opensearch.sql.utils.DateTimeFormatters.TIME_FORMATTER_VARIABLE_NANOS;
10+
911
import java.time.LocalTime;
1012
import java.time.format.DateTimeFormatter;
11-
import java.time.format.DateTimeFormatterBuilder;
1213
import java.time.format.DateTimeParseException;
13-
import java.time.temporal.ChronoField;
1414
import java.util.Objects;
1515
import lombok.RequiredArgsConstructor;
1616
import org.opensearch.sql.data.type.ExprCoreType;
@@ -24,27 +24,12 @@
2424
public class ExprTimeValue extends AbstractExprValue {
2525
private final LocalTime time;
2626

27-
private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
28-
private static final int MIN_FRACTION_SECONDS = 0;
29-
private static final int MAX_FRACTION_SECONDS = 9;
30-
31-
static {
32-
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
33-
.appendPattern("HH:mm:ss")
34-
.appendFraction(
35-
ChronoField.NANO_OF_SECOND,
36-
MIN_FRACTION_SECONDS,
37-
MAX_FRACTION_SECONDS,
38-
true)
39-
.toFormatter();
40-
}
41-
4227
/**
4328
* Constructor.
4429
*/
4530
public ExprTimeValue(String time) {
4631
try {
47-
this.time = LocalTime.parse(time, FORMATTER_VARIABLE_NANOS);
32+
this.time = LocalTime.parse(time, TIME_FORMATTER_VARIABLE_NANOS);
4833
} catch (DateTimeParseException e) {
4934
throw new SemanticCheckException(String.format("time:%s in unsupported format, please use "
5035
+ "HH:mm:ss[.SSSSSSSSS]", time));

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@
66

77
package org.opensearch.sql.data.model;
88

9+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS;
10+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_WITHOUT_NANO;
11+
912
import java.time.Instant;
1013
import java.time.LocalDate;
1114
import java.time.LocalDateTime;
1215
import java.time.LocalTime;
1316
import java.time.ZoneId;
14-
import java.time.format.DateTimeFormatter;
15-
import java.time.format.DateTimeFormatterBuilder;
1617
import java.time.format.DateTimeParseException;
17-
import java.time.temporal.ChronoField;
1818
import java.time.temporal.ChronoUnit;
1919
import java.util.Objects;
2020
import lombok.RequiredArgsConstructor;
@@ -31,34 +31,15 @@ public class ExprTimestampValue extends AbstractExprValue {
3131
* todo. only support UTC now.
3232
*/
3333
private static final ZoneId ZONE = ZoneId.of("UTC");
34-
/**
35-
* todo. only support timestamp in format yyyy-MM-dd HH:mm:ss.
36-
*/
37-
private static final DateTimeFormatter FORMATTER_WITHOUT_NANO = DateTimeFormatter
38-
.ofPattern("yyyy-MM-dd HH:mm:ss");
39-
private final Instant timestamp;
4034

41-
private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
42-
private static final int MIN_FRACTION_SECONDS = 0;
43-
private static final int MAX_FRACTION_SECONDS = 9;
44-
45-
static {
46-
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
47-
.appendPattern("yyyy-MM-dd HH:mm:ss")
48-
.appendFraction(
49-
ChronoField.NANO_OF_SECOND,
50-
MIN_FRACTION_SECONDS,
51-
MAX_FRACTION_SECONDS,
52-
true)
53-
.toFormatter();
54-
}
35+
private final Instant timestamp;
5536

5637
/**
5738
* Constructor.
5839
*/
5940
public ExprTimestampValue(String timestamp) {
6041
try {
61-
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_NANOS)
42+
this.timestamp = LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS)
6243
.atZone(ZONE)
6344
.toInstant();
6445
} catch (DateTimeParseException e) {
@@ -70,9 +51,9 @@ public ExprTimestampValue(String timestamp) {
7051

7152
@Override
7253
public String value() {
73-
return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE)
54+
return timestamp.getNano() == 0 ? DATE_TIME_FORMATTER_WITHOUT_NANO.withZone(ZONE)
7455
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
75-
: FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp);
56+
: DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp);
7657
}
7758

7859
@Override

‎core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,38 @@
1818
import static org.opensearch.sql.expression.function.FunctionDSL.define;
1919
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
2020
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
21+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_LONG_YEAR;
22+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SHORT_YEAR;
23+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_LONG_YEAR;
24+
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR;
2125

2226
import java.math.BigDecimal;
2327
import java.math.RoundingMode;
28+
import java.text.DecimalFormat;
29+
import java.time.Instant;
2430
import java.time.LocalDate;
2531
import java.time.LocalDateTime;
2632
import java.time.LocalTime;
33+
import java.time.ZoneId;
34+
import java.time.ZoneOffset;
2735
import java.time.format.DateTimeFormatter;
36+
import java.time.format.DateTimeParseException;
2837
import java.time.format.TextStyle;
2938
import java.util.Locale;
3039
import java.util.concurrent.TimeUnit;
3140
import javax.annotation.Nullable;
3241
import lombok.experimental.UtilityClass;
3342
import org.opensearch.sql.data.model.ExprDateValue;
3443
import org.opensearch.sql.data.model.ExprDatetimeValue;
44+
import org.opensearch.sql.data.model.ExprDoubleValue;
3545
import org.opensearch.sql.data.model.ExprIntegerValue;
3646
import org.opensearch.sql.data.model.ExprLongValue;
3747
import org.opensearch.sql.data.model.ExprNullValue;
3848
import org.opensearch.sql.data.model.ExprStringValue;
3949
import org.opensearch.sql.data.model.ExprTimeValue;
4050
import org.opensearch.sql.data.model.ExprTimestampValue;
4151
import org.opensearch.sql.data.model.ExprValue;
52+
import org.opensearch.sql.data.type.ExprCoreType;
4253
import org.opensearch.sql.expression.function.BuiltinFunctionName;
4354
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
4455
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
@@ -56,6 +67,10 @@ public class DateTimeFunction {
5667
// The number of days from year zero to year 1970.
5768
private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L);
5869

70+
// MySQL doesn't process any datetime/timestamp values which are greater than
71+
// 32536771199.999999, or equivalent '3001-01-18 23:59:59.999999' UTC
72+
private static final Double MYSQL_MAX_TIMESTAMP = 32536771200d;
73+
5974
/**
6075
* Register Date and Time Functions.
6176
*
@@ -72,6 +87,7 @@ public void register(BuiltinFunctionRepository repository) {
7287
repository.register(dayOfWeek());
7388
repository.register(dayOfYear());
7489
repository.register(from_days());
90+
repository.register(from_unixtime());
7591
repository.register(hour());
7692
repository.register(makedate());
7793
repository.register(maketime());
@@ -87,6 +103,7 @@ public void register(BuiltinFunctionRepository repository) {
87103
repository.register(timestamp());
88104
repository.register(date_format());
89105
repository.register(to_days());
106+
repository.register(unix_timestamp());
90107
repository.register(week());
91108
repository.register(year());
92109

@@ -313,6 +330,13 @@ private DefaultFunctionResolver from_days() {
313330
impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG));
314331
}
315332

333+
private FunctionResolver from_unixtime() {
334+
return define(BuiltinFunctionName.FROM_UNIXTIME.getName(),
335+
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE),
336+
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat),
337+
STRING, DOUBLE, STRING));
338+
}
339+
316340
/**
317341
* HOUR(STRING/TIME/DATETIME/TIMESTAMP). return the hour value for time.
318342
*/
@@ -461,6 +485,16 @@ private DefaultFunctionResolver to_days() {
461485
impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME));
462486
}
463487

488+
private FunctionResolver unix_timestamp() {
489+
return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(),
490+
impl(DateTimeFunction::unixTimeStamp, LONG),
491+
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE),
492+
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATETIME),
493+
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, TIMESTAMP),
494+
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DOUBLE)
495+
);
496+
}
497+
464498
/**
465499
* WEEK(DATE[,mode]). return the week number for date.
466500
*/
@@ -601,6 +635,35 @@ private ExprValue exprFromDays(ExprValue exprValue) {
601635
return new ExprDateValue(LocalDate.ofEpochDay(exprValue.longValue() - DAYS_0000_TO_1970));
602636
}
603637

638+
private ExprValue exprFromUnixTime(ExprValue time) {
639+
if (0 > time.doubleValue()) {
640+
return ExprNullValue.of();
641+
}
642+
// According to MySQL documentation:
643+
// effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC.
644+
// Regardless of platform or version, a greater value for first argument than the effective
645+
// maximum returns 0.
646+
if (MYSQL_MAX_TIMESTAMP <= time.doubleValue()) {
647+
return ExprNullValue.of();
648+
}
649+
return new ExprDatetimeValue(exprFromUnixTimeImpl(time));
650+
}
651+
652+
private LocalDateTime exprFromUnixTimeImpl(ExprValue time) {
653+
return LocalDateTime.ofInstant(
654+
Instant.ofEpochSecond((long)Math.floor(time.doubleValue())),
655+
ZoneId.of("UTC"))
656+
.withNano((int)((time.doubleValue() % 1) * 1E9));
657+
}
658+
659+
private ExprValue exprFromUnixTimeFormat(ExprValue time, ExprValue format) {
660+
var value = exprFromUnixTime(time);
661+
if (value.equals(ExprNullValue.of())) {
662+
return ExprNullValue.of();
663+
}
664+
return DateTimeFormatterUtil.getFormattedDate(value, format);
665+
}
666+
604667
/**
605668
* Hour implementation for ExprValue.
606669
*
@@ -803,6 +866,79 @@ private ExprValue exprWeek(ExprValue date, ExprValue mode) {
803866
CalendarLookup.getWeekNumber(mode.integerValue(), date.dateValue()));
804867
}
805868

869+
private ExprValue unixTimeStamp() {
870+
return new ExprLongValue(Instant.now().getEpochSecond());
871+
}
872+
873+
private ExprValue unixTimeStampOf(ExprValue value) {
874+
var res = unixTimeStampOfImpl(value);
875+
if (res == null) {
876+
return ExprNullValue.of();
877+
}
878+
if (res < 0) {
879+
// According to MySQL returns 0 if year < 1970, don't return negative values as java does.
880+
return new ExprDoubleValue(0);
881+
}
882+
if (res >= MYSQL_MAX_TIMESTAMP) {
883+
// Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec)
884+
return new ExprDoubleValue(0);
885+
}
886+
return new ExprDoubleValue(res);
887+
}
888+
889+
private Double unixTimeStampOfImpl(ExprValue value) {
890+
// Also, according to MySQL documentation:
891+
// The date argument may be a DATE, DATETIME, or TIMESTAMP ...
892+
switch ((ExprCoreType)value.type()) {
893+
case DATE: return value.dateValue().toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
894+
case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC)
895+
+ value.datetimeValue().getNano() / 1E9;
896+
case TIMESTAMP: return value.timestampValue().getEpochSecond()
897+
+ value.timestampValue().getNano() / 1E9;
898+
default:
899+
// ... or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format.
900+
// If the argument includes a time part, it may optionally include a fractional
901+
// seconds part.
902+
903+
var format = new DecimalFormat("0.#");
904+
format.setMinimumFractionDigits(0);
905+
format.setMaximumFractionDigits(6);
906+
String input = format.format(value.doubleValue());
907+
double fraction = 0;
908+
if (input.contains(".")) {
909+
// Keeping fraction second part and adding it to the result, don't parse it
910+
// Because `toEpochSecond` returns only `long`
911+
// input = 12345.6789 becomes input = 12345 and fraction = 0.6789
912+
fraction = value.doubleValue() - Math.round(Math.ceil(value.doubleValue()));
913+
input = input.substring(0, input.indexOf('.'));
914+
}
915+
try {
916+
var res = LocalDateTime.parse(input, DATE_TIME_FORMATTER_SHORT_YEAR);
917+
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
918+
} catch (DateTimeParseException ignored) {
919+
// nothing to do, try another format
920+
}
921+
try {
922+
var res = LocalDateTime.parse(input, DATE_TIME_FORMATTER_LONG_YEAR);
923+
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
924+
} catch (DateTimeParseException ignored) {
925+
// nothing to do, try another format
926+
}
927+
try {
928+
var res = LocalDate.parse(input, DATE_FORMATTER_SHORT_YEAR);
929+
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
930+
} catch (DateTimeParseException ignored) {
931+
// nothing to do, try another format
932+
}
933+
try {
934+
var res = LocalDate.parse(input, DATE_FORMATTER_LONG_YEAR);
935+
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
936+
} catch (DateTimeParseException ignored) {
937+
return null;
938+
}
939+
}
940+
}
941+
806942
/**
807943
* Week for date implementation for ExprValue.
808944
* When mode is not specified default value mode 0 is used for default_week_format.

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public enum BuiltinFunctionName {
6767
DAYOFWEEK(FunctionName.of("dayofweek")),
6868
DAYOFYEAR(FunctionName.of("dayofyear")),
6969
FROM_DAYS(FunctionName.of("from_days")),
70+
FROM_UNIXTIME(FunctionName.of("from_unixtime")),
7071
HOUR(FunctionName.of("hour")),
7172
MAKEDATE(FunctionName.of("makedate")),
7273
MAKETIME(FunctionName.of("maketime")),
@@ -82,6 +83,7 @@ public enum BuiltinFunctionName {
8283
TIMESTAMP(FunctionName.of("timestamp")),
8384
DATE_FORMAT(FunctionName.of("date_format")),
8485
TO_DAYS(FunctionName.of("to_days")),
86+
UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")),
8587
WEEK(FunctionName.of("week")),
8688
YEAR(FunctionName.of("year")),
8789
// `now`-like functions

0 commit comments

Comments
 (0)