Merge pull request #1063 from penghuo/hp/main/2.4merge

Merge 2.x to main after 2.4 release

Author: penghuo

.github/workflows/sql-odbc-main.yml

@@ -13,14 +13,15 @@ env:
   CI_OUTPUT_PATH: "sql-odbc/ci-output"
   ODBC_LIB_PATH: "./build/odbc/lib"
   ODBC_BIN_PATH: "./build/odbc/bin"
-  ODBC_BUILD_PATH: "./build/odbc/build"
-  AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
+  ODBC_BUILD_PATH: "./build/odbc/cmake"
+  VCPKG_X64_INSTALL_PATH: ".\\src\\vcpkg_installed\\x64-windows"
+  VCPKG_X86_INSTALL_PATH: ".\\src\\vcpkg_installed\\x86-windows"
 
 # Tests are disabled (commented out) in all jobs because they are fail and/or outdated
 # Keeping them for the brighten future when we can re-activate them
 jobs:    
   build-mac:
-    runs-on: macos-10.15
+    runs-on: macos-12
     defaults:
       run:
         working-directory: sql-odbc
@@ -103,7 +104,7 @@ jobs:
     - name: build-installer
       if: success()
       run: |
-        .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
+        .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:VCPKG_X86_INSTALL_PATH
     #- name: test
     #  run: |
     #    cp .\\libraries\\VisualLeakDetector\\bin32\\*.* .\\bin32\\Release
@@ -148,7 +149,7 @@ jobs:
     - name: build-installer
       if: success()
       run: |
-        .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH
+        .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:VCPKG_X64_INSTALL_PATH
     #- name: test
     #  run: |
     #    cp .\\libraries\\VisualLeakDetector\\bin64\\*.* .\\bin64\\Release

.github/workflows/sql-test-and-build-workflow.yml

@@ -25,10 +25,10 @@ jobs:
       matrix:
         entry:
           - { os: ubuntu-latest, java: 11 }
-          - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc}
+          - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows }
           - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
           - { os: ubuntu-latest, java: 17 }
-          - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
+          - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows }
           - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
     runs-on: ${{ matrix.entry.os }}
 

.github/workflows/sql-test-workflow.yml

@@ -2,6 +2,13 @@ name: SQL Plugin Tests
 
 on:
   workflow_dispatch:
+    inputs:
+      name:
+        required: false
+        type: string
+
+run-name:
+  ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }}
 
 jobs:
   build:
@@ -64,10 +71,10 @@ jobs:
 
     - name: Verify test results
       run: |
-        if [[ -e failures.log ]]
+        if [[ -e report.log ]]
         then
           echo "## FAILED TESTS :facepalm::warning::bangbang:" >> $GITHUB_STEP_SUMMARY
-          cat failures.log >> $GITHUB_STEP_SUMMARY
+          cat report.log >> $GITHUB_STEP_SUMMARY
           exit 1
         fi
 

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -77,4 +77,3 @@ jobs:
         with:
           name: workbench-${{ matrix.os }}
           path: ../OpenSearch-Dashboards/plugins/workbench/build
-          

.gitignore

@@ -42,3 +42,7 @@ gen
 .DS_Store
 
 /artifacts/
+/.pid.lock
+/.prom.pid.lock
+
+.java-version

DEVELOPER_GUIDE.rst

@@ -141,6 +141,8 @@ The plugin codebase is in standard layout of Gradle project::
    ├── core
    ├── doctest
    ├── opensearch
+   ├── filesystem
+   ├── prometheus
    ├── integ-test
    ├── legacy
    ├── plugin
@@ -159,6 +161,8 @@ Here are sub-folders (Gradle modules) for plugin source code:
 - ``ppl``: PPL language processor.
 - ``core``: core query engine.
 - ``opensearch``: OpenSearch storage engine.
+- ``prometheus``: Prometheus storage engine.
+- ``filesystem``: File System storage engine (in development).
 - ``protocol``: request/response protocol formatter.
 - ``common``: common util code.
 - ``integ-test``: integration and comparison test.

NOTICE

@@ -11,6 +11,9 @@ Foundation (http://www.apache.org/).
 This product includes software developed by
 Joda.org (http://www.joda.org/).
 
+This product includes software developed by
+Kraken (https://github.com/thekrakken/java-grok).
+
 This project is based on the Apache 2.0-licensed elasticsearch-sql project (https://github.com/NLPchina/elasticsearch-sql):
 
 Copyright 2014 omershelef

build.gradle

@@ -8,18 +8,46 @@ buildscript {
     ext {
         opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
         spring_version = "5.3.22"
-        jackson_version = "2.13.4"
+        jackson_version = "2.14.1"
+        jackson_databind_version = "2.14.1"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
+        prometheus_binary_version = "2.37.2"
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }
         if (isSnapshot) {
             // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT (opensearch_build)
             opensearch_build += "-SNAPSHOT"
         }
+        getArchType = {
+            if (System.getProperty("os.arch").startsWith("x") || System.getProperty("os.arch").startsWith("amd")) {
+                return "amd64"
+            }
+            else {
+                return "arm64"
+            }
+        }
+        getOSFamilyType = {
+            def os = org.gradle.internal.os.OperatingSystem.current();
+            if (os.isMacOsX()) {
+                return "darwin"
+            }
+            else if(os.isLinux()){
+                return "linux"
+            }
+            else if(os.isWindows()) {
+                return "windows"
+            }
+            else {
+                return os.getFamilyName().toString()
+            }
+        }
+        getPrometheusBinaryLocation = { ->
+            return "https://github.com/prometheus/prometheus/releases/download/v${prometheus_binary_version}/prometheus-${prometheus_binary_version}."+ getOSFamilyType() + "-" + getArchType() + ".tar.gz"
+        }
     }
 
     repositories {

common/build.gradle

@@ -35,6 +35,10 @@ dependencies {
     api "org.antlr:antlr4-runtime:4.7.1"
     api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1'
+    api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
+    testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'
+    testImplementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
+    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
 }

common/src/main/java/org/opensearch/sql/common/grok/Converter.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.common.grok;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Convert String argument to the right type.
+ */
+public class Converter {
+
+  public enum Type {
+    BYTE(Byte::valueOf),
+    BOOLEAN(Boolean::valueOf),
+    SHORT(Short::valueOf),
+    INT(Integer::valueOf, "integer"),
+    LONG(Long::valueOf),
+    FLOAT(Float::valueOf),
+    DOUBLE(Double::valueOf),
+    DATETIME(new DateConverter(), "date"),
+    STRING(v -> v, "text");
+
+    public final IConverter<? extends Object> converter;
+    public final List<String> aliases;
+
+    Type(IConverter<? extends Object> converter, String... aliases) {
+      this.converter = converter;
+      this.aliases = Arrays.asList(aliases);
+    }
+  }
+
+  private static final Pattern SPLITTER = Pattern.compile("[:;]");
+
+  private static final Map<String, Type> TYPES =
+      Arrays.stream(Type.values())
+          .collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t));
+
+  private static final Map<String, Type> TYPE_ALIASES =
+      Arrays.stream(Type.values())
+          .flatMap(type -> type.aliases.stream()
+              .map(alias -> new AbstractMap.SimpleEntry<>(alias, type)))
+          .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+  private static Type getType(String key) {
+    key = key.toLowerCase();
+    Type type = TYPES.getOrDefault(key, TYPE_ALIASES.get(key));
+    if (type == null) {
+      throw new IllegalArgumentException("Invalid data type :" + key);
+    }
+    return type;
+  }
+
+  /**
+   * getConverters.
+   */
+  public static Map<String, IConverter<? extends Object>>
+      getConverters(Collection<String> groupNames, Object... params) {
+    return groupNames.stream()
+        .filter(Converter::containsDelimiter)
+        .collect(Collectors.toMap(Function.identity(), key -> {
+          String[] list = splitGrokPattern(key);
+          IConverter<? extends Object> converter = getType(list[1]).converter;
+          if (list.length == 3) {
+            converter = converter.newConverter(list[2], params);
+          }
+          return converter;
+        }));
+  }
+
+  /**
+   * getGroupTypes.
+   */
+  public static Map<String, Type> getGroupTypes(Collection<String> groupNames) {
+    return groupNames.stream()
+        .filter(Converter::containsDelimiter)
+        .map(Converter::splitGrokPattern)
+        .collect(Collectors.toMap(
+            l -> l[0],
+            l -> getType(l[1])
+        ));
+  }
+
+  public static String extractKey(String key) {
+    return splitGrokPattern(key)[0];
+  }
+
+  private static boolean containsDelimiter(String string) {
+    return string.indexOf(':') >= 0 || string.indexOf(';') >= 0;
+  }
+
+  private static String[] splitGrokPattern(String string) {
+    return SPLITTER.split(string, 3);
+  }
+
+  interface IConverter<T> {
+
+    T convert(String value);
+
+    default IConverter<T> newConverter(String param, Object... params) {
+      return this;
+    }
+  }
+
+
+  static class DateConverter implements IConverter<Instant> {
+
+    private final DateTimeFormatter formatter;
+    private final ZoneId timeZone;
+
+    public DateConverter() {
+      this.formatter = DateTimeFormatter.ISO_DATE_TIME;
+      this.timeZone = ZoneOffset.UTC;
+    }
+
+    private DateConverter(DateTimeFormatter formatter, ZoneId timeZone) {
+      this.formatter = formatter;
+      this.timeZone = timeZone;
+    }
+
+    @Override
+    public Instant convert(String value) {
+      TemporalAccessor dt = formatter
+          .parseBest(value.trim(), ZonedDateTime::from, LocalDateTime::from, OffsetDateTime::from,
+              Instant::from,
+              LocalDate::from);
+      if (dt instanceof ZonedDateTime) {
+        return ((ZonedDateTime) dt).toInstant();
+      } else if (dt instanceof LocalDateTime) {
+        return ((LocalDateTime) dt).atZone(timeZone).toInstant();
+      } else if (dt instanceof OffsetDateTime) {
+        return ((OffsetDateTime) dt).atZoneSameInstant(timeZone).toInstant();
+      } else if (dt instanceof Instant) {
+        return ((Instant) dt);
+      } else if (dt instanceof LocalDate) {
+        return ((LocalDate) dt).atStartOfDay(timeZone).toInstant();
+      } else {
+        return null;
+      }
+    }
+
+    @Override
+    public DateConverter newConverter(String param, Object... params) {
+      if (!(params.length == 1 && params[0] instanceof ZoneId)) {
+        throw new IllegalArgumentException("Invalid parameters");
+      }
+      return new DateConverter(DateTimeFormatter.ofPattern(param), (ZoneId) params[0]);
+    }
+  }
+}