Merge pull request #830 from penghuo/backport/2.x

Merge main to 2.x

Author: penghuo

.github/workflows/draft-release-notes-workflow.yml

@@ -16,6 +16,6 @@ jobs:
         with:
           config-name: draft-release-notes-config.yml
           tag: (None)
-          version: 2.2.0.0
+          version: 2.3.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sql-odbc-release-workflow.yml

@@ -12,7 +12,7 @@ env:
   ODBC_BUILD_PATH: "./build/odbc/build"
   AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
   PLUGIN_NAME: opensearch-sql-odbc
-  OD_VERSION: 2.2.0.0
+  OD_VERSION: 2.3.0.0
 
 jobs:
   build-mac:

.github/workflows/sql-test-and-build-workflow.yml

@@ -22,6 +22,9 @@ jobs:
     - name: Build with Gradle
       run: ./gradlew build assemble
 
+    - name: Run backward compatibility tests
+      run: ./scripts/bwctest.sh
+
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds

.github/workflows/sql-workbench-release-workflow.yml

@@ -8,7 +8,7 @@ on:
 env: 
   PLUGIN_NAME: query-workbench-dashboards
   OPENSEARCH_VERSION: 'main'
-  OPENSEARCH_PLUGIN_VERSION: 2.2.0.0
+  OPENSEARCH_PLUGIN_VERSION: 2.3.0.0
 
 jobs:
 

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -5,7 +5,7 @@ on: [pull_request, push]
 env: 
   PLUGIN_NAME: query-workbench-dashboards
   OPENSEARCH_VERSION: 'main'
-  OPENSEARCH_PLUGIN_VERSION: 2.2.0.0
+  OPENSEARCH_PLUGIN_VERSION: 2.3.0.0
 
 jobs:
 

MAINTAINERS.md

@@ -9,4 +9,6 @@
 | Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon |
 | Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon |
 | Nick Knize | [nknize](https://github.com/nknize) | Amazon |
-| Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon |
+| Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon |
+| Max Ksyunz |  [MaxKsyunz](https://github.com/MaxKsyunz) | BitQuill |
+| Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | BitQuill |

README.md

@@ -4,6 +4,7 @@
 - [Code Summary](#code-summary)
 - [Highlights](#highlights)
 - [Documentation](#documentation)
+- [OpenSearch Forum](#forum)
 - [Contributing](#contributing)
 - [Attribution](#attribution)
 - [Code of Conduct](#code-of-conduct)
@@ -127,6 +128,10 @@ Recently we have been actively improving our query engine primarily for better c
 
 Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://opensearch.org/docs/latest/search-plugins/sql/index/) for detailed information on installing and configuring plugin.
 
+## Forum
+
+For additional help with the plugin, including questions about opening an issue, visit the OpenSearch [Forum](https://forum.opensearch.org/c/plugins/sql/8).
+
 ## Contributing
 
 See [developer guide](DEVELOPER_GUIDE.rst) and [how to contribute to this project](CONTRIBUTING.md).

bi-connectors/PowerBIConnector/src/OpenSearchProject.query.pq

@@ -7,13 +7,14 @@ shared MyExtension.UnitTest =
     Host = "localhost",
     Port = 9200,
     UseSSL = false,
+    HostnameVerification = false,
 
     facts =
     {        
         Fact("Connection Test",  
              7,                                    
              let
-                Source = OpenSearch.Contents(Host, Port, UseSSL),
+                Source = OpenSearchProject.Contents(Host, Port, UseSSL, HostnameVerification),
                 no_of_columns = Table.ColumnCount(Source)
              in
                 no_of_columns                
@@ -22,7 +23,7 @@ shared MyExtension.UnitTest =
             #table(type table [bool0 = logical],
                 { {null}, {false}, {true} }),
             let
-                Source = OpenSearch.Contents(Host, Port, UseSSL),
+                Source = OpenSearchProject.Contents(Host, Port, UseSSL, HostnameVerification),
                 calcs_null_null = Source{[Item="calcs",Schema=null,Catalog=null]}[Data],
                 grouped = Table.Group(calcs_null_null, {"bool0"}, {})
             in

build.gradle

@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT")
         spring_version = "5.3.22"
         jackson_version = "2.13.3"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")

core/build.gradle

@@ -45,6 +45,9 @@ dependencies {
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
     api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240'
     api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
+    api "com.fasterxml.jackson.core:jackson-core:${jackson_version}"
+    api "com.fasterxml.jackson.core:jackson-databind:${jackson_version}"
+    api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}"
     api project(':common')
 
     testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')
@@ -70,7 +73,7 @@ jacocoTestReport {
     afterEvaluate {
         classDirectories.setFrom(files(classDirectories.files.collect {
             fileTree(dir: it,
-                    exclude: ['**/ast/**'])
+                    exclude: ['**/ast/**', '**/catalog/model/**'])
         }))
     }
 }
@@ -80,7 +83,9 @@ jacocoTestCoverageVerification {
         rule {
             element = 'CLASS'
             excludes = [
-                    'org.opensearch.sql.utils.MLCommonsConstants'
+                    'org.opensearch.sql.utils.MLCommonsConstants',
+                    'org.opensearch.sql.utils.Constants',
+                    'org.opensearch.sql.catalog.model.*'
             ]
             limit {
                 counter = 'LINE'

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -55,6 +55,7 @@
 import org.opensearch.sql.ast.tree.Sort.SortOption;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
+import org.opensearch.sql.catalog.CatalogService;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.exception.SemanticCheckException;
@@ -81,7 +82,6 @@
 import org.opensearch.sql.planner.logical.LogicalRename;
 import org.opensearch.sql.planner.logical.LogicalSort;
 import org.opensearch.sql.planner.logical.LogicalValues;
-import org.opensearch.sql.storage.StorageEngine;
 import org.opensearch.sql.storage.Table;
 import org.opensearch.sql.utils.ParseUtils;
 
@@ -97,16 +97,16 @@ public class Analyzer extends AbstractNodeVisitor<LogicalPlan, AnalysisContext>
 
   private final NamedExpressionAnalyzer namedExpressionAnalyzer;
 
-  private final StorageEngine storageEngine;
+  private final CatalogService catalogService;
 
   /**
    * Constructor.
    */
   public Analyzer(
       ExpressionAnalyzer expressionAnalyzer,
-      StorageEngine storageEngine) {
+      CatalogService catalogService) {
     this.expressionAnalyzer = expressionAnalyzer;
-    this.storageEngine = storageEngine;
+    this.catalogService = catalogService;
     this.selectExpressionAnalyzer = new SelectExpressionAnalyzer(expressionAnalyzer);
     this.namedExpressionAnalyzer = new NamedExpressionAnalyzer(expressionAnalyzer);
   }
@@ -119,16 +119,33 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) {
   public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     context.push();
     TypeEnvironment curEnv = context.peek();
-    Table table = storageEngine.getTable(node.getTableName());
+    String catalogName = getCatalogName(node);
+    String tableName = getTableName(node);
+    if (catalogName != null && !catalogService.getCatalogs().contains(catalogName)) {
+      tableName = catalogName + "." + tableName;
+      catalogName = null;
+    }
+    Table table = catalogService
+        .getStorageEngine(catalogName)
+        .getTable(tableName);
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
 
     // Put index name or its alias in index namespace on type environment so qualifier
     // can be removed when analyzing qualified name. The value (expr type) here doesn't matter.
     curEnv.define(new Symbol(Namespace.INDEX_NAME, node.getTableNameOrAlias()), STRUCT);
 
-    return new LogicalRelation(node.getTableName());
+    return new LogicalRelation(tableName, table);
+  }
+
+  private String getTableName(Relation node) {
+    return node.getTableName();
   }
 
+  private String getCatalogName(Relation node) {
+    return node.getCatalogName();
+  }
+
+
   @Override
   public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext context) {
     LogicalPlan subquery = analyze(node.getChild().get(0), context);

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -7,6 +7,7 @@
 package org.opensearch.sql.ast.dsl;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import lombok.experimental.UtilityClass;
 import org.apache.commons.lang3.tuple.Pair;
@@ -71,6 +72,10 @@ public UnresolvedPlan relation(String tableName) {
     return new Relation(qualifiedName(tableName));
   }
 
+  public UnresolvedPlan relation(QualifiedName tableName) {
+    return new Relation(tableName);
+  }
+
   public UnresolvedPlan relation(String tableName, String alias) {
     return new Relation(qualifiedName(tableName), alias);
   }
@@ -114,7 +119,7 @@ public static UnresolvedPlan rename(UnresolvedPlan input, Map... maps) {
   /**
    * Initialize Values node by rows of literals.
    * @param values  rows in which each row is a list of literal values
-   * @return        Values node
+   * @return       Values node
    */
   @SafeVarargs
   public UnresolvedPlan values(List<Literal>... values) {
@@ -413,7 +418,8 @@ public static List<Argument> defaultTopArgs() {
   }
 
   public static RareTopN rareTopN(UnresolvedPlan input, CommandType commandType,
-      List<Argument> noOfResults, List<UnresolvedExpression> groupList, Field... fields) {
+                                  List<Argument> noOfResults, List<UnresolvedExpression> groupList,
+                                  Field... fields) {
     return new RareTopN(input, commandType, noOfResults, Arrays.asList(fields), groupList)
         .attach(input);
   }

core/src/main/java/org/opensearch/sql/ast/tree/Relation.java

@@ -15,6 +15,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.QualifiedName;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
 /**
@@ -46,17 +47,49 @@ public Relation(UnresolvedExpression tableName, String alias) {
   /**
    * Get original table name. Unwrap and get name if table name expression
    * is actually an Alias.
-   * @return    table name
+   * In case of federated queries we are assuming single table.
+   *
+   * @return table name
    */
   public String getTableName() {
+    if (tableName.size() == 1 && ((QualifiedName) tableName.get(0)).first().isPresent()) {
+      return ((QualifiedName) tableName.get(0)).rest().toString();
+    }
+    return tableName.stream()
+        .map(UnresolvedExpression::toString)
+        .collect(Collectors.joining(COMMA));
+  }
+
+  /**
+   * Get Catalog Name if present. Since in the initial phase we would be supporting single table
+   * federation queries, we are making an assumption of one table.
+   *
+   * @return catalog name
+   */
+  public String getCatalogName() {
+    if (tableName.size() == 1) {
+      if (tableName.get(0) instanceof QualifiedName) {
+        return ((QualifiedName) tableName.get(0)).first().orElse(null);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Return full qualified table name with catalog.
+   *
+   * @return fully qualified table name with catalog.
+   */
+  public String getFullyQualifiedTableNameWithCatalog() {
     return tableName.stream()
         .map(UnresolvedExpression::toString)
         .collect(Collectors.joining(COMMA));
   }
 
   /**
    * Get original table name or its alias if present in Alias.
-   * @return    table name or its alias
+   *
+   * @return table name or its alias
    */
   public String getTableNameOrAlias() {
     return (alias == null) ? getTableName() : alias;

core/src/main/java/org/opensearch/sql/catalog/CatalogService.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.catalog;
+
+import java.util.Set;
+import org.opensearch.sql.storage.StorageEngine;
+
+/**
+ * Catalog Service defines api for
+ * providing and managing storage engines and execution engines
+ * for all the catalogs.
+ * The storage and execution indirectly make connections to the underlying datastore catalog.
+ */
+public interface CatalogService {
+
+  StorageEngine getStorageEngine(String catalog);
+
+  Set<String> getCatalogs();
+
+  void registerOpenSearchStorageEngine(StorageEngine storageEngine);
+
+}

core/src/main/java/org/opensearch/sql/catalog/model/AbstractAuthenticationData.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.catalog.model;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import lombok.Getter;
+import lombok.Setter;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.EXISTING_PROPERTY,
+    property = "type",
+    defaultImpl = AbstractAuthenticationData.class,
+    visible = true)
+@JsonSubTypes({
+    @JsonSubTypes.Type(value = BasicAuthenticationData.class, name = "basicauth"),
+})
+@Getter
+@Setter
+public abstract class AbstractAuthenticationData {
+
+  @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
+  private AuthenticationType type;
+
+}

core/src/main/java/org/opensearch/sql/catalog/model/AuthenticationType.java

@@ -0,0 +1,10 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.catalog.model;
+
+public enum AuthenticationType {
+  BASICAUTH,NO
+}