Implement Google Analytics & Google Ads OAuth Flow (#5911)

Co-authored-by: Sherif Nada <[email protected]>

Author: ChristopheDuong

airbyte-api/src/main/openapi/config.yaml

@@ -1184,6 +1184,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/source_oauths/oauth_params/create:
+    post:
+      tags:
+        - oauth
+      summary: >
+        Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
+        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
+        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
+        about these variables.
+      operationId: setInstancewideSourceOauthParams
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful
+        "400":
+          $ref: "#/components/responses/ExceptionResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
   /v1/source_oauths/get_consent_url:
     post:
       tags:
@@ -1276,6 +1299,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/destination_oauths/oauth_params/create:
+    post:
+      tags:
+        - oauth
+      summary: >
+        Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
+        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
+        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
+        about these variables.
+      operationId: setInstancewideDestinationOauthParams
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful
+        "400":
+          $ref: "#/components/responses/ExceptionResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
   /v1/web_backend/connections/list:
     post:
       tags:
@@ -1620,52 +1666,6 @@ paths:
                 $ref: "#/components/schemas/ImportRead"
         "404":
           $ref: "#/components/responses/NotFoundResponse"
-  /v1/source_oauths/oauth_params/create:
-    post:
-      tags:
-        - oauth
-      summary: >
-        Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
-        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
-        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
-        about these variables.
-      operationId: setInstancewideSourceOauthParams
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody"
-        required: true
-      responses:
-        "200":
-          description: Successful
-        "400":
-          $ref: "#/components/responses/ExceptionResponse"
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
-  /v1/destination_oauths/oauth_params/create:
-    post:
-      tags:
-        - oauth
-      summary: >
-        Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
-        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
-        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
-        about these variables.
-      operationId: setInstancewideDestinationOauthParams
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody"
-        required: true
-      responses:
-        "200":
-          description: Successful
-        "400":
-          $ref: "#/components/responses/ExceptionResponse"
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
 components:
   securitySchemes:
     bearerAuth:
@@ -2991,20 +2991,30 @@ components:
       type: object
       required:
         - sourceDefinitionId
+        - workspaceId
+        - redirectUrl
       properties:
         sourceDefinitionId:
           $ref: "#/components/schemas/SourceDefinitionId"
         workspaceId:
           $ref: "#/components/schemas/WorkspaceId"
+        redirectUrl:
+          description: The url to redirect to after getting the user consent
+          type: string
     DestinationOauthConsentRequest:
       type: object
       required:
         - destinationDefinitionId
+        - workspaceId
+        - redirectUrl
       properties:
         destinationDefinitionId:
           $ref: "#/components/schemas/DestinationDefinitionId"
         workspaceId:
           $ref: "#/components/schemas/WorkspaceId"
+        redirectUrl:
+          description: The url to redirect to after getting the user consent
+          type: string
     OAuthConsentRead:
       type: object
       required:
@@ -3016,11 +3026,15 @@ components:
       type: object
       required:
         - sourceDefinitionId
+        - workspaceId
       properties:
         sourceDefinitionId:
           $ref: "#/components/schemas/SourceDefinitionId"
         workspaceId:
           $ref: "#/components/schemas/WorkspaceId"
+        redirectUrl:
+          description: When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
+          type: string
         queryParams:
           description: The query parameters present in the redirect URL after a user granted consent e.g auth code
           type: object
@@ -3029,11 +3043,15 @@ components:
       type: object
       required:
         - destinationDefinitionId
+        - workspaceId
       properties:
         destinationDefinitionId:
           $ref: "#/components/schemas/DestinationDefinitionId"
         workspaceId:
           $ref: "#/components/schemas/WorkspaceId"
+        redirectUrl:
+          description: When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
+          type: string
         queryParams:
           description: The query parameters present in the redirect URL after a user granted consent e.g auth code
           type: object

airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java

@@ -140,12 +140,6 @@ protected List<TableInfo<CommonField<StandardSQLTypeName>>> discoverInternal(Big
     return result;
   }
 
-  @Override
-  protected List<TableInfo<CommonField<StandardSQLTypeName>>> discoverInternal(BigQueryDatabase database, String schema) throws Exception {
-    // todo to be added
-    return discoverInternal(database);
-  }
-
   @Override
   protected Map<String, List<String>> discoverPrimaryKeys(BigQueryDatabase database, List<TableInfo<CommonField<StandardSQLTypeName>>> tableInfos) {
     return Collections.emptyMap();

airbyte-oauth/build.gradle

@@ -1,7 +1,11 @@
 plugins {
     id "java-library"
+    id 'airbyte-integration-test-java'
 }
 
 dependencies {
-
+    implementation project(':airbyte-config:models')
+    implementation project(':airbyte-config:persistence')
+    implementation project(':airbyte-json-validation')
+    testImplementation project(':airbyte-oauth')
 }

airbyte-oauth/src/main/java/io/airbyte/oauth/MoreOAuthParameters.java

@@ -0,0 +1,60 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.oauth;
+
+import io.airbyte.config.DestinationOAuthParameter;
+import io.airbyte.config.SourceOAuthParameter;
+import java.util.Comparator;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+public class MoreOAuthParameters {
+
+  public static Optional<SourceOAuthParameter> getSourceOAuthParameter(
+                                                                       Stream<SourceOAuthParameter> stream,
+                                                                       UUID workspaceId,
+                                                                       UUID sourceDefinitionId) {
+    return stream
+        .filter(p -> sourceDefinitionId.equals(p.getSourceDefinitionId()))
+        .filter(p -> p.getWorkspaceId() == null || workspaceId.equals(p.getWorkspaceId()))
+        // we prefer params specific to a workspace before global ones (ie workspace is null)
+        .min(Comparator.comparing(SourceOAuthParameter::getWorkspaceId, Comparator.nullsLast(Comparator.naturalOrder()))
+            .thenComparing(SourceOAuthParameter::getOauthParameterId));
+  }
+
+  public static Optional<DestinationOAuthParameter> getDestinationOAuthParameter(
+                                                                                 Stream<DestinationOAuthParameter> stream,
+                                                                                 UUID workspaceId,
+                                                                                 UUID destinationDefinitionId) {
+    return stream
+        .filter(p -> destinationDefinitionId.equals(p.getDestinationDefinitionId()))
+        .filter(p -> p.getWorkspaceId() == null || workspaceId.equals(p.getWorkspaceId()))
+        // we prefer params specific to a workspace before global ones (ie workspace is null)
+        .min(Comparator.comparing(DestinationOAuthParameter::getWorkspaceId, Comparator.nullsLast(Comparator.naturalOrder()))
+            .thenComparing(DestinationOAuthParameter::getOauthParameterId));
+  }
+
+}

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthFlowImplementation.java

@@ -24,13 +24,21 @@
 
 package io.airbyte.oauth;
 
+import io.airbyte.config.persistence.ConfigNotFoundException;
+import java.io.IOException;
 import java.util.Map;
 import java.util.UUID;
 
 public interface OAuthFlowImplementation {
 
-  String getConsentUrl();
+  String getSourceConsentUrl(UUID workspaceId, UUID sourceDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException;
 
-  Map<String, Object> completeOAuth(UUID workspaceId, Map<String, Object> queryParams);
+  String getDestinationConsentUrl(UUID workspaceId, UUID destinationDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException;
+
+  Map<String, Object> completeSourceOAuth(UUID workspaceId, UUID sourceDefinitionId, Map<String, Object> queryParams, String redirectUrl)
+      throws IOException, ConfigNotFoundException;
+
+  Map<String, Object> completeDestinationOAuth(UUID workspaceId, UUID destinationDefinitionId, Map<String, Object> queryParams, String redirectUrl)
+      throws IOException, ConfigNotFoundException;
 
 }

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -25,15 +25,23 @@
 package io.airbyte.oauth;
 
 import com.google.common.collect.ImmutableMap;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.oauth.google.GoogleAdsOauthFlow;
+import io.airbyte.oauth.google.GoogleAnalyticsOauthFlow;
 import java.util.Map;
 
 public class OAuthImplementationFactory {
 
-  static final Map<String, OAuthFlowImplementation> OAUTH_FLOW_MAPPING =
-      ImmutableMap.<String, OAuthFlowImplementation>builder()
-          .build();
+  private final Map<String, OAuthFlowImplementation> OAUTH_FLOW_MAPPING;
 
-  public static OAuthFlowImplementation create(String imageName) {
+  public OAuthImplementationFactory(ConfigRepository configRepository) {
+    OAUTH_FLOW_MAPPING = ImmutableMap.<String, OAuthFlowImplementation>builder()
+        .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOauthFlow(configRepository))
+        .put("airbyte/source-google-ads", new GoogleAdsOauthFlow(configRepository))
+        .build();
+  }
+
+  public OAuthFlowImplementation create(String imageName) {
     if (OAUTH_FLOW_MAPPING.containsKey(imageName)) {
       return OAUTH_FLOW_MAPPING.get(imageName);
     } else {

airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAdsOauthFlow.java

@@ -0,0 +1,48 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.oauth.google;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.config.persistence.ConfigRepository;
+
+public class GoogleAdsOauthFlow extends GoogleOAuthFlow {
+
+  public GoogleAdsOauthFlow(ConfigRepository configRepository) {
+    super(configRepository, "https://www.googleapis.com/auth/adwords");
+  }
+
+  @Override
+  protected String getClientIdUnsafe(JsonNode config) {
+    // the config object containing client ID and secret is nested inside the "credentials" object
+    return super.getClientIdUnsafe(config.get("credentials"));
+  }
+
+  @Override
+  protected String getClientSecretUnsafe(JsonNode config) {
+    // the config object containing client ID and secret is nested inside the "credentials" object
+    return super.getClientSecretUnsafe(config.get("credentials"));
+  }
+
+}

airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAnalyticsOauthFlow.java

@@ -0,0 +1,35 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.oauth.google;
+
+import io.airbyte.config.persistence.ConfigRepository;
+
+public class GoogleAnalyticsOauthFlow extends GoogleOAuthFlow {
+
+  public GoogleAnalyticsOauthFlow(ConfigRepository configRepository) {
+    super(configRepository, "https://www.googleapis.com/auth/analytics.readonly");
+  }
+
+}