Skip to content

Commit f4afffe

Browse files
letiescancianoletiescanciano
committed
Merge branch 'master' into leti/fix-page-tracking-in-segment
* master: (47 commits) Add email to identify users analytics call (#16327) 🎉 Source Amazon Ads: improve `config.start_date` validation (#16191) Add comments about intermediate state emission (#16262) MySQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16216) MSSQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16215) Hide a bunch more destination with potential unsecure API access (#16320) Skip unit tests when run-tests is false (#16267) Hide Destination connections from UI (#16310) Add scheduled task to clean up old files from workspace (#16247) Source Google Analytics v4: Re-name google analytics connector (#16306) 🐛 Source Facebook Marketing: remove "end_date" from config if empty value (re-implement #16096) (#16222) Fix github action syntax (#16277) Re-name google analytics cionnectors (#16287) Bump Airbyte version from 0.40.3 to 0.40.4 (#16275) Hide ES and Redis destination connectors from Cloud (#16276) 15700 add tests for PokeAPI (#15701) Add ProtocolVersion to StandardDefs (#16237) 🪟 🔧 🧹 Migrate attempt `bytesSynced` to `totalStats.bytesEmitted` and cleanup `AttemptDetails` component (#16126) Improve behavior of password input field (#16011) Improve airbyte-metrics support in the Helm chart (#16166) ...
2 parents fc63541 + d49c306 commit f4afffe

File tree

240 files changed

+17835
-852
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

240 files changed

+17835
-852
lines changed

‎.bumpversion.cfg

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.40.3
2+
current_version = 0.40.4
33
commit = False
44
tag = False
55
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -14,6 +14,8 @@ serialize =
1414

1515
[bumpversion:file:airbyte-container-orchestrator/Dockerfile]
1616

17+
[bumpversion:file:airbyte-cron/Dockerfile]
18+
1719
[bumpversion:file:airbyte-metrics/reporter/Dockerfile]
1820

1921
[bumpversion:file:airbyte-server/Dockerfile]

‎.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111

1212
### SHARED ###
13-
VERSION=0.40.3
13+
VERSION=0.40.4
1414

1515
# When using the airbyte-db via default docker image
1616
CONFIG_ROOT=/data

‎.github/workflows/project.yaml renamed to ‎.github/workflows/assign-issue-to-project.yaml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# https://github.com/marketplace/actions/assign-to-one-project#:~:text=GitHub%20Action%20for%20Assign%20to,columns%20in%20your%20project%20dashboard.
2-
name: Auto Assign to Airbyte Github Project
2+
name: Assign Issue to Airbyte Github Project
33

44
on:
55
issues:
@@ -32,3 +32,9 @@ jobs:
3232
with:
3333
project: "https://github.com/airbytehq/airbyte/projects/15"
3434
column_name: "To do"
35+
- name: Assign Onboarding Improvements PR to Project
36+
uses: srggrs/[email protected]
37+
if: contains(github.event.pull_request.labels.*.name, 'project/onboarding-improvements')
38+
with:
39+
project: "https://github.com/airbytehq/airbyte/projects/15"
40+
column_name: "To do"

‎.github/workflows/assign-issue-to-project.yml

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎.github/workflows/build-report.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
name: Build Report
1+
name: Generate Build Report
2+
3+
# Uses Python to Generate a build report
4+
# and send it to Slack
25

36
on:
47
workflow_dispatch:

‎.github/workflows/connector_integration_tests.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
name: Connector Integration Tests
22

3+
# Launches the connector integration tests
4+
35
on:
46
workflow_dispatch:
57
schedule:

‎.github/workflows/create-release.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# This is an action that runs when an Airbyte version bump is merged into master.
22
# It fetches the changelog from the version bump PR and automatically creates a
33
# Release for the version bump.
4-
name: Create Release
4+
5+
name: Create Airbyte GH Release
56

67
on:
78
push:

‎.github/workflows/doc-link-check.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Perform link check on all markdown files
2+
# https://github.com/gaurav-nelson/github-action-markdown-link-check
23

3-
name: Doc Link Checker
4+
name: Check for broken links in docs
45

56
on:
67
workflow_dispatch:

‎.github/workflows/fe-validate-links.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Frontend Link Validation
1+
name: Check for broken links in FE
22

33
on:
44
workflow_dispatch:

‎.github/workflows/gke-kube-test-command.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: GKE Kube Acceptance Test
22
on:
33
schedule:
4-
- cron '0 0 * * 0' # runs at midnight UTC every Sunday
4+
- cron: '0 0 * * 0' # runs at midnight UTC every Sunday
55
workflow_dispatch:
66
inputs:
77
repo:

‎.github/workflows/labeler.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
name: "Pull Request Labeler"
1+
# the mapping from filepath to label
2+
# is defined in .github/labeler.yml
3+
4+
name: "Label PR based on filepath"
25
on:
36
- pull_request_target
47

‎.github/workflows/notify-on-label.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
name: Notify team
1+
# Notify users/teams when labels are added to an issue.
2+
3+
name: Notify when adding label to issue
24

35
on:
46
issues:

‎.github/workflows/run-specific-test-command.yml renamed to ‎.github/workflows/run-performance-test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: performance-test
1+
name: Run Performance Test
22
on:
33
workflow_dispatch:
44
inputs:

‎.github/workflows/shared-issues.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
name: "Shared Issues"
1+
# Adds metadata and labels to issues based on context
2+
# https://github.com/airbytehq/workflow-actions/blob/main/src/bin_issue.ts <-- this gets run
3+
4+
name: "Add metadata and labels to issues based on context"
25
on:
36
issues:
47
types: [opened, labeled, unlabeled]

‎.github/workflows/shared-pulls.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
name: "Shared Pull Requests"
1+
# Runs internal automation for pull requests
2+
3+
name: "Add metadata and labels to PRs based on context"
24
on:
35
pull_request_target:
46
types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened]

‎.github/workflows/slash-commands.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
id: scd
1919
uses: peter-evans/slash-command-dispatch@v2
2020
with:
21-
token: ${{ secrets.DAVINCHIA_PAT }}
21+
token: ${{ secrets.SUPERTOPHER_PAT }}
2222
commands: |
2323
test
2424
test-performance

‎.github/workflows/test-command.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: integration-test
1+
name: Run Integration Test
22
on:
33
workflow_dispatch:
44
inputs:

‎.github/workflows/test-performance-command.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
name: performance-test
1+
# runs ./tools/bin/ci_performance_test.sh
2+
# which is more or less ./gradlew performanceTest limited to connectors with changes
3+
name: Run Performance Test
24
on:
35
workflow_dispatch:
46
inputs:

‎.github/workflows/workflow-cleanup.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Cleanup old GH action workflow runs
1+
name: Cleanup old GH workflow runs
22

33
on:
44
schedule:

‎airbyte-bootloader/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye
22
ARG JDK_IMAGE=openjdk:${JDK_VERSION}
33
FROM ${JDK_IMAGE}
44

5-
ARG VERSION=0.40.3
5+
ARG VERSION=0.40.4
66

77
ENV APPLICATION airbyte-bootloader
88
ENV VERSION ${VERSION}

‎airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ void testBootloaderAppBlankDb() throws Exception {
133133
val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
134134
// this line should change with every new migration
135135
// to show that you meant to make a new migration to the prod database
136-
assertEquals("0.40.3.001", configsMigrator.getLatestMigration().getVersion().getVersion());
136+
assertEquals("0.40.3.002", configsMigrator.getLatestMigration().getVersion().getVersion());
137137

138138
val jobsPersistence = new DefaultJobPersistence(jobDatabase);
139139
assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());

‎airbyte-cdk/python/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
packages=find_packages(exclude=("unit_tests",)),
4545
install_requires=[
4646
"backoff",
47-
"dataclasses-jsonschema~=2.15.1",
47+
"dataclasses-jsonschema==2.15.1", # pinned to the last working version for us temporarily while we fix
4848
"dpath~=2.0.1",
4949
"jsonschema~=3.2.0",
5050
"jsonref~=0.2",

‎airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import io.airbyte.commons.version.AirbyteVersion;
88
import io.airbyte.config.helpers.LogConfigs;
99
import io.airbyte.config.storage.CloudStorageConfigs;
10+
import java.net.URI;
1011
import java.nio.file.Path;
1112
import java.util.List;
1213
import java.util.Map;
@@ -71,6 +72,11 @@ public interface Configs {
7172
*/
7273
Path getWorkspaceRoot();
7374

75+
/**
76+
* Defines the URL to pull the remote connector catalog from.
77+
*/
78+
URI getRemoteConnectorCatalogUrl();
79+
7480
// Docker Only
7581

7682
/**

‎airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import io.airbyte.config.storage.CloudStorageConfigs.GcsConfig;
1717
import io.airbyte.config.storage.CloudStorageConfigs.MinioConfig;
1818
import io.airbyte.config.storage.CloudStorageConfigs.S3Config;
19+
import java.net.URI;
1920
import java.nio.file.Path;
2021
import java.util.Arrays;
2122
import java.util.HashSet;
@@ -148,6 +149,8 @@ public class EnvConfigs implements Configs {
148149
public static final String METRIC_CLIENT = "METRIC_CLIENT";
149150
private static final String OTEL_COLLECTOR_ENDPOINT = "OTEL_COLLECTOR_ENDPOINT";
150151

152+
public static final String REMOTE_CONNECTOR_CATALOG_URL = "REMOTE_CONNECTOR_CATALOG_URL";
153+
151154
// job-type-specific overrides
152155
public static final String SPEC_JOB_KUBE_NODE_SELECTORS = "SPEC_JOB_KUBE_NODE_SELECTORS";
153156
public static final String CHECK_JOB_KUBE_NODE_SELECTORS = "CHECK_JOB_KUBE_NODE_SELECTORS";
@@ -323,6 +326,12 @@ public Path getWorkspaceRoot() {
323326
return getPath(WORKSPACE_ROOT);
324327
}
325328

329+
@Override
330+
public URI getRemoteConnectorCatalogUrl() {
331+
// Default to reuse the job database
332+
return URI.create(getEnsureEnv(REMOTE_CONNECTOR_CATALOG_URL));
333+
}
334+
326335
// Docker Only
327336
@Override
328337
public String getWorkspaceDockerMount() {

‎airbyte-config/config-models/src/main/resources/types/CombinedConnectorCatalog.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
"$schema": http://json-schema.org/draft-07/schema#
3+
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/CombinedConnectorCatalog.yaml
4+
title: CombinedConnectorCatalog
5+
description: Used to provide the connector catalog from a remote source
6+
type: object
7+
required:
8+
- destinations
9+
- sources
10+
properties:
11+
destinations:
12+
type: array
13+
items:
14+
existingJavaType: io.airbyte.config.StandardDestinationDefinition
15+
sources:
16+
type: array
17+
items:
18+
existingJavaType: io.airbyte.config.StandardSourceDefinition

‎airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,6 @@ properties:
5555
format: date
5656
resourceRequirements:
5757
"$ref": ActorDefinitionResourceRequirements.yaml
58+
protocolVersion:
59+
type: string
60+
description: the Airbyte Protocol version supported by the connector

‎airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,6 @@ properties:
6262
format: date
6363
resourceRequirements:
6464
"$ref": ActorDefinitionResourceRequirements.yaml
65+
protocolVersion:
66+
type: string
67+
description: the Airbyte Protocol version supported by the connector

‎airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ static void writeStandardSourceDefinition(final List<StandardSourceDefinition> c
5050
: Enums.toEnum(standardSourceDefinition.getSourceType().value(),
5151
SourceType.class).orElseThrow())
5252
.set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec())))
53+
.set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion())
5354
.set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone())
5455
.set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic())
5556
.set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom())
@@ -79,6 +80,7 @@ static void writeStandardSourceDefinition(final List<StandardSourceDefinition> c
7980
: Enums.toEnum(standardSourceDefinition.getSourceType().value(),
8081
SourceType.class).orElseThrow())
8182
.set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec())))
83+
.set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion())
8284
.set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone() != null && standardSourceDefinition.getTombstone())
8385
.set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic())
8486
.set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom())
@@ -115,6 +117,7 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
115117
.set(Tables.ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon())
116118
.set(Tables.ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination)
117119
.set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec())))
120+
.set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardDestinationDefinition.getProtocolVersion())
118121
.set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardDestinationDefinition.getTombstone())
119122
.set(Tables.ACTOR_DEFINITION.PUBLIC, standardDestinationDefinition.getPublic())
120123
.set(Tables.ACTOR_DEFINITION.CUSTOM, standardDestinationDefinition.getCustom())
@@ -140,6 +143,7 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
140143
.set(Tables.ACTOR_DEFINITION.ICON, standardDestinationDefinition.getIcon())
141144
.set(Tables.ACTOR_DEFINITION.ACTOR_TYPE, ActorType.destination)
142145
.set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getSpec())))
146+
.set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardDestinationDefinition.getProtocolVersion())
143147
.set(Tables.ACTOR_DEFINITION.TOMBSTONE,
144148
standardDestinationDefinition.getTombstone() != null && standardDestinationDefinition.getTombstone())
145149
.set(Tables.ACTOR_DEFINITION.PUBLIC, standardDestinationDefinition.getPublic())

‎airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ public static StandardSourceDefinition buildStandardSourceDefinition(final Recor
100100
.withSourceType(record.get(ACTOR_DEFINITION.SOURCE_TYPE) == null ? null
101101
: Enums.toEnum(record.get(ACTOR_DEFINITION.SOURCE_TYPE, String.class), SourceType.class).orElseThrow())
102102
.withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))
103+
.withProtocolVersion(record.get(ACTOR_DEFINITION.PROTOCOL_VERSION, String.class))
103104
.withTombstone(record.get(ACTOR_DEFINITION.TOMBSTONE))
104105
.withPublic(record.get(ACTOR_DEFINITION.PUBLIC))
105106
.withCustom(record.get(ACTOR_DEFINITION.CUSTOM))
@@ -121,6 +122,7 @@ public static StandardDestinationDefinition buildStandardDestinationDefinition(f
121122
.withDocumentationUrl(record.get(ACTOR_DEFINITION.DOCUMENTATION_URL))
122123
.withName(record.get(ACTOR_DEFINITION.NAME))
123124
.withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION.SPEC).data(), ConnectorSpecification.class))
125+
.withProtocolVersion(record.get(ACTOR_DEFINITION.PROTOCOL_VERSION, String.class))
124126
.withTombstone(record.get(ACTOR_DEFINITION.TOMBSTONE))
125127
.withPublic(record.get(ACTOR_DEFINITION.PUBLIC))
126128
.withCustom(record.get(ACTOR_DEFINITION.CUSTOM))

‎airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ public static StandardSourceDefinition publicSourceDefinition() {
178178
.withDocumentationUrl("documentation-url-1")
179179
.withIcon("icon-1")
180180
.withSpec(connectorSpecification())
181+
.withProtocolVersion("0.2.1")
181182
.withTombstone(false)
182183
.withPublic(true)
183184
.withCustom(false)
@@ -206,6 +207,7 @@ public static StandardSourceDefinition grantableSourceDefinition2() {
206207
.withDockerImageTag("tag-3")
207208
.withDockerRepository("repository-3")
208209
.withDocumentationUrl("documentation-url-3")
210+
.withProtocolVersion("0.2.2")
209211
.withIcon("icon-3")
210212
.withTombstone(false)
211213
.withPublic(false)
@@ -220,6 +222,7 @@ public static StandardSourceDefinition customSourceDefinition() {
220222
.withDockerImageTag("tag-4")
221223
.withDockerRepository("repository-4")
222224
.withDocumentationUrl("documentation-url-4")
225+
.withProtocolVersion("0.2.4")
223226
.withIcon("icon-4")
224227
.withTombstone(false)
225228
.withPublic(false)
@@ -256,6 +259,7 @@ public static StandardDestinationDefinition publicDestinationDefinition() {
256259
.withDocumentationUrl("documentation-url-3")
257260
.withIcon("icon-3")
258261
.withSpec(connectorSpecification())
262+
.withProtocolVersion("0.3.1")
259263
.withTombstone(false)
260264
.withPublic(true)
261265
.withCustom(false)
@@ -299,6 +303,7 @@ public static StandardDestinationDefinition cusstomDestinationDefinition() {
299303
.withDocumentationUrl("documentation-url-44")
300304
.withIcon("icon-4")
301305
.withSpec(connectorSpecification())
306+
.withProtocolVersion("0.3.2")
302307
.withTombstone(false)
303308
.withPublic(false)
304309
.withCustom(true);

‎airbyte-config/init/build.gradle

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ dependencies {
1010
implementation project(':airbyte-protocol:protocol-models')
1111
implementation project(':airbyte-commons-docker')
1212
implementation project(':airbyte-json-validation')
13+
14+
testImplementation project(':airbyte-test-utils')
15+
testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1'
1316
}
1417

1518
task copyScripts(type: Copy) {

0 commit comments

Comments
 (0)