Skip to content

Commit 8c1af87

Browse files
alexandr-shegedaalexandr-shegeda
authored
Merge branch 'master' into osheg/10148-bigquery-docs
2 parents fa569fd + b75295a commit 8c1af87

File tree

66 files changed

+1439
-632
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+1439
-632
lines changed

‎.github/workflows/platform-project-automation.yml

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ on:
88
env:
99
GH_PROJECT_TOKEN: ${{ secrets.PARKER_PAT_FOR_PLATFORM_PROJECT_AUTOMATION }}
1010
ORG: airbytehq
11-
PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
11+
CLOUD_PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
12+
OSS_PROJECT_ID: 11 # https://github.com/orgs/airbytehq/projects/11
1213
FIELD_STATUS: Status
1314
STATUS_TODO: Todo
1415
FIELD_DATE_ADDED: Date Added
@@ -21,14 +22,44 @@ jobs:
2122
- name: Set current date env var
2223
id: set_date
2324
run: echo ::set-output name=CURRENT_DATE::$(date +'%Y-%m-%dT%H:%M:%S%z')
24-
25-
- name: Add issue to project if labelled with area/platform
25+
26+
- name: Add issue to the cloud project if labelled with area/platform and not area/platform-oss
27+
uses: leonsteinhaeuser/[email protected]
28+
if: contains(github.event.issue.labels.*.name, 'area/platform') && !contains(github.event.issue.labels.*.name, 'area/platform-oss')
29+
with:
30+
gh_token: ${{ env.GH_PROJECT_TOKEN }}
31+
organization: ${{ env.ORG }}
32+
project_id: ${{ env.CLOUD_PROJECT_ID }}
33+
resource_node_id: ${{ github.event.issue.node_id }}
34+
operation_mode: custom_field
35+
custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
36+
- name: Add issue to the cloud oss if labelled with area/platform and not area/platform-cloud
37+
uses: leonsteinhaeuser/[email protected]
38+
if: contains(github.event.issue.labels.*.name, 'area/platform') && !contains(github.event.issue.labels.*.name, 'area/platform-cloud')
39+
with:
40+
gh_token: ${{ env.GH_PROJECT_TOKEN }}
41+
organization: ${{ env.ORG }}
42+
project_id: ${{ env.OSS_PROJECT_ID }}
43+
resource_node_id: ${{ github.event.issue.node_id }}
44+
operation_mode: custom_field
45+
custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
46+
- name: Add issue to project if labelled with area/platform-cloud
47+
uses: leonsteinhaeuser/[email protected]
48+
if: contains(github.event.issue.labels.*.name, 'area/platform-cloud')
49+
with:
50+
gh_token: ${{ env.GH_PROJECT_TOKEN }}
51+
organization: ${{ env.ORG }}
52+
project_id: ${{ env.CLOUD_PROJECT_ID }}
53+
resource_node_id: ${{ github.event.issue.node_id }}
54+
operation_mode: custom_field
55+
custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
56+
- name: Add issue to project if labelled with area/platform-oss
2657
uses: leonsteinhaeuser/[email protected]
27-
if: contains(github.event.issue.labels.*.name, 'area/platform')
58+
if: contains(github.event.issue.labels.*.name, 'area/platform-oss')
2859
with:
2960
gh_token: ${{ env.GH_PROJECT_TOKEN }}
3061
organization: ${{ env.ORG }}
31-
project_id: ${{ env.PROJECT_ID }}
62+
project_id: ${{ env.OSS_PROJECT_ID }}
3263
resource_node_id: ${{ github.event.issue.node_id }}
3364
operation_mode: custom_field
3465
custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'

‎.github/workflows/release-airbyte-os.yml

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,10 @@ jobs:
6363
- uses: actions/setup-node@v1
6464
with:
6565
node-version: "16.13.0"
66-
6766
# necessary to install pip
6867
- uses: actions/setup-python@v2
6968
with:
7069
python-version: "3.7"
71-
- name: Save Old Version
72-
id: old_version
73-
run: |
74-
echo ::set-output name=OLD_VERSION::$(grep -w VERSION .env | cut -d"=" -f2)
7570
- name: Release Airbyte
7671
id: release_airbyte
7772
env:
@@ -81,14 +76,63 @@ jobs:
8176
CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
8277
run: |
8378
./tools/bin/release_version.sh
84-
- name: Save New Version
85-
id: new_version
86-
run: |
87-
echo ::set-output name=NEW_VERSION::$(grep -w VERSION .env | cut -d"=" -f2)
79+
80+
# We are releasing octavia from a separate job because:
81+
# - The self hosted runner used in releaseAirbyte does not have the docker buildx command to build multi-arch images
82+
# - Plaform build requires Python 3.7, Octavia Build requires 3.8
83+
releaseOctavia:
84+
runs-on: ubuntu-latest
85+
environment: more-secrets
86+
steps:
87+
- name: Checkout
88+
uses: actions/checkout@v2
89+
with:
90+
fetch-depth: 0
91+
- uses: actions/setup-java@v1
92+
with:
93+
java-version: "17"
94+
95+
- uses: actions/setup-node@v1
96+
with:
97+
node-version: "16.13.0"
98+
# octavia-cli build requires Python 3.8.
99+
# We use 3.8 in this project because 3.7 is not supported on Apple M1.
100+
- uses: actions/setup-python@v2
101+
with:
102+
python-version: "3.8"
103+
- name: Release Octavia
104+
id: release_octavia
105+
env:
106+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
107+
PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
108+
run: ./tools/bin/release_version_octavia.sh
109+
110+
createPullRequest:
111+
needs:
112+
- releaseAirbyte
113+
- releaseOctavia
114+
runs-on: ubuntu-latest
115+
environment: more-secrets
116+
steps:
117+
- name: Checkout
118+
uses: actions/checkout@v2
119+
with:
120+
fetch-depth: 0
121+
# necessary to install pip
122+
- uses: actions/setup-python@v2
123+
with:
124+
python-version: "3.7"
125+
- name: Bump version
126+
id: bump_version
127+
env:
128+
PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
129+
run: ./tools/bin/bump_version.sh
88130
- name: Get PR Body
89131
id: pr_body
90132
env:
91-
PREV_VERSION: ${{ steps.old_version.outputs.OLD_VERSION }}
133+
PREV_VERSION: ${{ steps.bump_version.outputs.PREV_VERSION }}
134+
NEW_VERSION: ${{ steps.bump_version.outputs.NEW_VERSION }}
135+
GIT_REVISION: ${{ steps.bump_version.outputs.GIT_REVISION }}
92136
run: |
93137
chmod +x tools/bin/pr_body.sh
94138
body=$(./tools/bin/pr_body.sh)
@@ -102,9 +146,9 @@ jobs:
102146
branch: bump-version
103147
branch-suffix: random
104148
delete-branch: true
105-
title: Bump Airbyte version from ${{ steps.old_version.outputs.OLD_VERSION }} to ${{ steps.new_version.outputs.NEW_VERSION }}
149+
title: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
106150
body: ${{ steps.pr_body.outputs.PR_BODY }}
107-
commit-message: Bump Airbyte version from ${{ steps.old_version.outputs.OLD_VERSION }} to ${{ steps.new_version.outputs.NEW_VERSION }}
151+
commit-message: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
108152
- name: PR Details
109153
run: |
110154
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"

‎.github/workflows/slash-commands.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
id: scd
1919
uses: peter-evans/slash-command-dispatch@v2
2020
with:
21-
token: ${{ secrets.SUPERTOPHER_PAT }}
21+
token: ${{ secrets.DAVINCHIA_PAT }}
2222
commands: |
2323
test
2424
test-performance

‎airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,12 @@ public BootloaderApp() {
8686
final Database configDatabase =
8787
new ConfigsDatabaseInstance(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), configs.getConfigDatabaseUrl())
8888
.getAndInitialize();
89-
final JsonSecretsProcessor jsonSecretsProcessor = new JsonSecretsProcessor();
89+
final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder()
90+
.maskSecrets(!featureFlags.exposeSecretsInExport())
91+
.copySecrets(true)
92+
.build();
9093
final ConfigPersistence configPersistence =
91-
DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor, featureFlags);
94+
DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor);
9295
configPersistence.loadData(YamlSeedConfigPersistence.getDefault());
9396
LOGGER.info("Loaded seed data..");
9497
} catch (final IOException e) {
@@ -115,8 +118,11 @@ public void load() throws Exception {
115118
runFlywayMigration(configs, configDatabase, jobDatabase);
116119
LOGGER.info("Ran Flyway migrations...");
117120

118-
final JsonSecretsProcessor jsonSecretsProcessor = new JsonSecretsProcessor();
119-
final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor, featureFlags);
121+
final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder()
122+
.maskSecrets(!featureFlags.exposeSecretsInExport())
123+
.copySecrets(false)
124+
.build();
125+
final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor);
120126
final ConfigRepository configRepository =
121127
new ConfigRepository(configPersistence, configDatabase);
122128

‎airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ void testBootloaderAppBlankDb() throws Exception {
7272
container.getPassword(),
7373
container.getJdbcUrl()).getInitialized();
7474
val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, this.getClass().getName());
75-
assertEquals("0.35.40.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
75+
assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
7676

7777
val configDatabase = new ConfigsDatabaseInstance(
7878
mockedConfigs.getConfigDatabaseUser(),

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
- name: Amazon Ads
1616
sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
1717
dockerRepository: airbyte/source-amazon-ads
18-
dockerImageTag: 0.1.3
18+
dockerImageTag: 0.1.4
1919
documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
2020
icon: amazonads.svg
2121
sourceType: api
@@ -252,7 +252,7 @@
252252
- name: GitHub
253253
sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
254254
dockerRepository: airbyte/source-github
255-
dockerImageTag: 0.2.23
255+
dockerImageTag: 0.2.25
256256
documentationUrl: https://docs.airbyte.io/integrations/sources/github
257257
icon: github.svg
258258
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2380,7 +2380,7 @@
23802380
supportsNormalization: false
23812381
supportsDBT: false
23822382
supported_destination_sync_modes: []
2383-
- dockerImage: "airbyte/source-github:0.2.23"
2383+
- dockerImage: "airbyte/source-github:0.2.25"
23842384
spec:
23852385
documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
23862386
connectionSpecification:

‎airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.google.common.annotations.VisibleForTesting;
2323
import com.google.common.collect.Sets;
2424
import io.airbyte.commons.enums.Enums;
25-
import io.airbyte.commons.features.FeatureFlags;
2625
import io.airbyte.commons.json.Jsons;
2726
import io.airbyte.commons.util.MoreIterators;
2827
import io.airbyte.commons.version.AirbyteVersion;
@@ -81,7 +80,6 @@ public class DatabaseConfigPersistence implements ConfigPersistence {
8180

8281
private final ExceptionWrappingDatabase database;
8382
private final JsonSecretsProcessor jsonSecretsProcessor;
84-
private final FeatureFlags featureFlags;
8583
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseConfigPersistence.class);
8684

8785
/**
@@ -90,20 +88,16 @@ public class DatabaseConfigPersistence implements ConfigPersistence {
9088
*
9189
* @param database - database where configs are stored
9290
* @param jsonSecretsProcessor - for filtering secrets in export
93-
* @param featureFlags - feature flags that govern secret export behavior
9491
* @return database config persistence wrapped in validation decorators
9592
*/
9693
public static ConfigPersistence createWithValidation(final Database database,
97-
final JsonSecretsProcessor jsonSecretsProcessor,
98-
final FeatureFlags featureFlags) {
99-
return new ClassEnforcingConfigPersistence(
100-
new ValidatingConfigPersistence(new DatabaseConfigPersistence(database, jsonSecretsProcessor, featureFlags)));
94+
final JsonSecretsProcessor jsonSecretsProcessor) {
95+
return new ValidatingConfigPersistence(new DatabaseConfigPersistence(database, jsonSecretsProcessor));
10196
}
10297

103-
public DatabaseConfigPersistence(final Database database, final JsonSecretsProcessor jsonSecretsProcessor, final FeatureFlags featureFlags) {
98+
public DatabaseConfigPersistence(final Database database, final JsonSecretsProcessor jsonSecretsProcessor) {
10499
this.database = new ExceptionWrappingDatabase(database);
105100
this.jsonSecretsProcessor = jsonSecretsProcessor;
106-
this.featureFlags = featureFlags;
107101
}
108102

109103
@Override
@@ -1431,18 +1425,15 @@ public Map<String, Stream<JsonNode>> dumpConfigs() throws IOException {
14311425
sourceConnectionWithMetadata
14321426
.stream()
14331427
.map(configWithMetadata -> {
1434-
if (featureFlags.exposeSecretsInExport()) {
1435-
return Jsons.jsonNode(configWithMetadata.getConfig());
1436-
}
1437-
14381428
try {
14391429
final UUID sourceDefinitionId = configWithMetadata.getConfig().getSourceDefinitionId();
14401430
final StandardSourceDefinition standardSourceDefinition = getConfig(
14411431
ConfigSchema.STANDARD_SOURCE_DEFINITION,
14421432
sourceDefinitionId.toString(),
14431433
StandardSourceDefinition.class);
14441434
final JsonNode connectionSpecs = standardSourceDefinition.getSpec().getConnectionSpecification();
1445-
final JsonNode sanitizedConfig = jsonSecretsProcessor.maskSecrets(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpecs);
1435+
final JsonNode sanitizedConfig =
1436+
jsonSecretsProcessor.prepareSecretsForOutput(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpecs);
14461437
return sanitizedConfig;
14471438
} catch (final ConfigNotFoundException | JsonValidationException | IOException e) {
14481439
throw new RuntimeException(e);
@@ -1454,18 +1445,15 @@ public Map<String, Stream<JsonNode>> dumpConfigs() throws IOException {
14541445
final Stream<JsonNode> jsonNodeStream = destinationConnectionWithMetadata
14551446
.stream()
14561447
.map(configWithMetadata -> {
1457-
if (featureFlags.exposeSecretsInExport()) {
1458-
return Jsons.jsonNode(configWithMetadata.getConfig());
1459-
}
1460-
14611448
try {
14621449
final UUID destinationDefinition = configWithMetadata.getConfig().getDestinationDefinitionId();
14631450
final StandardDestinationDefinition standardDestinationDefinition = getConfig(
14641451
ConfigSchema.STANDARD_DESTINATION_DEFINITION,
14651452
destinationDefinition.toString(),
14661453
StandardDestinationDefinition.class);
14671454
final JsonNode connectionSpec = standardDestinationDefinition.getSpec().getConnectionSpecification();
1468-
final JsonNode sanitizedConfig = jsonSecretsProcessor.maskSecrets(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpec);
1455+
final JsonNode sanitizedConfig =
1456+
jsonSecretsProcessor.prepareSecretsForOutput(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpec);
14691457
return sanitizedConfig;
14701458
} catch (final ConfigNotFoundException | JsonValidationException | IOException e) {
14711459
throw new RuntimeException(e);

0 commit comments

Comments
 (0)