Skip to content

Commit 74d574a

Browse files
lmossmanerohmensing
lmossman
authored and
erohmensing
committed
Simplify Github and Postgres forms #2 (#24255)
* add grouping and collapsing fields to postgres source * add auth group to github source connector * revert postgres field order changes and adjust group of schemas field * inject group into ssh tunnel spec for postgres only, through overloaded methods * Automated Change * bump Dockerfile versions and update changelogs * bump strict encrypt version as well * fix postgres acceptance test * fix acceptance test again * fix all postgres acceptance tests * add newline * undo other changes to postgres readme file * add security group to tunnel_method in expected_spec.json * bump version of strict encrypt * manually bump versions in seed files --------- Co-authored-by: lmossman <[email protected]>
1 parent f1fa7cd commit 74d574a

File tree

22 files changed

+218
-485
lines changed

22 files changed

+218
-485
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,7 @@
691691
- name: GitHub
692692
sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
693693
dockerRepository: airbyte/source-github
694-
dockerImageTag: 0.4.3
694+
dockerImageTag: 0.4.4
695695
documentationUrl: https://docs.airbyte.com/integrations/sources/github
696696
icon: github.svg
697697
sourceType: api
@@ -1565,7 +1565,7 @@
15651565
- name: Postgres
15661566
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
15671567
dockerRepository: airbyte/source-postgres
1568-
dockerImageTag: 2.0.7
1568+
dockerImageTag: 2.0.8
15691569
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
15701570
icon: postgresql.svg
15711571
sourceType: database

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4938,7 +4938,7 @@
49384938
supportsNormalization: false
49394939
supportsDBT: false
49404940
supported_destination_sync_modes: []
4941-
- dockerImage: "airbyte/source-github:0.4.3"
4941+
- dockerImage: "airbyte/source-github:0.4.4"
49424942
spec:
49434943
documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
49444944
connectionSpecification:
@@ -4955,6 +4955,7 @@
49554955
description: "Choose how to authenticate to GitHub"
49564956
type: "object"
49574957
order: 0
4958+
group: "auth"
49584959
oneOf:
49594960
- type: "object"
49604961
title: "OAuth"
@@ -11911,7 +11912,7 @@
1191111912
supportsNormalization: false
1191211913
supportsDBT: false
1191311914
supported_destination_sync_modes: []
11914-
- dockerImage: "airbyte/source-postgres:2.0.7"
11915+
- dockerImage: "airbyte/source-postgres:2.0.8"
1191511916
spec:
1191611917
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
1191711918
connectionSpecification:
@@ -11929,6 +11930,7 @@
1192911930
description: "Hostname of the database."
1193011931
type: "string"
1193111932
order: 0
11933+
group: "db"
1193211934
port:
1193311935
title: "Port"
1193411936
description: "Port of the database."
@@ -11939,11 +11941,13 @@
1193911941
examples:
1194011942
- "5432"
1194111943
order: 1
11944+
group: "db"
1194211945
database:
1194311946
title: "Database Name"
1194411947
description: "Name of the database."
1194511948
type: "string"
1194611949
order: 2
11950+
group: "db"
1194711951
schemas:
1194811952
title: "Schemas"
1194911953
description: "The list of schemas (case sensitive) to sync from. Defaults\
@@ -11956,17 +11960,21 @@
1195611960
default:
1195711961
- "public"
1195811962
order: 3
11963+
group: "db"
1195911964
username:
1196011965
title: "Username"
1196111966
description: "Username to access the database."
1196211967
type: "string"
1196311968
order: 4
11969+
group: "auth"
1196411970
password:
1196511971
title: "Password"
1196611972
description: "Password associated with the username."
1196711973
type: "string"
1196811974
airbyte_secret: true
1196911975
order: 5
11976+
group: "auth"
11977+
always_show: true
1197011978
jdbc_url_params:
1197111979
description: "Additional properties to pass to the JDBC URL string when\
1197211980
\ connecting to the database formatted as 'key=value' pairs separated\
@@ -11976,19 +11984,23 @@
1197611984
title: "JDBC URL Parameters (Advanced)"
1197711985
type: "string"
1197811986
order: 6
11987+
group: "advanced"
1197911988
ssl:
1198011989
title: "Connect using SSL"
1198111990
description: "Encrypt data using SSL. When activating SSL, please select\
1198211991
\ one of the connection modes."
1198311992
type: "boolean"
1198411993
default: false
1198511994
order: 7
11995+
group: "security"
11996+
always_show: true
1198611997
ssl_mode:
1198711998
title: "SSL Modes"
1198811999
description: "SSL connection modes. \n Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"\
1198912000
> in the docs</a>."
1199012001
type: "object"
11991-
order: 7
12002+
order: 8
12003+
group: "security"
1199212004
oneOf:
1199312005
- title: "disable"
1199412006
additionalProperties: true
@@ -12117,7 +12129,8 @@
1211712129
type: "object"
1211812130
title: "Replication Method"
1211912131
description: "Replication method for extracting data from the database."
12120-
order: 8
12132+
order: 9
12133+
group: "advanced"
1212112134
oneOf:
1212212135
- title: "Standard"
1212312136
description: "Standard replication requires no setup on the DB side but\
@@ -12294,6 +12307,14 @@
1229412307
type: "string"
1229512308
airbyte_secret: true
1229612309
order: 4
12310+
group: "security"
12311+
groups:
12312+
- id: "db"
12313+
- id: "auth"
12314+
- id: "security"
12315+
title: "Security"
12316+
- id: "advanced"
12317+
title: "Advanced"
1229712318
supportsNormalization: false
1229812319
supportsDBT: false
1229912320
supported_destination_sync_modes: []

‎airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,32 @@
99
import io.airbyte.commons.resources.MoreResources;
1010
import io.airbyte.protocol.models.v0.ConnectorSpecification;
1111
import java.io.IOException;
12+
import java.util.Optional;
1213

1314
public class SshHelpers {
1415

1516
public static ConnectorSpecification getSpecAndInjectSsh() throws IOException {
17+
return getSpecAndInjectSsh(Optional.empty());
18+
}
19+
20+
public static ConnectorSpecification getSpecAndInjectSsh(final Optional<String> group) throws IOException {
1621
final ConnectorSpecification originalSpec = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class);
17-
return injectSshIntoSpec(originalSpec);
22+
return injectSshIntoSpec(originalSpec, group);
1823
}
1924

2025
public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification) throws IOException {
26+
return injectSshIntoSpec(connectorSpecification, Optional.empty());
27+
}
28+
29+
public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification, final Optional<String> group)
30+
throws IOException {
2131
final ConnectorSpecification originalSpec = Jsons.clone(connectorSpecification);
2232
final ObjectNode propNode = (ObjectNode) originalSpec.getConnectionSpecification().get("properties");
23-
propNode.set("tunnel_method", Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json")));
33+
final ObjectNode tunnelMethod = (ObjectNode) Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json"));
34+
if (group.isPresent()) {
35+
tunnelMethod.put("group", group.get());
36+
}
37+
propNode.set("tunnel_method", tunnelMethod);
2438
return originalSpec;
2539
}
2640

‎airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
1717
import io.airbyte.protocol.models.v0.ConnectorSpecification;
1818
import java.util.List;
19+
import java.util.Optional;
1920
import org.slf4j.Logger;
2021
import org.slf4j.LoggerFactory;
2122

@@ -25,16 +26,25 @@ public class SshWrappedSource implements Source {
2526
private final Source delegate;
2627
private final List<String> hostKey;
2728
private final List<String> portKey;
29+
private final Optional<String> sshGroup;
2830

2931
public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey) {
3032
this.delegate = delegate;
3133
this.hostKey = hostKey;
3234
this.portKey = portKey;
35+
this.sshGroup = Optional.empty();
36+
}
37+
38+
public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey, final String sshGroup) {
39+
this.delegate = delegate;
40+
this.hostKey = hostKey;
41+
this.portKey = portKey;
42+
this.sshGroup = Optional.of(sshGroup);
3343
}
3444

3545
@Override
3646
public ConnectorSpecification spec() throws Exception {
37-
return SshHelpers.injectSshIntoSpec(delegate.spec());
47+
return SshHelpers.injectSshIntoSpec(delegate.spec(), sshGroup);
3848
}
3949

4050
@Override

‎airbyte-integrations/connectors/source-github/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ RUN pip install .
1212
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
1313
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1414

15-
LABEL io.airbyte.version=0.4.3
15+
LABEL io.airbyte.version=0.4.4
1616
LABEL io.airbyte.name=airbyte/source-github

‎airbyte-integrations/connectors/source-github/source_github/spec.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"description": "Choose how to authenticate to GitHub",
1313
"type": "object",
1414
"order": 0,
15+
"group": "auth",
1516
"oneOf": [
1617
{
1718
"type": "object",

‎airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=2.0.7
19+
LABEL io.airbyte.version=2.0.8
2020
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

‎airbyte-integrations/connectors/source-postgres/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=2.0.7
19+
LABEL io.airbyte.version=2.0.8
2020
LABEL io.airbyte.name=airbyte/source-postgres

‎airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public class PostgresSource extends AbstractJdbcSource<PostgresType> implements
117117
private static final Set<String> INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer");
118118

119119
public static Source sshWrappedSource() {
120-
return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY);
120+
return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security");
121121
}
122122

123123
PostgresSource() {

‎airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
"title": "Host",
1111
"description": "Hostname of the database.",
1212
"type": "string",
13-
"order": 0
13+
"order": 0,
14+
"group": "db"
1415
},
1516
"port": {
1617
"title": "Port",
@@ -20,13 +21,15 @@
2021
"maximum": 65536,
2122
"default": 5432,
2223
"examples": ["5432"],
23-
"order": 1
24+
"order": 1,
25+
"group": "db"
2426
},
2527
"database": {
2628
"title": "Database Name",
2729
"description": "Name of the database.",
2830
"type": "string",
29-
"order": 2
31+
"order": 2,
32+
"group": "db"
3033
},
3134
"schemas": {
3235
"title": "Schemas",
@@ -38,39 +41,47 @@
3841
"minItems": 0,
3942
"uniqueItems": true,
4043
"default": ["public"],
41-
"order": 3
44+
"order": 3,
45+
"group": "db"
4246
},
4347
"username": {
4448
"title": "Username",
4549
"description": "Username to access the database.",
4650
"type": "string",
47-
"order": 4
51+
"order": 4,
52+
"group": "auth"
4853
},
4954
"password": {
5055
"title": "Password",
5156
"description": "Password associated with the username.",
5257
"type": "string",
5358
"airbyte_secret": true,
54-
"order": 5
59+
"order": 5,
60+
"group": "auth",
61+
"always_show": true
5562
},
5663
"jdbc_url_params": {
5764
"description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
5865
"title": "JDBC URL Parameters (Advanced)",
5966
"type": "string",
60-
"order": 6
67+
"order": 6,
68+
"group": "advanced"
6169
},
6270
"ssl": {
6371
"title": "Connect using SSL",
6472
"description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.",
6573
"type": "boolean",
6674
"default": false,
67-
"order": 7
75+
"order": 7,
76+
"group": "security",
77+
"always_show": true
6878
},
6979
"ssl_mode": {
7080
"title": "SSL Modes",
7181
"description": "SSL connection modes. \n Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"> in the docs</a>.",
7282
"type": "object",
73-
"order": 7,
83+
"order": 8,
84+
"group": "security",
7485
"oneOf": [
7586
{
7687
"title": "disable",
@@ -218,7 +229,8 @@
218229
"type": "object",
219230
"title": "Replication Method",
220231
"description": "Replication method for extracting data from the database.",
221-
"order": 8,
232+
"order": 9,
233+
"group": "advanced",
222234
"oneOf": [
223235
{
224236
"title": "Standard",
@@ -276,14 +288,33 @@
276288
"type": "string",
277289
"title": "LSN commit behaviour",
278290
"description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.",
279-
"enum": ["While reading Data", "After loading Data in the destination"],
291+
"enum": [
292+
"While reading Data",
293+
"After loading Data in the destination"
294+
],
280295
"default": "After loading Data in the destination",
281296
"order": 6
282297
}
283298
}
284299
}
285300
]
286301
}
287-
}
302+
},
303+
"groups": [
304+
{
305+
"id": "db"
306+
},
307+
{
308+
"id": "auth"
309+
},
310+
{
311+
"id": "security",
312+
"title": "Security"
313+
},
314+
{
315+
"id": "advanced",
316+
"title": "Advanced"
317+
}
318+
]
288319
}
289320
}

‎airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package io.airbyte.integrations.io.airbyte.integration_tests.sources;
2+
3+
import io.airbyte.integrations.base.ssh.SshHelpers;
4+
import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
5+
import io.airbyte.protocol.models.v0.ConnectorSpecification;
6+
import java.util.Optional;
7+
8+
public abstract class AbstractPostgresSourceAcceptanceTest extends SourceAcceptanceTest {
9+
@Override
10+
protected String getImageName() {
11+
return "airbyte/source-postgres:dev";
12+
}
13+
14+
@Override
15+
protected ConnectorSpecification getSpec() throws Exception {
16+
return SshHelpers.getSpecAndInjectSsh(Optional.of("security"));
17+
}
18+
}

0 commit comments

Comments
 (0)