Skip to content

Commit 52d6e8d

Browse files
suhomudsuhomud
authored
13535 Fixed bastion network for integration tests (#14007)
1 parent 2cbee5a commit 52d6e8d

File tree

10 files changed

+43
-37
lines changed

10 files changed

+43
-37
lines changed

‎airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java

+5-14
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,9 @@ public class SshBastionContainer {
2222

2323
private static final String SSH_USER = "sshuser";
2424
private static final String SSH_PASSWORD = "secret";
25-
private Network network;
2625
private GenericContainer bastion;
2726

28-
public void initAndStartBastion() {
29-
network = Network.newNetwork();
27+
public void initAndStartBastion(Network network) {
3028
bastion = new GenericContainer(
3129
new ImageFromDockerfile("bastion-test")
3230
.withFileFromClasspath("Dockerfile", "bastion/Dockerfile"))
@@ -43,8 +41,7 @@ public JsonNode getTunnelConfig(final SshTunnel.TunnelMethod tunnelMethod, final
4341
.put("tunnel_host",
4442
Objects.requireNonNull(bastion.getContainerInfo().getNetworkSettings()
4543
.getNetworks()
46-
.get(((Network.NetworkImpl) network).getName())
47-
.getIpAddress()))
44+
.entrySet().stream().findFirst().get().getValue().getIpAddress()))
4845
.put("tunnel_method", tunnelMethod)
4946
.put("tunnel_port", bastion.getExposedPorts().get(0))
5047
.put("tunnel_user", SSH_USER)
@@ -66,25 +63,19 @@ public ImmutableMap.Builder<Object, Object> getBasicDbConfigBuider(final JdbcDat
6663
return ImmutableMap.builder()
6764
.put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
6865
.getNetworks()
69-
.get(((Network.NetworkImpl) getNetWork()).getName())
70-
.getIpAddress()))
66+
.entrySet().stream().findFirst().get().getValue().getIpAddress()))
7167
.put("username", db.getUsername())
7268
.put("password", db.getPassword())
7369
.put("port", db.getExposedPorts().get(0))
7470
.put("database", schemaName)
7571
.put("ssl", false);
7672
}
7773

78-
public Network getNetWork() {
79-
return this.network;
80-
}
81-
8274
public void stopAndCloseContainers(final JdbcDatabaseContainer<?> db) {
83-
db.stop();
84-
db.close();
8575
bastion.stop();
8676
bastion.close();
87-
network.close();
77+
db.stop();
78+
db.close();
8879
}
8980

9081
}

‎airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.stream.Collectors;
2323
import org.junit.jupiter.api.Disabled;
2424
import org.testcontainers.containers.ClickHouseContainer;
25+
import org.testcontainers.containers.Network;
2526

2627
/**
2728
* Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file
@@ -32,6 +33,7 @@ public abstract class SshClickhouseDestinationAcceptanceTest extends Destination
3233
public abstract SshTunnel.TunnelMethod getTunnelMethod();
3334

3435
private static final String DB_NAME = "default";
36+
private static final Network network = Network.newNetwork();
3537

3638
private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
3739

@@ -158,8 +160,8 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
158160

159161
@Override
160162
protected void setup(final TestDestinationEnv testEnv) {
161-
bastion.initAndStartBastion();
162-
db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server").withNetwork(bastion.getNetWork());
163+
bastion.initAndStartBastion(network);
164+
db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server").withNetwork(network);
163165
db.start();
164166
}
165167

‎airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.slf4j.Logger;
2626
import org.slf4j.LoggerFactory;
2727
import org.testcontainers.containers.MariaDBContainer;
28+
import org.testcontainers.containers.Network;
2829
import org.testcontainers.utility.DockerImageName;
2930

3031
/**
@@ -34,6 +35,7 @@
3435
public abstract class SshMariadbColumnstoreDestinationAcceptanceTest extends DestinationAcceptanceTest {
3536

3637
private static final Logger LOGGER = LoggerFactory.getLogger(MariadbColumnstoreDestinationAcceptanceTest.class);
38+
private static final Network network = Network.newNetwork();
3739

3840
private final ExtendedNameTransformer namingResolver = new MariadbColumnstoreNameTransformer();
3941

@@ -128,14 +130,14 @@ protected List<String> resolveIdentifier(final String identifier) {
128130

129131
@Override
130132
protected void setup(final TestDestinationEnv testEnv) throws Exception {
131-
bastion.initAndStartBastion();
133+
bastion.initAndStartBastion(network);
132134
startAndInitJdbcContainer();
133135
}
134136

135137
private void startAndInitJdbcContainer() throws Exception {
136138
final DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb");
137139
db = new MariaDBContainer<>(mcsImage)
138-
.withNetwork(bastion.getNetWork());
140+
.withNetwork(network);
139141
db.start();
140142

141143
final String createUser = String.format("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", db.getUsername(), db.getPassword());

‎airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public abstract class SshMSSQLDestinationAcceptanceTest extends DestinationAccep
3939
private final String schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase();
4040
private static final String database = "test";
4141
private static MSSQLServerContainer<?> db;
42+
private static final Network network = Network.newNetwork();
4243
private final SshBastionContainer bastion = new SshBastionContainer();
4344

4445
public abstract SshTunnel.TunnelMethod getTunnelMethod();
@@ -111,7 +112,7 @@ public ImmutableMap.Builder<Object, Object> getMSSQLDbConfigBuilder(final JdbcDa
111112
return ImmutableMap.builder()
112113
.put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
113114
.getNetworks()
114-
.get(((Network.NetworkImpl) bastion.getNetWork()).getName())
115+
.get(((Network.NetworkImpl) network).getName())
115116
.getIpAddress()))
116117
.put("username", db.getUsername())
117118
.put("password", db.getPassword())
@@ -173,13 +174,13 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception {
173174
}
174175

175176
private void startTestContainers() {
176-
bastion.initAndStartBastion();
177+
bastion.initAndStartBastion(network);
177178
initAndStartJdbcContainer();
178179
}
179180

180181
private void initAndStartJdbcContainer() {
181182
db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04")
182-
.withNetwork(bastion.getNetWork())
183+
.withNetwork(network)
183184
.acceptLicense();
184185
db.start();
185186
}

‎airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ public abstract class SshOracleDestinationAcceptanceTest extends DestinationAcce
3434

3535
private final SshBastionContainer sshBastionContainer = new SshBastionContainer();
3636

37+
private static final Network network = Network.newNetwork();
38+
3739
private OracleContainer db;
3840

3941
public abstract SshTunnel.TunnelMethod getTunnelMethod();
@@ -53,7 +55,7 @@ public ImmutableMap.Builder<Object, Object> getBasicOracleDbConfigBuilder(final
5355
return ImmutableMap.builder()
5456
.put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
5557
.getNetworks()
56-
.get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName())
58+
.get(((Network.NetworkImpl) network).getName())
5759
.getIpAddress()))
5860
.put("username", db.getUsername())
5961
.put("password", db.getPassword())
@@ -143,7 +145,7 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception {
143145
}
144146

145147
private void startTestContainers() {
146-
sshBastionContainer.initAndStartBastion();
148+
sshBastionContainer.initAndStartBastion(network);
147149
initAndStartJdbcContainer();
148150
}
149151

@@ -152,7 +154,7 @@ private void initAndStartJdbcContainer() {
152154
.withUsername("test")
153155
.withPassword("oracle")
154156
.usingSid()
155-
.withNetwork(sshBastionContainer.getNetWork());
157+
.withNetwork(network);
156158
db.start();
157159
}
158160

‎airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.stream.Collectors;
2222
import org.apache.commons.lang3.RandomStringUtils;
2323
import org.jooq.SQLDialect;
24+
import org.testcontainers.containers.Network;
2425
import org.testcontainers.containers.PostgreSQLContainer;
2526

2627
// todo (cgardens) - likely some of this could be further de-duplicated with
@@ -34,6 +35,7 @@ public abstract class SshPostgresDestinationAcceptanceTest extends JdbcDestinati
3435

3536
private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
3637
private static final String schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase();
38+
private static final Network network = Network.newNetwork();
3739
private static PostgreSQLContainer<?> db;
3840
private final SshBastionContainer bastion = new SshBastionContainer();
3941

@@ -154,13 +156,13 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception {
154156
}
155157

156158
private void startTestContainers() {
157-
bastion.initAndStartBastion();
159+
bastion.initAndStartBastion(network);
158160
initAndStartJdbcContainer();
159161
}
160162

161163
private void initAndStartJdbcContainer() {
162164
db = new PostgreSQLContainer<>("postgres:13-alpine")
163-
.withNetwork(bastion.getNetWork());
165+
.withNetwork(network);
164166
db.start();
165167
}
166168

‎airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.util.HashMap;
3030
import javax.sql.DataSource;
3131
import org.testcontainers.containers.ClickHouseContainer;
32+
import org.testcontainers.containers.Network;
3233

3334
public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest {
3435

@@ -38,6 +39,7 @@ public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAc
3839
private static final String STREAM_NAME = "id_and_name";
3940
private static final String STREAM_NAME2 = "starships";
4041
private static final String SCHEMA_NAME = "default";
42+
private static final Network network = Network.newNetwork();
4143

4244
public abstract SshTunnel.TunnelMethod getTunnelMethod();
4345

@@ -93,12 +95,12 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
9395
}
9496

9597
private void startTestContainers() {
96-
bastion.initAndStartBastion();
98+
bastion.initAndStartBastion(network);
9799
initAndStartJdbcContainer();
98100
}
99101

100102
private void initAndStartJdbcContainer() {
101-
db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine").withNetwork(bastion.getNetWork());
103+
db = (ClickHouseContainer) new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine").withNetwork(network);
102104
db.start();
103105
}
104106

‎airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,11 @@ public abstract class AbstractSshMssqlSourceAcceptanceTest extends SourceAccepta
3737

3838
private static final String STREAM_NAME = "dbo.id_and_name";
3939
private static final String STREAM_NAME2 = "dbo.starships";
40+
private static final Network network = Network.newNetwork();
41+
private static JsonNode config;
4042
private String dbName;
4143
private MSSQLServerContainer<?> db;
4244
private final SshBastionContainer bastion = new SshBastionContainer();
43-
private static JsonNode config;
4445

4546
public abstract SshTunnel.TunnelMethod getTunnelMethod();
4647

@@ -56,7 +57,7 @@ public ImmutableMap.Builder<Object, Object> getMSSQLDbConfigBuilder(final JdbcDa
5657
return ImmutableMap.builder()
5758
.put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
5859
.getNetworks()
59-
.get(((Network.NetworkImpl) bastion.getNetWork()).getName())
60+
.get(((Network.NetworkImpl) network).getName())
6061
.getIpAddress()))
6162
.put("username", db.getUsername())
6263
.put("password", db.getPassword())
@@ -77,13 +78,13 @@ private static Database getDatabaseFromConfig(final JsonNode config) {
7778
}
7879

7980
private void startTestContainers() {
80-
bastion.initAndStartBastion();
81+
bastion.initAndStartBastion(network);
8182
initAndStartJdbcContainer();
8283
}
8384

8485
private void initAndStartJdbcContainer() {
8586
db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2017-latest")
86-
.withNetwork(bastion.getNetWork())
87+
.withNetwork(network)
8788
.acceptLicense();
8889
db.start();
8990
}

‎airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAccept
3434

3535
private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME";
3636
private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS";
37+
private static final Network network = Network.newNetwork();
3738
private final SshBastionContainer sshBastionContainer = new SshBastionContainer();
3839
private OracleContainer db;
3940

@@ -84,7 +85,7 @@ protected void tearDown(final TestDestinationEnv testEnv) {
8485
}
8586

8687
private void startTestContainers() {
87-
sshBastionContainer.initAndStartBastion();
88+
sshBastionContainer.initAndStartBastion(network);
8889
initAndStartJdbcContainer();
8990
}
9091

@@ -93,7 +94,7 @@ private void initAndStartJdbcContainer() {
9394
.withUsername("test")
9495
.withPassword("oracle")
9596
.usingSid()
96-
.withNetwork(sshBastionContainer.getNetWork());;
97+
.withNetwork(network);
9798
db.start();
9899
}
99100

@@ -111,7 +112,7 @@ public ImmutableMap.Builder<Object, Object> getBasicOracleDbConfigBuider(final O
111112
return ImmutableMap.builder()
112113
.put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
113114
.getNetworks()
114-
.get(((Network.NetworkImpl) sshBastionContainer.getNetWork()).getName())
115+
.get(((Network.NetworkImpl) network).getName())
115116
.getIpAddress()))
116117
.put("username", db.getUsername())
117118
.put("password", db.getPassword())

‎airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,14 @@
2727
import java.util.HashMap;
2828
import java.util.List;
2929
import org.jooq.SQLDialect;
30+
import org.testcontainers.containers.Network;
3031
import org.testcontainers.containers.PostgreSQLContainer;
3132

3233
public abstract class AbstractSshPostgresSourceAcceptanceTest extends SourceAcceptanceTest {
3334

3435
private static final String STREAM_NAME = "public.id_and_name";
3536
private static final String STREAM_NAME2 = "public.starships";
37+
private static final Network network = Network.newNetwork();
3638
private static JsonNode config;
3739
private final SshBastionContainer bastion = new SshBastionContainer();
3840
private PostgreSQLContainer<?> db;
@@ -78,12 +80,12 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
7880
}
7981

8082
private void startTestContainers() {
81-
bastion.initAndStartBastion();
83+
bastion.initAndStartBastion(network);
8284
initAndStartJdbcContainer();
8385
}
8486

8587
private void initAndStartJdbcContainer() {
86-
db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(bastion.getNetWork());
88+
db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network);
8789
db.start();
8890
}
8991

0 commit comments

Comments
 (0)