Skip to content

Commit e4661fb

Browse files
tulirentuliren
authored
Remove regex check from Java source acceptance test (#9829)
* Move getRegexTests to python source acceptance test * Remove unused imports * Update test template
1 parent d7052a4 commit e4661fb

File tree

37 files changed

+43
-229
lines changed

37 files changed

+43
-229
lines changed

‎airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
1313
import io.airbyte.protocol.models.ConnectorSpecification;
1414
import java.nio.file.Path;
15-
import java.util.ArrayList;
16-
import java.util.List;
1715
import javax.annotation.Nullable;
1816

1917
/**
@@ -94,11 +92,6 @@ protected JsonNode getState() {
9492

9593
}
9694

97-
@Override
98-
protected List<String> getRegexTests() throws Exception {
99-
return new ArrayList<>();
100-
}
101-
10295
@Override
10396
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
10497
// no-op, for now

‎airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
package io.airbyte.integrations.standardtest.source;
66

7-
import static java.util.stream.Collectors.toList;
7+
import static org.junit.jupiter.api.Assertions.assertTrue;
88

99
import com.fasterxml.jackson.databind.JsonNode;
1010
import com.google.common.collect.Lists;
@@ -13,6 +13,7 @@
1313
import io.airbyte.commons.io.LineGobbler;
1414
import io.airbyte.commons.json.Jsons;
1515
import io.airbyte.config.EnvConfigs;
16+
import io.airbyte.protocol.models.AirbyteMessage;
1617
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
1718
import io.airbyte.protocol.models.ConnectorSpecification;
1819
import io.airbyte.workers.WorkerConfigs;
@@ -65,10 +66,15 @@ protected JsonNode getState() throws IOException {
6566
}
6667

6768
@Override
68-
protected List<String> getRegexTests() throws IOException {
69-
return Streams.stream(runExecutable(Command.GET_REGEX_TESTS).withArray("tests").elements())
70-
.map(JsonNode::textValue)
71-
.collect(toList());
69+
protected void assertFullRefreshMessages(final List<AirbyteMessage> allMessages) throws IOException {
70+
final List<String> regexTests = Streams.stream(runExecutable(Command.GET_REGEX_TESTS).withArray("tests").elements())
71+
.map(JsonNode::textValue).toList();
72+
final List<String> stringMessages = allMessages.stream().map(Jsons::serialize).toList();
73+
LOGGER.info("Running " + regexTests.size() + " regex tests...");
74+
regexTests.forEach(regex -> {
75+
LOGGER.info("Looking for [" + regex + "]");
76+
assertTrue(stringMessages.stream().anyMatch(line -> line.matches(regex)), "Failed to find regex: " + regex);
77+
});
7278
}
7379

7480
@Override

‎airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java

Lines changed: 6 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,6 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest {
105105
*/
106106
protected abstract JsonNode getState() throws Exception;
107107

108-
/**
109-
* List of regular expressions that should match the output of the test sync.
110-
*
111-
* @return the regular expressions to test
112-
* @throws Exception - thrown when attempting ot access the regexes fails
113-
*/
114-
protected abstract List<String> getRegexTests() throws Exception;
115-
116108
/**
117109
* Verify that a spec operation issued to the connector returns a valid spec.
118110
*/
@@ -160,26 +152,15 @@ public void testDiscover() throws Exception {
160152
public void testFullRefreshRead() throws Exception {
161153
final ConfiguredAirbyteCatalog catalog = withFullRefreshSyncModes(getConfiguredCatalog());
162154
final List<AirbyteMessage> allMessages = runRead(catalog);
163-
final List<AirbyteRecordMessage> recordMessages = filterRecords(allMessages);
164-
// the worker validates the message formats, so we just validate the message content
165-
// We don't need to validate message format as long as we use the worker, which we will not want to
166-
// do long term.
167-
assertFalse(recordMessages.isEmpty(), "Expected a full refresh sync to produce records");
168-
assertFullRefreshRecordMessages(recordMessages);
169-
170-
final List<String> regexTests = getRegexTests();
171-
final List<String> stringMessages = allMessages.stream().map(Jsons::serialize).collect(Collectors.toList());
172-
LOGGER.info("Running " + regexTests.size() + " regex tests...");
173-
regexTests.forEach(regex -> {
174-
LOGGER.info("Looking for [" + regex + "]");
175-
assertTrue(stringMessages.stream().anyMatch(line -> line.matches(regex)), "Failed to find regex: " + regex);
176-
});
155+
156+
assertFalse(filterRecords(allMessages).isEmpty(), "Expected a full refresh sync to produce records");
157+
assertFullRefreshMessages(allMessages);
177158
}
178159

179160
/**
180-
* Override this method to perform more specific assertion on the record messages.
161+
* Override this method to perform more specific assertion on the messages.
181162
*/
182-
protected void assertFullRefreshRecordMessages(final List<AirbyteRecordMessage> recordMessages) {
163+
protected void assertFullRefreshMessages(final List<AirbyteMessage> allMessages) throws Exception {
183164
// do nothing by default
184165
}
185166

@@ -290,7 +271,7 @@ public void testEntrypointEnvVar() throws Exception {
290271
checkEntrypointEnvVariable();
291272
}
292273

293-
private List<AirbyteRecordMessage> filterRecords(final Collection<AirbyteMessage> messages) {
274+
protected static List<AirbyteRecordMessage> filterRecords(final Collection<AirbyteMessage> messages) {
294275
return messages.stream()
295276
.filter(m -> m.getType() == Type.RECORD)
296277
.map(AirbyteMessage::getRecord)

‎airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,6 @@ public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTes
5555
return null;
5656
}
5757

58-
@Override
59-
protected List<String> getRegexTests() {
60-
return Collections.emptyList();
61-
}
62-
6358
@Override
6459
protected JsonNode getState() {
6560
return Jsons.jsonNode(new HashMap<>());

‎airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
9595
Field.of("name", JsonSchemaPrimitive.STRING));
9696
}
9797

98-
@Override
99-
protected List<String> getRegexTests() {
100-
return Collections.emptyList();
101-
}
102-
10398
@Override
10499
protected JsonNode getState() {
105100
return Jsons.jsonNode(new HashMap<>());

‎airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,6 @@ protected JsonNode getState() {
8383
return Jsons.jsonNode(new HashMap<>());
8484
}
8585

86-
@Override
87-
protected List<String> getRegexTests() {
88-
return Collections.emptyList();
89-
}
90-
9186
@Override
9287
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
9388
startTestContainers();

‎airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,6 @@ protected JsonNode getState() {
7979
return Jsons.jsonNode(new HashMap<>());
8080
}
8181

82-
@Override
83-
protected List<String> getRegexTests() {
84-
return Collections.emptyList();
85-
}
86-
8782
@Override
8883
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
8984
db = new ClickHouseContainer("yandex/clickhouse-server:21.8.8.29-alpine");

‎airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
116116
Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))));
117117
}
118118

119-
@Override
120-
protected List<String> getRegexTests() {
121-
return Collections.emptyList();
122-
}
123-
124119
@Override
125120
protected JsonNode getState() {
126121
return Jsons.jsonNode(new HashMap<>());

‎airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
118118
Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))));
119119
}
120120

121-
@Override
122-
protected List<String> getRegexTests() {
123-
return Collections.emptyList();
124-
}
125-
126121
@Override
127122
protected JsonNode getState() {
128123
return Jsons.jsonNode(new HashMap<>());

‎airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,15 @@ protected JsonNode getState() {
9191
}
9292

9393
@Override
94-
protected List<String> getRegexTests() {
95-
return Collections.emptyList();
96-
}
97-
98-
@Override
99-
protected void setupEnvironment(TestDestinationEnv environment) throws Exception {
94+
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
10095
db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense()
10196
.withExposedPorts(50000);
10297
db.start();
10398

104-
var certificate = getCertificate();
99+
final var certificate = getCertificate();
105100
try {
106101
convertAndImportCertificate(certificate);
107-
} catch (IOException | InterruptedException e) {
102+
} catch (final IOException | InterruptedException e) {
108103
throw new RuntimeException("Failed to import certificate into Java Keystore");
109104
}
110105

@@ -121,7 +116,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception
121116
.build()))
122117
.build());
123118

124-
String jdbcUrl = String.format("jdbc:db2://%s:%s/%s",
119+
final String jdbcUrl = String.format("jdbc:db2://%s:%s/%s",
125120
config.get("host").asText(),
126121
db.getMappedPort(50000),
127122
config.get("db").asText()) + SSL_CONFIG;
@@ -154,7 +149,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception
154149
}
155150

156151
@Override
157-
protected void tearDown(TestDestinationEnv testEnv) {
152+
protected void tearDown(final TestDestinationEnv testEnv) {
158153
new File("certificate.pem").delete();
159154
new File("certificate.der").delete();
160155
new File(KEY_STORE_FILE_PATH).delete();
@@ -186,9 +181,9 @@ private String getCertificate() throws IOException, InterruptedException {
186181
return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout();
187182
}
188183

189-
private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException {
190-
Runtime run = Runtime.getRuntime();
191-
try (PrintWriter out = new PrintWriter("certificate.pem")) {
184+
private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException {
185+
final Runtime run = Runtime.getRuntime();
186+
try (final PrintWriter out = new PrintWriter("certificate.pem")) {
192187
out.print(certificate);
193188
}
194189
runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);
@@ -198,8 +193,8 @@ private static void convertAndImportCertificate(String certificate) throws IOExc
198193
run);
199194
}
200195

201-
private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException {
202-
Process pr = run.exec(cmd);
196+
private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException {
197+
final Process pr = run.exec(cmd);
203198
if (!pr.waitFor(30, TimeUnit.SECONDS)) {
204199
pr.destroy();
205200
throw new RuntimeException("Timeout while executing: " + cmd);

‎airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,6 @@ protected JsonNode getState() throws Exception {
8181
return Jsons.jsonNode(new HashMap<>());
8282
}
8383

84-
@Override
85-
protected List<String> getRegexTests() throws Exception {
86-
return Collections.emptyList();
87-
}
88-
8984
@Override
9085
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
9186
db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense();

‎airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -89,20 +89,15 @@ protected JsonNode getState() {
8989
}
9090

9191
@Override
92-
protected List<String> getRegexTests() {
93-
return Collections.emptyList();
94-
}
95-
96-
@Override
97-
protected void setupEnvironment(TestDestinationEnv environment) throws Exception {
92+
protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
9893
db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense()
9994
.withExposedPorts(50000);
10095
db.start();
10196

102-
var certificate = getCertificate();
97+
final var certificate = getCertificate();
10398
try {
10499
convertAndImportCertificate(certificate);
105-
} catch (IOException | InterruptedException e) {
100+
} catch (final IOException | InterruptedException e) {
106101
throw new RuntimeException("Failed to import certificate into Java Keystore");
107102
}
108103

@@ -119,7 +114,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception
119114
.build()))
120115
.build());
121116

122-
String jdbcUrl = String.format("jdbc:db2://%s:%s/%s",
117+
final String jdbcUrl = String.format("jdbc:db2://%s:%s/%s",
123118
config.get("host").asText(),
124119
db.getMappedPort(50000),
125120
config.get("db").asText()) + ":sslConnection=true;sslTrustStoreLocation=" + KEY_STORE_FILE_PATH +
@@ -153,7 +148,7 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception
153148
}
154149

155150
@Override
156-
protected void tearDown(TestDestinationEnv testEnv) {
151+
protected void tearDown(final TestDestinationEnv testEnv) {
157152
new File("certificate.pem").delete();
158153
new File("certificate.der").delete();
159154
new File(KEY_STORE_FILE_PATH).delete();
@@ -180,9 +175,9 @@ private String getCertificate() throws IOException, InterruptedException {
180175
return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout();
181176
}
182177

183-
private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException {
184-
Runtime run = Runtime.getRuntime();
185-
try (PrintWriter out = new PrintWriter("certificate.pem")) {
178+
private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException {
179+
final Runtime run = Runtime.getRuntime();
180+
try (final PrintWriter out = new PrintWriter("certificate.pem")) {
186181
out.print(certificate);
187182
}
188183
runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);
@@ -192,8 +187,8 @@ private static void convertAndImportCertificate(String certificate) throws IOExc
192187
run);
193188
}
194189

195-
private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException {
196-
Process pr = run.exec(cmd);
190+
private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException {
191+
final Process pr = run.exec(cmd);
197192
if (!pr.waitFor(30, TimeUnit.SECONDS)) {
198193
pr.destroy();
199194
throw new RuntimeException("Timeout while executing: " + cmd);

‎airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType;
1616
import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
1717
import io.airbyte.integrations.standardtest.source.TestDestinationEnv;
18+
import io.airbyte.protocol.models.AirbyteMessage;
1819
import io.airbyte.protocol.models.AirbyteRecordMessage;
1920
import io.airbyte.protocol.models.CatalogHelpers;
2021
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
2122
import io.airbyte.protocol.models.ConnectorSpecification;
2223
import io.airbyte.validation.json.JsonSchemaValidator;
2324
import io.airbyte.validation.json.JsonValidationException;
2425
import java.io.IOException;
25-
import java.util.Collections;
2626
import java.util.HashMap;
2727
import java.util.List;
2828
import java.util.concurrent.ThreadLocalRandom;
@@ -115,12 +115,9 @@ protected JsonNode getState() {
115115
}
116116

117117
@Override
118-
protected List<String> getRegexTests() {
119-
return Collections.emptyList();
120-
}
118+
protected void assertFullRefreshMessages(final List<AirbyteMessage> allMessages) {
119+
final List<AirbyteRecordMessage> recordMessages = filterRecords(allMessages);
121120

122-
@Override
123-
protected void assertFullRefreshRecordMessages(final List<AirbyteRecordMessage> recordMessages) {
124121
int index = 0;
125122
// the first N messages are from stream 1
126123
while (index < MAX_MESSAGES) {

‎airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType;
1616
import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
1717
import io.airbyte.integrations.standardtest.source.TestDestinationEnv;
18+
import io.airbyte.protocol.models.AirbyteMessage;
1819
import io.airbyte.protocol.models.AirbyteRecordMessage;
1920
import io.airbyte.protocol.models.CatalogHelpers;
2021
import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
2122
import io.airbyte.protocol.models.ConnectorSpecification;
2223
import io.airbyte.validation.json.JsonSchemaValidator;
2324
import io.airbyte.validation.json.JsonValidationException;
2425
import java.io.IOException;
25-
import java.util.Collections;
2626
import java.util.HashMap;
2727
import java.util.List;
2828
import java.util.concurrent.ThreadLocalRandom;
@@ -111,12 +111,9 @@ protected JsonNode getState() {
111111
}
112112

113113
@Override
114-
protected List<String> getRegexTests() {
115-
return Collections.emptyList();
116-
}
114+
protected void assertFullRefreshMessages(final List<AirbyteMessage> allMessages) {
115+
final List<AirbyteRecordMessage> recordMessages = filterRecords(allMessages);
117116

118-
@Override
119-
protected void assertFullRefreshRecordMessages(final List<AirbyteRecordMessage> recordMessages) {
120117
int index = 0;
121118
// the first N messages are from stream 1
122119
while (index < MAX_MESSAGES) {

0 commit comments

Comments
 (0)