Skip to content

Commit c38c3be

Browse files
stephane-airbytestephane-airbyte
authored
enabling spotbugs for gcs-destinations submodule (#36703)
1 parent f82847c commit c38c3be

File tree

12 files changed

+49
-48
lines changed

12 files changed

+49
-48
lines changed

‎airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ abstract class DestinationAcceptanceTest {
178178
@Throws(Exception::class)
179179
protected abstract fun retrieveRecords(
180180
testEnv: TestDestinationEnv?,
181-
streamName: String?,
182-
namespace: String?,
181+
streamName: String,
182+
namespace: String,
183183
streamSchema: JsonNode
184184
): List<JsonNode>
185185

@@ -1454,7 +1454,7 @@ abstract class DestinationAcceptanceTest {
14541454
false
14551455
)
14561456
val destinationOutput =
1457-
retrieveRecords(testEnv, stream.name, getDefaultSchema(config), stream.jsonSchema)
1457+
retrieveRecords(testEnv, stream.name, getDefaultSchema(config)!!, stream.jsonSchema)
14581458
// Remove state message
14591459
secondSyncMessagesWithNewFields.removeIf {
14601460
airbyteMessage: io.airbyte.protocol.models.v0.AirbyteMessage ->

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle

-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ compileTestFixturesKotlin {
2020
}
2121
}
2222

23-
spotbugsTestFixtures.enabled = false
24-
2523
dependencies {
2624
implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies')
2725
implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt

+6-5
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,14 @@ abstract class GcsAvroParquetDestinationAcceptanceTest(s3Format: S3Format) :
9090
else fieldDefinition["type"]
9191
val airbyteTypeProperty = fieldDefinition["airbyte_type"]
9292
val airbyteTypePropertyText = airbyteTypeProperty?.asText()
93-
return Arrays.stream(JsonSchemaType.entries.toTypedArray())
93+
return JsonSchemaType.entries
94+
.toTypedArray()
9495
.filter { value: JsonSchemaType ->
9596
value.jsonSchemaType == typeProperty.asText() &&
9697
compareAirbyteTypes(airbyteTypePropertyText, value)
9798
}
98-
.map(JsonSchemaType::avroType)
99-
.collect(Collectors.toSet())
99+
.map { it.avroType }
100+
.toSet()
100101
}
101102

102103
private fun compareAirbyteTypes(
@@ -126,8 +127,8 @@ abstract class GcsAvroParquetDestinationAcceptanceTest(s3Format: S3Format) :
126127

127128
@Throws(Exception::class)
128129
protected abstract fun retrieveDataTypesFromPersistedFiles(
129-
streamName: String?,
130-
namespace: String?
130+
streamName: String,
131+
namespace: String
131132
): Map<String?, Set<Schema.Type?>?>
132133

133134
protected fun getTypes(record: GenericData.Record): Map<String, Set<Schema.Type>> {

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt

+8-8
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,17 @@ abstract class GcsBaseAvroDestinationAcceptanceTest :
3535
@Throws(Exception::class)
3636
override fun retrieveRecords(
3737
testEnv: TestDestinationEnv?,
38-
streamName: String?,
39-
namespace: String?,
38+
streamName: String,
39+
namespace: String,
4040
streamSchema: JsonNode
4141
): List<JsonNode> {
42-
val nameUpdater = getFieldNameUpdater(streamName!!, namespace, streamSchema)
42+
val nameUpdater = getFieldNameUpdater(streamName, namespace, streamSchema)
4343

44-
val objectSummaries = getAllSyncedObjects(streamName, namespace)
44+
val objectSummaries = getAllSyncedObjects(streamName, namespace!!)
4545
val jsonRecords: MutableList<JsonNode> = LinkedList()
4646

47-
for (objectSummary in objectSummaries!!) {
48-
val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key)
47+
for (objectSummary in objectSummaries) {
48+
val `object` = s3Client.getObject(objectSummary.bucketName, objectSummary.key)
4949
DataFileReader<GenericData.Record>(
5050
SeekableByteArrayInput(`object`.objectContent.readAllBytes()),
5151
GenericDatumReader<GenericData.Record>()
@@ -67,8 +67,8 @@ abstract class GcsBaseAvroDestinationAcceptanceTest :
6767

6868
@Throws(Exception::class)
6969
override fun retrieveDataTypesFromPersistedFiles(
70-
streamName: String?,
71-
namespace: String?
70+
streamName: String,
71+
namespace: String
7272
): Map<String?, Set<Schema.Type?>?> {
7373
val objectSummaries = getAllSyncedObjects(streamName, namespace)
7474
val resultDataTypes: MutableMap<String?, Set<Schema.Type?>?> = HashMap()

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ abstract class GcsBaseCsvDestinationAcceptanceTest : GcsDestinationAcceptanceTes
4040
@Throws(IOException::class)
4141
override fun retrieveRecords(
4242
testEnv: TestDestinationEnv?,
43-
streamName: String?,
44-
namespace: String?,
43+
streamName: String,
44+
namespace: String,
4545
streamSchema: JsonNode
4646
): List<JsonNode> {
4747
val objectSummaries = getAllSyncedObjects(streamName, namespace)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ abstract class GcsBaseJsonlDestinationAcceptanceTest :
3636
@Throws(IOException::class)
3737
override fun retrieveRecords(
3838
testEnv: TestDestinationEnv?,
39-
streamName: String?,
40-
namespace: String?,
39+
streamName: String,
40+
namespace: String,
4141
streamSchema: JsonNode
4242
): List<JsonNode> {
4343
val objectSummaries = getAllSyncedObjects(streamName, namespace)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt

+9-9
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,17 @@ abstract class GcsBaseParquetDestinationAcceptanceTest :
3737
@Throws(IOException::class, URISyntaxException::class)
3838
override fun retrieveRecords(
3939
testEnv: TestDestinationEnv?,
40-
streamName: String?,
41-
namespace: String?,
40+
streamName: String,
41+
namespace: String,
4242
streamSchema: JsonNode
4343
): List<JsonNode> {
44-
val nameUpdater = getFieldNameUpdater(streamName!!, namespace, streamSchema)
44+
val nameUpdater = getFieldNameUpdater(streamName, namespace, streamSchema)
4545

4646
val objectSummaries = getAllSyncedObjects(streamName, namespace)
4747
val jsonRecords: MutableList<JsonNode> = LinkedList()
4848

49-
for (objectSummary in objectSummaries!!) {
50-
val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key)
49+
for (objectSummary in objectSummaries) {
50+
val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key)
5151
val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key))
5252
val path = Path(uri)
5353
val hadoopConfig = GcsParquetWriter.getHadoopConfig(config)
@@ -73,14 +73,14 @@ abstract class GcsBaseParquetDestinationAcceptanceTest :
7373

7474
@Throws(Exception::class)
7575
override fun retrieveDataTypesFromPersistedFiles(
76-
streamName: String?,
77-
namespace: String?
76+
streamName: String,
77+
namespace: String
7878
): Map<String?, Set<Schema.Type?>?> {
7979
val objectSummaries = getAllSyncedObjects(streamName, namespace)
8080
val resultDataTypes: MutableMap<String?, Set<Schema.Type?>?> = HashMap()
8181

82-
for (objectSummary in objectSummaries!!) {
83-
val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key)
82+
for (objectSummary in objectSummaries) {
83+
val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key)
8484
val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key))
8585
val path = Path(uri)
8686
val hadoopConfig = getHadoopConfig(config)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.kt

+11-9
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import org.joda.time.DateTime
3030
import org.joda.time.DateTimeZone
3131
import org.junit.jupiter.api.Assertions
3232
import org.junit.jupiter.api.Test
33+
import org.mockito.Mockito.mock
3334
import org.slf4j.Logger
3435
import org.slf4j.LoggerFactory
3536

@@ -46,9 +47,10 @@ import org.slf4j.LoggerFactory
4647
abstract class GcsDestinationAcceptanceTest(protected val outputFormat: S3Format) :
4748
DestinationAcceptanceTest() {
4849
protected var configJson: JsonNode? = null
49-
protected lateinit var config: GcsDestinationConfig
50-
protected lateinit var s3Client: AmazonS3
51-
protected lateinit var nameTransformer: NamingConventionTransformer
50+
// Not a big fan of those mocks(). Here to make spotbugs happy
51+
protected var config: GcsDestinationConfig = mock()
52+
protected var s3Client: AmazonS3 = mock()
53+
protected var nameTransformer: NamingConventionTransformer = mock()
5254
protected var s3StorageOperations: S3StorageOperations? = null
5355

5456
protected val baseConfigJson: JsonNode
@@ -96,23 +98,23 @@ abstract class GcsDestinationAcceptanceTest(protected val outputFormat: S3Format
9698

9799
/** Helper method to retrieve all synced objects inside the configured bucket path. */
98100
protected fun getAllSyncedObjects(
99-
streamName: String?,
100-
namespace: String?
101+
streamName: String,
102+
namespace: String
101103
): List<S3ObjectSummary> {
102-
val namespaceStr = nameTransformer!!.getNamespace(namespace!!)
103-
val streamNameStr = nameTransformer!!.getIdentifier(streamName!!)
104+
val namespaceStr = nameTransformer.getNamespace(namespace)
105+
val streamNameStr = nameTransformer.getIdentifier(streamName)
104106
val outputPrefix =
105107
s3StorageOperations!!.getBucketObjectPath(
106108
namespaceStr,
107109
streamNameStr,
108110
DateTime.now(DateTimeZone.UTC),
109-
config!!.pathFormat!!
111+
config.pathFormat!!
110112
)
111113
// the child folder contains a non-deterministic epoch timestamp, so use the parent folder
112114
val parentFolder = outputPrefix.substring(0, outputPrefix.lastIndexOf("/") + 1)
113115
val objectSummaries =
114116
s3Client
115-
.listObjects(config!!.bucketName, parentFolder)
117+
.listObjects(config.bucketName, parentFolder)
116118
.objectSummaries
117119
.stream()
118120
.filter { o: S3ObjectSummary -> o.key.contains("$streamNameStr/") }

‎airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ abstract class S3BaseAvroDestinationAcceptanceTest protected constructor() :
3939
@Throws(Exception::class)
4040
override fun retrieveRecords(
4141
testEnv: TestDestinationEnv?,
42-
streamName: String?,
43-
namespace: String?,
42+
streamName: String,
43+
namespace: String,
4444
streamSchema: JsonNode
4545
): List<JsonNode> {
4646
val nameUpdater =

‎airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ abstract class S3BaseCsvDestinationAcceptanceTest : S3DestinationAcceptanceTest(
3636
@Throws(IOException::class)
3737
override fun retrieveRecords(
3838
testEnv: TestDestinationEnv?,
39-
streamName: String?,
40-
namespace: String?,
39+
streamName: String,
40+
namespace: String,
4141
streamSchema: JsonNode
4242
): List<JsonNode> {
4343
val objectSummaries = getAllSyncedObjects(streamName, namespace)

‎airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ abstract class S3BaseJsonlDestinationAcceptanceTest protected constructor() :
3535
@Throws(IOException::class)
3636
override fun retrieveRecords(
3737
testEnv: TestDestinationEnv?,
38-
streamName: String?,
39-
namespace: String?,
38+
streamName: String,
39+
namespace: String,
4040
streamSchema: JsonNode
4141
): List<JsonNode> {
4242
val objectSummaries = getAllSyncedObjects(streamName, namespace)

‎airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ abstract class S3BaseParquetDestinationAcceptanceTest protected constructor() :
2929
@Throws(IOException::class, URISyntaxException::class)
3030
override fun retrieveRecords(
3131
testEnv: TestDestinationEnv?,
32-
streamName: String?,
33-
namespace: String?,
32+
streamName: String,
33+
namespace: String,
3434
streamSchema: JsonNode
3535
): List<JsonNode> {
3636
val nameUpdater =

0 commit comments

Comments
 (0)