Skip to content

Commit f493c7b

Browse files
gisripagisripa
authored
Rename File format related classes to be agnostic of S3 (#37442)
1 parent 7adfb7a commit f493c7b

File tree

55 files changed

+253
-242
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+253
-242
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ Maven and Gradle will automatically reference the correct (pinned) version of th
144144

145145
| Version | Date | Pull Request | Subject |
146146
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
147+
| 0.30.6 | 2024-04-19 | [\#37442](https://github.com/airbytehq/airbyte/pull/37442) | Destinations: Rename File format related classes to be agnostic of S3 |
147148
| 0.30.3 | 2024-04-12 | [\#37106](https://github.com/airbytehq/airbyte/pull/37106) | Destinations: Simplify constructors in `AsyncStreamConsumer` |
148149
| 0.30.2 | 2024-04-12 | [\#36926](https://github.com/airbytehq/airbyte/pull/36926) | Destinations: Remove `JdbcSqlOperations#formatData`; misc changes for java interop |
149150
| 0.30.1 | 2024-04-11 | [\#36919](https://github.com/airbytehq/airbyte/pull/36919) | Fix regression in sources conversion of null values |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.30.5
1+
version=0.30.6

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.kt

+4-4
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialType
1515
import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig
1616
import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig
1717
import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants
18-
import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig
19-
import io.airbyte.cdk.integrations.destination.s3.S3FormatConfigs.getS3FormatConfig
2018
import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations
19+
import io.airbyte.cdk.integrations.destination.s3.UploadFormatConfig
20+
import io.airbyte.cdk.integrations.destination.s3.UploadFormatConfigFactory.getUploadFormatConfig
2121

2222
/**
2323
* Currently we always reuse the S3 client for GCS. So the GCS config extends from the S3 config.
@@ -28,7 +28,7 @@ class GcsDestinationConfig(
2828
bucketPath: String,
2929
bucketRegion: String?,
3030
val gcsCredentialConfig: GcsCredentialConfig,
31-
formatConfig: S3FormatConfig
31+
formatConfig: UploadFormatConfig
3232
) :
3333
S3DestinationConfig(
3434
GCS_ENDPOINT,
@@ -76,7 +76,7 @@ class GcsDestinationConfig(
7676
config["gcs_bucket_path"].asText(),
7777
config["gcs_bucket_region"].asText(),
7878
GcsCredentialConfigs.getCredentialConfig(config),
79-
getS3FormatConfig(config)
79+
getUploadFormatConfig(config)
8080
)
8181
}
8282
}

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.kt

+6-7
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ import com.fasterxml.jackson.databind.JsonNode
1010
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
1111
import io.airbyte.cdk.integrations.destination.gcs.util.GcsUtils
1212
import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter
13-
import io.airbyte.cdk.integrations.destination.s3.S3Format
13+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1414
import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory
1515
import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter
16-
import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig
16+
import io.airbyte.cdk.integrations.destination.s3.avro.UploadAvroFormatConfig
1717
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory
1818
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.create
1919
import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter
@@ -63,8 +63,7 @@ constructor(
6363
)
6464
LOGGER.info("Avro schema for stream {}: {}", stream.name, schema!!.toString(false))
6565

66-
val outputFilename: String =
67-
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, S3Format.AVRO)
66+
val outputFilename: String = getOutputFilename(uploadTimestamp, FileUploadFormat.AVRO)
6867
outputPath = java.lang.String.join("/", outputPrefix, outputFilename)
6968
fileLocation = String.format("gs://%s/%s", config.bucketName, outputPath)
7069

@@ -84,7 +83,7 @@ constructor(
8483
// performant.
8584
this.outputStream = uploadManager.multiPartOutputStreams[0]
8685

87-
val formatConfig = config.formatConfig as S3AvroFormatConfig
86+
val formatConfig = config.formatConfig as UploadAvroFormatConfig
8887
// The DataFileWriter always uses binary encoding.
8988
// If json encoding is needed in the future, use the GenericDatumWriter directly.
9089
this.dataFileWriter =
@@ -118,8 +117,8 @@ constructor(
118117
uploadManager.abort()
119118
}
120119

121-
override val fileFormat: S3Format
122-
get() = S3Format.AVRO
120+
override val fileFormat: FileUploadFormat
121+
get() = FileUploadFormat.AVRO
123122

124123
companion object {
125124
protected val LOGGER: Logger = LoggerFactory.getLogger(GcsAvroWriter::class.java)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.kt

+6-6
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ import com.amazonaws.services.s3.AmazonS3
99
import com.fasterxml.jackson.databind.JsonNode
1010
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
1111
import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter
12-
import io.airbyte.cdk.integrations.destination.s3.S3Format
12+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1313
import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator
1414
import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator.Factory.create
15-
import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig
15+
import io.airbyte.cdk.integrations.destination.s3.csv.UploadCsvFormatConfig
1616
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory
1717
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.create
1818
import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter
@@ -43,11 +43,11 @@ class GcsCsvWriter(
4343
override val outputPath: String
4444

4545
init {
46-
val formatConfig = config.formatConfig as S3CsvFormatConfig
46+
val formatConfig = config.formatConfig as UploadCsvFormatConfig
4747
this.csvSheetGenerator = create(configuredStream.stream.jsonSchema, formatConfig)
4848

4949
val outputFilename: String =
50-
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, S3Format.CSV)
50+
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, FileUploadFormat.CSV)
5151
outputPath = java.lang.String.join("/", outputPrefix, outputFilename)
5252
fileLocation = String.format("gs://%s/%s", config.bucketName, outputPath)
5353

@@ -97,8 +97,8 @@ class GcsCsvWriter(
9797
uploadManager.abort()
9898
}
9999

100-
override val fileFormat: S3Format
101-
get() = S3Format.CSV
100+
override val fileFormat: FileUploadFormat
101+
get() = FileUploadFormat.CSV
102102

103103
companion object {
104104
private val LOGGER: Logger = LoggerFactory.getLogger(GcsCsvWriter::class.java)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.kt

+4-4
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
1111
import io.airbyte.cdk.integrations.base.JavaBaseConstants
1212
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
1313
import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter
14-
import io.airbyte.cdk.integrations.destination.s3.S3Format
14+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1515
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.create
1616
import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter
1717
import io.airbyte.commons.jackson.MoreMappers
@@ -40,7 +40,7 @@ class GcsJsonlWriter(
4040

4141
init {
4242
val outputFilename: String =
43-
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, S3Format.JSONL)
43+
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, FileUploadFormat.JSONL)
4444
outputPath = java.lang.String.join("/", outputPrefix, outputFilename)
4545

4646
fileLocation = String.format("gs://%s/%s", config.bucketName, outputPath)
@@ -84,8 +84,8 @@ class GcsJsonlWriter(
8484
uploadManager.abort()
8585
}
8686

87-
override val fileFormat: S3Format
88-
get() = S3Format.JSONL
87+
override val fileFormat: FileUploadFormat
88+
get() = FileUploadFormat.JSONL
8989

9090
companion object {
9191
protected val LOGGER: Logger = LoggerFactory.getLogger(GcsJsonlWriter::class.java)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.kt

+6-6
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
1010
import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig
1111
import io.airbyte.cdk.integrations.destination.gcs.util.GcsS3FileSystem
1212
import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter
13-
import io.airbyte.cdk.integrations.destination.s3.S3Format
13+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1414
import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory
15-
import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig
15+
import io.airbyte.cdk.integrations.destination.s3.parquet.UploadParquetFormatConfig
1616
import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter
1717
import io.airbyte.protocol.models.v0.AirbyteRecordMessage
1818
import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream
@@ -46,7 +46,7 @@ class GcsParquetWriter(
4646

4747
init {
4848
val outputFilename: String =
49-
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, S3Format.PARQUET)
49+
BaseGcsWriter.Companion.getOutputFilename(uploadTimestamp, FileUploadFormat.PARQUET)
5050
outputPath = java.lang.String.join("/", outputPrefix, outputFilename)
5151
LOGGER.info(
5252
"Storage path for stream '{}': {}/{}",
@@ -62,7 +62,7 @@ class GcsParquetWriter(
6262

6363
LOGGER.info("Full GCS path for stream '{}': {}", stream.name, path)
6464

65-
val formatConfig = config.formatConfig as S3ParquetFormatConfig
65+
val formatConfig = config.formatConfig as UploadParquetFormatConfig
6666
val hadoopConfig = getHadoopConfig(config)
6767
this.parquetWriter =
6868
AvroParquetWriter.builder<GenericData.Record>(
@@ -102,8 +102,8 @@ class GcsParquetWriter(
102102
}
103103
}
104104

105-
override val fileFormat: S3Format
106-
get() = S3Format.PARQUET
105+
override val fileFormat: FileUploadFormat
106+
get() = FileUploadFormat.PARQUET
107107

108108
companion object {
109109
private val LOGGER: Logger = LoggerFactory.getLogger(GcsParquetWriter::class.java)

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import com.amazonaws.services.s3.AmazonS3
77
import com.amazonaws.services.s3.model.DeleteObjectsRequest
88
import com.amazonaws.services.s3.model.HeadBucketRequest
99
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
10+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1011
import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants
11-
import io.airbyte.cdk.integrations.destination.s3.S3Format
1212
import io.airbyte.cdk.integrations.destination.s3.util.S3OutputPathHelper.getOutputPrefix
1313
import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter
1414
import io.airbyte.protocol.models.v0.AirbyteStream
@@ -128,7 +128,7 @@ protected constructor(
128128
private val LOGGER: Logger = LoggerFactory.getLogger(BaseGcsWriter::class.java)
129129

130130
// Filename: <upload-date>_<upload-millis>_0.<format-extension>
131-
fun getOutputFilename(timestamp: Timestamp, format: S3Format): String {
131+
fun getOutputFilename(timestamp: Timestamp, format: FileUploadFormat): String {
132132
val formatter: DateFormat =
133133
SimpleDateFormat(S3DestinationConstants.YYYY_MM_DD_FORMAT_STRING)
134134
formatter.timeZone = TimeZone.getTimeZone("UTC")

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
package io.airbyte.cdk.integrations.destination.gcs
55

66
import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig
7-
import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig
7+
import io.airbyte.cdk.integrations.destination.s3.avro.UploadAvroFormatConfig
88
import io.airbyte.commons.json.Jsons
99
import io.airbyte.commons.resources.MoreResources
1010
import java.io.IOException
@@ -30,9 +30,9 @@ internal class GcsDestinationConfigTest {
3030
Assertions.assertEquals("test_secret", hmacKeyConfig.hmacKeySecret)
3131

3232
val formatConfig = config.formatConfig
33-
Assertions.assertTrue(formatConfig is S3AvroFormatConfig)
33+
Assertions.assertTrue(formatConfig is UploadAvroFormatConfig)
3434

35-
val avroFormatConfig = formatConfig as S3AvroFormatConfig
35+
val avroFormatConfig = formatConfig as UploadAvroFormatConfig
3636
Assertions.assertEquals("deflate-5", avroFormatConfig.codecFactory.toString())
3737
}
3838
}

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import com.amazonaws.services.s3.internal.Constants
77
import com.google.common.collect.Lists
88
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
99
import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils
10-
import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig.Companion.parseCodecConfig
10+
import io.airbyte.cdk.integrations.destination.s3.avro.UploadAvroFormatConfig.Companion.parseCodecConfig
1111
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory
1212
import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.create
1313
import io.airbyte.commons.json.Jsons

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import com.google.common.collect.Lists
99
import io.airbyte.cdk.integrations.base.DestinationConfig.Companion.initialize
1010
import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig
1111
import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig
12-
import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig
12+
import io.airbyte.cdk.integrations.destination.s3.avro.UploadAvroFormatConfig
1313
import io.airbyte.commons.json.Jsons
1414
import io.airbyte.protocol.models.v0.AirbyteStream
1515
import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream
@@ -34,7 +34,7 @@ internal class GcsAvroWriterTest {
3434
"fake-bucketPath",
3535
"fake-bucketRegion",
3636
GcsHmacKeyCredentialConfig("fake-access-id", "fake-secret"),
37-
S3AvroFormatConfig(ObjectMapper().createObjectNode())
37+
UploadAvroFormatConfig(ObjectMapper().createObjectNode())
3838
),
3939
Mockito.mock(AmazonS3::class.java, Mockito.RETURNS_DEEP_STUBS),
4040
ConfiguredAirbyteStream()

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
package io.airbyte.cdk.integrations.destination.gcs
55

66
import com.fasterxml.jackson.databind.JsonNode
7-
import io.airbyte.cdk.integrations.destination.s3.S3Format
7+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
88
import io.airbyte.cdk.integrations.destination.s3.avro.JsonSchemaType
99
import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion
1010
import io.airbyte.cdk.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider
@@ -25,8 +25,8 @@ import org.junit.jupiter.api.Assertions
2525
import org.junit.jupiter.params.ParameterizedTest
2626
import org.junit.jupiter.params.provider.ArgumentsSource
2727

28-
abstract class GcsAvroParquetDestinationAcceptanceTest(s3Format: S3Format) :
29-
GcsDestinationAcceptanceTest(s3Format) {
28+
abstract class GcsAvroParquetDestinationAcceptanceTest(fileUploadFormat: FileUploadFormat) :
29+
GcsDestinationAcceptanceTest(fileUploadFormat) {
3030
override fun getProtocolVersion() = ProtocolVersion.V1
3131

3232
@ParameterizedTest

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ package io.airbyte.cdk.integrations.destination.gcs
55

66
import com.fasterxml.jackson.databind.JsonNode
77
import com.fasterxml.jackson.databind.ObjectReader
8-
import io.airbyte.cdk.integrations.destination.s3.S3Format
8+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
99
import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants
1010
import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper.getFieldNameUpdater
1111
import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper.pruneAirbyteJson
@@ -20,7 +20,7 @@ import org.apache.avro.generic.GenericData
2020
import org.apache.avro.generic.GenericDatumReader
2121

2222
abstract class GcsBaseAvroDestinationAcceptanceTest :
23-
GcsAvroParquetDestinationAcceptanceTest(S3Format.AVRO) {
23+
GcsAvroParquetDestinationAcceptanceTest(FileUploadFormat.AVRO) {
2424
override val formatConfig: JsonNode?
2525
get() =
2626
Jsons.deserialize(

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import com.amazonaws.services.s3.model.S3Object
77
import com.fasterxml.jackson.databind.JsonNode
88
import com.fasterxml.jackson.databind.node.ObjectNode
99
import io.airbyte.cdk.integrations.base.JavaBaseConstants
10-
import io.airbyte.cdk.integrations.destination.s3.S3Format
10+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1111
import io.airbyte.cdk.integrations.destination.s3.util.Flattening
1212
import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion
1313
import io.airbyte.commons.json.Jsons
@@ -21,7 +21,8 @@ import org.apache.commons.csv.CSVFormat
2121
import org.apache.commons.csv.CSVRecord
2222
import org.apache.commons.csv.QuoteMode
2323

24-
abstract class GcsBaseCsvDestinationAcceptanceTest : GcsDestinationAcceptanceTest(S3Format.CSV) {
24+
abstract class GcsBaseCsvDestinationAcceptanceTest :
25+
GcsDestinationAcceptanceTest(FileUploadFormat.CSV) {
2526
override fun getProtocolVersion() = ProtocolVersion.V1
2627

2728
override val formatConfig: JsonNode?

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ package io.airbyte.cdk.integrations.destination.gcs
66
import com.amazonaws.services.s3.model.S3Object
77
import com.fasterxml.jackson.databind.JsonNode
88
import io.airbyte.cdk.integrations.base.JavaBaseConstants
9-
import io.airbyte.cdk.integrations.destination.s3.S3Format
9+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1010
import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion
1111
import io.airbyte.commons.json.Jsons
1212
import java.io.BufferedReader
@@ -19,7 +19,7 @@ import kotlin.collections.List
1919
import kotlin.collections.MutableList
2020

2121
abstract class GcsBaseJsonlDestinationAcceptanceTest :
22-
GcsDestinationAcceptanceTest(S3Format.JSONL) {
22+
GcsDestinationAcceptanceTest(FileUploadFormat.JSONL) {
2323
override fun getProtocolVersion() = ProtocolVersion.V1
2424

2525
override val formatConfig: JsonNode?

‎airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ package io.airbyte.cdk.integrations.destination.gcs
66
import com.fasterxml.jackson.databind.JsonNode
77
import com.fasterxml.jackson.databind.ObjectReader
88
import io.airbyte.cdk.integrations.destination.gcs.parquet.GcsParquetWriter
9-
import io.airbyte.cdk.integrations.destination.s3.S3Format
9+
import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat
1010
import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants
1111
import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter.Companion.getHadoopConfig
1212
import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper.getFieldNameUpdater
@@ -25,7 +25,7 @@ import org.apache.parquet.avro.AvroReadSupport
2525
import org.apache.parquet.hadoop.ParquetReader
2626

2727
abstract class GcsBaseParquetDestinationAcceptanceTest :
28-
GcsAvroParquetDestinationAcceptanceTest(S3Format.PARQUET) {
28+
GcsAvroParquetDestinationAcceptanceTest(FileUploadFormat.PARQUET) {
2929
override fun getProtocolVersion() = ProtocolVersion.V1
3030

3131
override val formatConfig: JsonNode?

0 commit comments

Comments
 (0)