|
| 1 | +/* |
| 2 | + * MIT License |
| 3 | + * |
| 4 | + * Copyright (c) 2020 Airbyte |
| 5 | + * |
| 6 | + * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | + * of this software and associated documentation files (the "Software"), to deal |
| 8 | + * in the Software without restriction, including without limitation the rights |
| 9 | + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | + * copies of the Software, and to permit persons to whom the Software is |
| 11 | + * furnished to do so, subject to the following conditions: |
| 12 | + * |
| 13 | + * The above copyright notice and this permission notice shall be included in all |
| 14 | + * copies or substantial portions of the Software. |
| 15 | + * |
| 16 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | + * SOFTWARE. |
| 23 | + */ |
| 24 | + |
| 25 | +package io.airbyte.integrations.source.bigquery; |
| 26 | + |
| 27 | +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_CREDS; |
| 28 | +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_DATASET_ID; |
| 29 | +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_PROJECT_ID; |
| 30 | +import static org.junit.jupiter.api.Assertions.assertEquals; |
| 31 | +import static org.junit.jupiter.api.Assertions.assertNotNull; |
| 32 | + |
| 33 | +import com.fasterxml.jackson.databind.JsonNode; |
| 34 | +import com.google.cloud.bigquery.Dataset; |
| 35 | +import com.google.cloud.bigquery.DatasetInfo; |
| 36 | +import com.google.common.collect.ImmutableMap; |
| 37 | +import io.airbyte.commons.json.Jsons; |
| 38 | +import io.airbyte.commons.string.Strings; |
| 39 | +import io.airbyte.commons.util.MoreIterators; |
| 40 | +import io.airbyte.db.bigquery.BigQueryDatabase; |
| 41 | +import io.airbyte.protocol.models.AirbyteMessage; |
| 42 | +import io.airbyte.protocol.models.CatalogHelpers; |
| 43 | +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; |
| 44 | +import io.airbyte.protocol.models.Field; |
| 45 | +import io.airbyte.protocol.models.JsonSchemaPrimitive; |
| 46 | +import java.io.IOException; |
| 47 | +import java.nio.file.Files; |
| 48 | +import java.nio.file.Path; |
| 49 | +import java.sql.SQLException; |
| 50 | +import java.util.List; |
| 51 | +import org.junit.jupiter.api.AfterEach; |
| 52 | +import org.junit.jupiter.api.BeforeEach; |
| 53 | +import org.junit.jupiter.api.Test; |
| 54 | + |
| 55 | +class BigQuerySourceTest { |
| 56 | + |
| 57 | + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); |
| 58 | + private static final String STREAM_NAME = "id_and_name"; |
| 59 | + |
| 60 | + private BigQueryDatabase database; |
| 61 | + private Dataset dataset; |
| 62 | + private JsonNode config; |
| 63 | + |
| 64 | + @BeforeEach |
| 65 | + void setUp() throws IOException, SQLException { |
| 66 | + if (!Files.exists(CREDENTIALS_PATH)) { |
| 67 | + throw new IllegalStateException( |
| 68 | + "Must provide path to a big query credentials file. By default {module-root}/" + CREDENTIALS_PATH |
| 69 | + + ". Override by setting setting path with the CREDENTIALS_PATH constant."); |
| 70 | + } |
| 71 | + |
| 72 | + final String credentialsJsonString = new String(Files.readAllBytes(CREDENTIALS_PATH)); |
| 73 | + |
| 74 | + final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); |
| 75 | + final String projectId = credentialsJson.get(CONFIG_PROJECT_ID).asText(); |
| 76 | + final String datasetLocation = "US"; |
| 77 | + |
| 78 | + final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8); |
| 79 | + |
| 80 | + config = Jsons.jsonNode(ImmutableMap.builder() |
| 81 | + .put(CONFIG_PROJECT_ID, projectId) |
| 82 | + .put(CONFIG_CREDS, credentialsJsonString) |
| 83 | + .put(CONFIG_DATASET_ID, datasetId) |
| 84 | + .build()); |
| 85 | + |
| 86 | + database = new BigQueryDatabase(config.get(CONFIG_PROJECT_ID).asText(), credentialsJsonString); |
| 87 | + |
| 88 | + final DatasetInfo datasetInfo = |
| 89 | + DatasetInfo.newBuilder(config.get(CONFIG_DATASET_ID).asText()).setLocation(datasetLocation).build(); |
| 90 | + dataset = database.getBigQuery().create(datasetInfo); |
| 91 | + |
| 92 | + database.execute( |
| 93 | + "CREATE TABLE " + datasetId |
| 94 | + + ".id_and_name(id INT64, array_val ARRAY<STRUCT<key string, value STRUCT<string_val string>>>, object_val STRUCT<val_array ARRAY<STRUCT<value_str1 string>>, value_str2 string>);"); |
| 95 | + database.execute( |
| 96 | + "INSERT INTO " + datasetId |
| 97 | + + ".id_and_name (id, array_val, object_val) VALUES " |
| 98 | + + "(1, [STRUCT('test1_1', STRUCT('struct1_1')), STRUCT('test1_2', STRUCT('struct1_2'))], STRUCT([STRUCT('value1_1'), STRUCT('value1_2')], 'test1_1')), " |
| 99 | + + "(2, [STRUCT('test2_1', STRUCT('struct2_1')), STRUCT('test2_2', STRUCT('struct2_2'))], STRUCT([STRUCT('value2_1'), STRUCT('value2_2')], 'test2_1')), " |
| 100 | + + "(3, [STRUCT('test3_1', STRUCT('struct3_1')), STRUCT('test3_2', STRUCT('struct3_2'))], STRUCT([STRUCT('value3_1'), STRUCT('value3_2')], 'test3_1'));"); |
| 101 | + } |
| 102 | + |
| 103 | + @AfterEach |
| 104 | + void tearDown() { |
| 105 | + database.cleanDataSet(dataset.getDatasetId().getDataset()); |
| 106 | + } |
| 107 | + |
| 108 | + @Test |
| 109 | + public void testReadSuccess() throws Exception { |
| 110 | + final List<AirbyteMessage> actualMessages = MoreIterators.toList(new BigQuerySource().read(config, getConfiguredCatalog(), null)); |
| 111 | + |
| 112 | + assertNotNull(actualMessages); |
| 113 | + assertEquals(3, actualMessages.size()); |
| 114 | + } |
| 115 | + |
| 116 | + private ConfiguredAirbyteCatalog getConfiguredCatalog() { |
| 117 | + return CatalogHelpers.createConfiguredAirbyteCatalog( |
| 118 | + STREAM_NAME, |
| 119 | + config.get(CONFIG_DATASET_ID).asText(), |
| 120 | + Field.of("id", JsonSchemaPrimitive.NUMBER), |
| 121 | + Field.of("array_val", JsonSchemaPrimitive.ARRAY), |
| 122 | + Field.of("object_val", JsonSchemaPrimitive.OBJECT)); |
| 123 | + } |
| 124 | + |
| 125 | +} |
0 commit comments