diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 232c04aaa85b4..78225c251703a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -122,6 +122,9 @@ jobs: - name: Run End-to-End Acceptance Tests run: ./tools/bin/acceptance_test.sh + - name: Automatic Migration Acceptance Test + run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew --no-daemon :airbyte-tests:automaticMigrationAcceptanceTest --rerun-tasks --scan -i + - name: Push Core Docker Images if: success() && github.ref == 'refs/heads/master' run: | diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java index c65f501a54fa7..e9b09c53f41e5 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java @@ -136,4 +136,17 @@ public String toString() { '}'; } + public static AirbyteVersion versionWithoutPatch(AirbyteVersion airbyteVersion) { + String versionWithoutPatch = "" + airbyteVersion.getMajorVersion() + + "." + + airbyteVersion.getMinorVersion() + + ".0-" + + airbyteVersion.getVersion().replace("\n", "").strip().split("-")[1]; + return new AirbyteVersion(versionWithoutPatch); + } + + public static AirbyteVersion versionWithoutPatch(String airbyteVersion) { + return versionWithoutPatch(new AirbyteVersion(airbyteVersion)); + } + } diff --git a/airbyte-config/init/Dockerfile b/airbyte-config/init/Dockerfile index 9bae75f6768cc..afba972dedf19 100644 --- a/airbyte-config/init/Dockerfile +++ b/airbyte-config/init/Dockerfile @@ -5,4 +5,3 @@ WORKDIR /app # the sole purpose of this image is to seed the data volume with the default data # that the app should have when it is first installed. COPY scripts scripts -COPY src/main/resources/config seed/config diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 92f170162b201..945051223ea9f 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -7,42 +7,3 @@ dependencies { implementation project(':airbyte-config:models') } - -// generate seed for each yaml file. -task generateSeed { - def seeds = [ - [ - "sourceDefinitionId", - new File(project.projectDir, '/src/main/resources/seed/source_definitions.yaml'), - new File(project.projectDir, '/src/main/resources/config/STANDARD_SOURCE_DEFINITION') - ], - [ - "destinationDefinitionId", - new File(project.projectDir, '/src/main/resources/seed/destination_definitions.yaml'), - new File(project.projectDir, '/src/main/resources/config/STANDARD_DESTINATION_DEFINITION') - ], - ] - seeds.each{val -> - def name = val[0] - def taskName = "generateSeed$name" - dependsOn taskName - task "$taskName"(type: JavaExec) { - classpath = sourceSets.main.runtimeClasspath - - main = 'io.airbyte.config.init.SeedRepository' - - // arguments to pass to the application - args '--id-name' - args val[0] - args '--input-path' - args val[1] - args '--output-path' - args val[2] - } - } -} - -// we only want to attempt generateSeed if tests have passed. -generateSeed.dependsOn(check) -generateSeed.dependsOn(assemble) -build.dependsOn(generateSeed) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java index aeb1de0023742..f555d70bed8a1 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java @@ -27,45 +27,102 @@ import io.airbyte.commons.json.JsonSchemas; import java.io.File; import java.nio.file.Path; +import java.util.function.Function; public enum ConfigSchema { // workspace - STANDARD_WORKSPACE("StandardWorkspace.yaml"), + STANDARD_WORKSPACE("StandardWorkspace.yaml", StandardWorkspace.class, standardWorkspace -> { + return standardWorkspace.getWorkspaceId().toString(); + }), // source - STANDARD_SOURCE_DEFINITION("StandardSourceDefinition.yaml"), - SOURCE_CONNECTION("SourceConnection.yaml"), + STANDARD_SOURCE_DEFINITION("StandardSourceDefinition.yaml", StandardSourceDefinition.class, + standardSourceDefinition -> { + return standardSourceDefinition.getSourceDefinitionId().toString(); + }), + SOURCE_CONNECTION("SourceConnection.yaml", SourceConnection.class, + sourceConnection -> { + return sourceConnection.getSourceId().toString(); + }), // destination - STANDARD_DESTINATION_DEFINITION("StandardDestinationDefinition.yaml"), - DESTINATION_CONNECTION("DestinationConnection.yaml"), + STANDARD_DESTINATION_DEFINITION("StandardDestinationDefinition.yaml", + StandardDestinationDefinition.class, standardDestinationDefinition -> { + return standardDestinationDefinition.getDestinationDefinitionId().toString(); + }), + DESTINATION_CONNECTION("DestinationConnection.yaml", DestinationConnection.class, + destinationConnection -> { + return destinationConnection.getDestinationId().toString(); + }), // sync - STANDARD_SYNC("StandardSync.yaml"), - STANDARD_SYNC_OPERATION("StandardSyncOperation.yaml"), - STANDARD_SYNC_SUMMARY("StandardSyncSummary.yaml"), + STANDARD_SYNC("StandardSync.yaml", StandardSync.class, standardSync -> { + return standardSync.getConnectionId().toString(); + }), + STANDARD_SYNC_OPERATION("StandardSyncOperation.yaml", StandardSyncOperation.class, + standardSyncOperation -> { + return standardSyncOperation.getOperationId().toString(); + }), + STANDARD_SYNC_SUMMARY("StandardSyncSummary.yaml", StandardSyncSummary.class, + standardSyncSummary -> { + throw new RuntimeException("StandardSyncSummary doesn't have an id"); + }), // worker - STANDARD_SYNC_INPUT("StandardSyncInput.yaml"), - NORMALIZATION_INPUT("NormalizationInput.yaml"), - OPERATOR_DBT_INPUT("OperatorDbtInput.yaml"), + STANDARD_SYNC_INPUT("StandardSyncInput.yaml", StandardSyncInput.class, + standardSyncInput -> { + throw new RuntimeException("StandardSyncInput doesn't have an id"); + }), + NORMALIZATION_INPUT("NormalizationInput.yaml", NormalizationInput.class, + normalizationInput -> { + throw new RuntimeException("NormalizationInput doesn't have an id"); + }), + OPERATOR_DBT_INPUT("OperatorDbtInput.yaml", OperatorDbtInput.class, + operatorDbtInput -> { + throw new RuntimeException("OperatorDbtInput doesn't have an id"); + }), - STANDARD_SYNC_OUTPUT("StandardSyncOutput.yaml"), - REPLICATION_OUTPUT("ReplicationOutput.yaml"), + STANDARD_SYNC_OUTPUT("StandardSyncOutput.yaml", StandardSyncOutput.class, + standardWorkspace -> { + throw new RuntimeException("StandardSyncOutput doesn't have an id"); + }), + REPLICATION_OUTPUT("ReplicationOutput.yaml", ReplicationOutput.class, + standardWorkspace -> { + throw new RuntimeException("ReplicationOutput doesn't have an id"); + }), - STATE("State.yaml"); + STATE("State.yaml", State.class, standardWorkspace -> { + throw new RuntimeException("State doesn't have an id"); + }); static final Path KNOWN_SCHEMAS_ROOT = JsonSchemas.prepareSchemas("types", ConfigSchema.class); private final String schemaFilename; + private final Class className; + private final Function extractId; - ConfigSchema(final String schemaFilename) { + ConfigSchema(final String schemaFilename, + Class className, + Function extractId) { this.schemaFilename = schemaFilename; + this.className = className; + this.extractId = extractId; } public File getFile() { return KNOWN_SCHEMAS_ROOT.resolve(schemaFilename).toFile(); } + public Class getClassName() { + return (Class) className; + } + + public String getId(T object) { + if (getClassName().isInstance(object)) { + return ((Function) extractId).apply(object); + } + throw new RuntimeException("Object: " + object + " is not instance of class " + getClassName().getName()); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java index 8ad20e5760042..0dcb39632712b 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java @@ -24,10 +24,13 @@ package io.airbyte.config.persistence; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.config.ConfigSchema; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.stream.Stream; public interface ConfigPersistence { @@ -37,4 +40,8 @@ public interface ConfigPersistence { void writeConfig(ConfigSchema configType, String configId, T config) throws JsonValidationException, IOException; + void replaceAllConfigs(Map> configs, boolean dryRun) throws IOException; + + Map> dumpConfigs() throws IOException; + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index f50cf5636327e..f355eb19bc47c 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -24,6 +24,7 @@ package io.airbyte.config.persistence; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.lang.MoreBooleans; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; @@ -37,7 +38,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Stream; public class ConfigRepository { @@ -203,4 +206,12 @@ public List listStandardSyncOperations() throws IOExcepti return persistence.listConfigs(ConfigSchema.STANDARD_SYNC_OPERATION, StandardSyncOperation.class); } + public void replaceAllConfigs(Map> configs, boolean dryRun) throws IOException { + persistence.replaceAllConfigs(configs, dryRun); + } + + public Map> dumpConfigs() throws IOException { + return persistence.dumpConfigs(); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DefaultConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DefaultConfigPersistence.java index 6841097e73322..40338daf56005 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DefaultConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DefaultConfigPersistence.java @@ -34,18 +34,27 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; // we force all interaction with disk storage to be effectively single threaded. public class DefaultConfigPersistence implements ConfigPersistence { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultConfigPersistence.class); private static final String CONFIG_DIR = "config"; private static final Object lock = new Object(); private final JsonSchemaValidator jsonSchemaValidator; + private final Path storageRootWithConfigDirectory; private final Path storageRoot; public DefaultConfigPersistence(final Path storageRoot) { @@ -53,8 +62,9 @@ public DefaultConfigPersistence(final Path storageRoot) { } public DefaultConfigPersistence(final Path storageRoot, final JsonSchemaValidator schemaValidator) { - this.storageRoot = storageRoot.resolve(CONFIG_DIR); - jsonSchemaValidator = schemaValidator; + this.storageRoot = storageRoot; + this.storageRootWithConfigDirectory = storageRoot.resolve(CONFIG_DIR); + this.jsonSchemaValidator = schemaValidator; } @Override @@ -75,26 +85,127 @@ public List listConfigs(ConfigSchema configType, Class clazz) throws J @Override public void writeConfig(ConfigSchema configType, String configId, T config) throws JsonValidationException, IOException { synchronized (lock) { - writeConfigInternal(configType, configId, config); + writeConfigInternal(configType, configId, config, storageRootWithConfigDirectory); } } - private T getConfigInternal(ConfigSchema configType, String configId, Class clazz) - throws ConfigNotFoundException, JsonValidationException, IOException { + private void writeConfig(ConfigSchema configType, String configId, T config, Path rootOverride) throws JsonValidationException, IOException { + writeConfigInternal(configType, configId, config, rootOverride); + } + + private void writeConfigs(ConfigSchema configType, Stream configs, Path rootOverride) { + configs.forEach(config -> { + String id = configType.getId(config); + try { + writeConfig(configType, id, config, rootOverride); + } catch (JsonValidationException | IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public Map> dumpConfigs() throws IOException { + final Map> configs = new HashMap<>(); + + final List directories = listDirectories(); + for (String directory : directories) { + final List configList = listConfig(directory); + configs.put(directory, configList.stream()); + } + return configs; + } + + private List listDirectories() throws IOException { + try (Stream files = Files.list(storageRootWithConfigDirectory)) { + return files.map(c -> c.getFileName().toString()).collect(Collectors.toList()); + } + } + + private List listConfig(String configType) throws IOException { + final Path configTypePath = storageRootWithConfigDirectory.resolve(configType); + if (!Files.exists(configTypePath)) { + return Collections.emptyList(); + } + try (Stream files = Files.list(configTypePath)) { + final List ids = files + .filter(p -> !p.endsWith(".json")) + .map(p -> p.getFileName().toString().replace(".json", "")) + .collect(Collectors.toList()); + + final List configs = Lists.newArrayList(); + for (String id : ids) { + try { + final Path configPath = storageRootWithConfigDirectory.resolve(configType).resolve(String.format("%s.json", id)); + if (!Files.exists(configPath)) { + throw new RuntimeException("Config NotFound"); + } + + final JsonNode config = Jsons.deserialize(Files.readString(configPath), JsonNode.class); + configs.add(config); + } catch (RuntimeException e) { + throw new IOException(e); + } + } + + return configs; + } + } + + @Override + public void replaceAllConfigs(Map> configs, boolean dryRun) + throws IOException { + // create a new folder + String importDirectory = CONFIG_DIR + UUID.randomUUID().toString(); + Path rootOverride = storageRoot.resolve(importDirectory); + Files.createDirectories(rootOverride); + + // write everything + for (Map.Entry> config : configs.entrySet()) { + writeConfigs(config.getKey(), config.getValue(), rootOverride); + } + + if (dryRun) { + FileUtils.deleteDirectory(rootOverride.toFile()); + return; + } + + boolean configToDeprecated = storageRootWithConfigDirectory.toFile().renameTo(storageRoot.resolve("config_deprecated").toFile()); + if (configToDeprecated) { + LOGGER.info("Renamed config to config_deprecated successfully"); + } + boolean newConfig = rootOverride.toFile().renameTo(storageRootWithConfigDirectory.toFile()); + if (newConfig) { + LOGGER.info("Renamed " + importDirectory + " to config successfully"); + } + + LOGGER.info("Deleting config_deprecated"); + FileUtils.deleteDirectory(storageRoot.resolve("config_deprecated").toFile()); + + } + + private Optional getConfigInternalWithoutValidation(ConfigSchema configType, String configId, Class clazz) throws IOException { // validate file with schema - final Path configPath = buildConfigPath(configType, configId); + final Path configPath = buildConfigPath(configType, configId, storageRootWithConfigDirectory); if (!Files.exists(configPath)) { - throw new ConfigNotFoundException(configType, configId); + return Optional.empty(); + } else { + return Optional.of(Jsons.deserialize(Files.readString(configPath), clazz)); } + } + + private T getConfigInternal(ConfigSchema configType, String configId, Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + final T config = getConfigInternalWithoutValidation(configType, configId, clazz) + .orElseThrow(() -> new ConfigNotFoundException(configType, configId)); - final T config = Jsons.deserialize(Files.readString(configPath), clazz); validateJson(config, configType); return config; } private List listConfigsInternal(ConfigSchema configType, Class clazz) throws JsonValidationException, IOException { - final Path configTypePath = buildTypePath(configType); + final Path configTypePath = buildTypePath(configType, storageRootWithConfigDirectory); if (!Files.exists(configTypePath)) { return Collections.emptyList(); } @@ -119,22 +230,23 @@ private List listConfigsInternal(ConfigSchema configType, Class clazz) } } - private void writeConfigInternal(ConfigSchema configType, String configId, T config) throws JsonValidationException, IOException { + private void writeConfigInternal(ConfigSchema configType, String configId, T config, Path storageRootWithConfigDirectory) + throws JsonValidationException, IOException { // validate config with schema validateJson(Jsons.jsonNode(config), configType); - final Path configPath = buildConfigPath(configType, configId); + final Path configPath = buildConfigPath(configType, configId, storageRootWithConfigDirectory); Files.createDirectories(configPath.getParent()); Files.writeString(configPath, Jsons.serialize(config)); } - private Path buildConfigPath(ConfigSchema type, String configId) { - return buildTypePath(type).resolve(String.format("%s.json", configId)); + private static Path buildConfigPath(ConfigSchema type, String configId, Path storageRootWithConfigDirectory) { + return buildTypePath(type, storageRootWithConfigDirectory).resolve(String.format("%s.json", configId)); } - private Path buildTypePath(ConfigSchema type) { - return storageRoot.resolve(type.toString()); + private static Path buildTypePath(ConfigSchema type, Path storageRootWithConfigDirectory) { + return storageRootWithConfigDirectory.resolve(type.toString()); } private void validateJson(T config, ConfigSchema configType) throws JsonValidationException { diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index 62b82c8f96a7f..2ad906d868ec4 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -24,7 +24,7 @@ ${additionalMessage || ""} module.exports = function (plop) { const docRoot = '../../../docs/integrations'; - const definitionRoot = '../../../airbyte-config/init/src/main/resources'; + const definitionRoot = '../../../airbyte-server/src/main/resources'; const pythonSourceInputRoot = '../source-python'; const singerSourceInputRoot = '../source-singer'; diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java b/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java index 5fd59b9cca67b..c1f298ade2a8d 100644 --- a/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java +++ b/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java @@ -25,6 +25,7 @@ package io.airbyte.migrate; import io.airbyte.commons.io.Archives; +import io.airbyte.commons.version.AirbyteVersion; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -41,10 +42,14 @@ public class MigrationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(MigrationRunner.class); public static void run(String[] args) throws IOException { + MigrateConfig migrateConfig = parse(args); + run(migrateConfig); + } + public static void run(MigrateConfig migrateConfig) throws IOException { final Path workspaceRoot = Files.createTempDirectory(Path.of("/tmp"), "airbyte_migrate"); - - MigrateConfig migrateConfig = parse(args); + migrateConfig = new MigrateConfig(migrateConfig.getInputPath(), migrateConfig.getOutputPath(), + AirbyteVersion.versionWithoutPatch(migrateConfig.getTargetVersion()).getVersion()); if (migrateConfig.getInputPath().toString().endsWith(".gz")) { LOGGER.info("Unpacking tarball"); diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_14_3/airbyte_config/StandardSync.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_14_3/airbyte_config/StandardSync.yaml index f9b6d548ad929..e58b9b9e8fba5 100644 --- a/airbyte-migration/src/main/resources/migrations/migrationV0_14_3/airbyte_config/StandardSync.yaml +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_14_3/airbyte_config/StandardSync.yaml @@ -11,6 +11,9 @@ required: - catalog additionalProperties: false properties: + prefix: + description: Prefix that will be prepended to the name of each stream when it is written to the destination. + type: string sourceId: type: string format: uuid diff --git a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java index 355e66c3122df..1be59aebcf6ee 100644 --- a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java +++ b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java @@ -239,7 +239,8 @@ public static void main(String[] args) throws IOException, InterruptedException Optional airbyteDatabaseVersion = jobPersistence.getVersion(); int loopCount = 0; - while (airbyteDatabaseVersion.isEmpty() && loopCount < 300) { + while ((airbyteDatabaseVersion.isEmpty() || !AirbyteVersion.isCompatible(configs.getAirbyteVersion(), airbyteDatabaseVersion.get())) + && loopCount < 300) { LOGGER.warn("Waiting for Server to start..."); TimeUnit.SECONDS.sleep(1); airbyteDatabaseVersion = jobPersistence.getVersion(); diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java index 8c1085a6011ce..3ae66efefc823 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java @@ -83,6 +83,10 @@ public class DefaultJobPersistence implements JobPersistence { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultJobPersistence.class); + private static final Set SYSTEM_SCHEMA = Set + .of("pg_toast", "information_schema", "pg_catalog", "import_backup", "pg_internal", + "catalog_history"); + private static final JSONFormat DB_JSON_FORMAT = new JSONFormat().recordFormat(RecordFormat.OBJECT); protected static final String DEFAULT_SCHEMA = "public"; private static final String BACKUP_SCHEMA = "import_backup"; @@ -436,10 +440,10 @@ public Optional getVersion() throws IOException { @Override public void setVersion(String airbyteVersion) throws IOException { database.query(ctx -> ctx.execute(String.format( - "INSERT INTO %s VALUES('%s', '%s'), ('%s_init_db', '%s');", + "INSERT INTO %s VALUES('%s', '%s'), ('%s_init_db', '%s') ON CONFLICT (key) DO UPDATE SET value = '%s'", AIRBYTE_METADATA_TABLE, AirbyteVersion.AIRBYTE_VERSION_KEY_NAME, airbyteVersion, - current_timestamp(), airbyteVersion))); + current_timestamp(), airbyteVersion, airbyteVersion))); } private static String current_timestamp() { @@ -451,6 +455,27 @@ public Map> exportDatabase() throws IOException return exportDatabase(DEFAULT_SCHEMA); } + /** + * This is different from {@link #exportDatabase()} cause it exports all the tables in all the + * schemas available + */ + @Override + public Map> dump() throws IOException { + final Map> result = new HashMap<>(); + for (String schema : listSchemas()) { + final List tables = listAllTables(schema); + + for (final String table : tables) { + if (result.containsKey(table)) { + throw new RuntimeException("Multiple tables found with the same name " + table); + } + result.put(table.toUpperCase(), exportTable(schema, table)); + } + } + + return result; + } + private Map> exportDatabase(final String schema) throws IOException { final List tables = listTables(schema); final Map> result = new HashMap<>(); @@ -477,6 +502,25 @@ private List listTables(final String schema) throws IOException { } } + private List listAllTables(final String schema) throws IOException { + if (schema != null) { + return database.query(context -> context.meta().getSchemas(schema).stream() + .flatMap(s -> context.meta(s).getTables().stream()) + .map(Named::getName) + .collect(Collectors.toList())); + } else { + return List.of(); + } + } + + private List listSchemas() throws IOException { + return database.query(context -> context.meta().getSchemas().stream() + .map(Named::getName) + .filter(c -> !SYSTEM_SCHEMA.contains(c)) + .collect(Collectors.toList())); + + } + private Stream exportTable(final String schema, final String tableName) throws IOException { final Table tableSql = getTable(schema, tableName); try (final Stream records = database.query(ctx -> ctx.select(DSL.asterisk()).from(tableSql).fetchStream())) { diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java index 5ed04b1ed4cbe..068c871e1ab3f 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java @@ -173,6 +173,8 @@ public interface JobPersistence { */ Map> exportDatabase() throws IOException; + Map> dump() throws IOException; + /** * Import all SQL tables from streams of JsonNode objects. * diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 6ea7658ccae10..9a438a7200813 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -13,6 +13,9 @@ RUN chmod +x wait COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar +RUN mkdir latest_seeds +COPY build/resources/main/config latest_seeds + RUN tar xf ${APPLICATION}.tar --strip-components=1 # wait for upstream dependencies to become available before starting server diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 45cc73bfb3b1e..cb96105eb4980 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -7,9 +7,9 @@ dependencies { implementation 'org.apache.cxf:cxf-core:3.4.2' + implementation 'commons-cli:commons-cli:1.4' implementation 'org.eclipse.jetty:jetty-server:9.4.31.v20200723' implementation 'org.eclipse.jetty:jetty-servlet:9.4.31.v20200723' - implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.3' implementation 'org.glassfish.jersey.containers:jersey-container-servlet' implementation 'org.glassfish.jersey.inject:jersey-hk2' @@ -26,6 +26,7 @@ dependencies { implementation project(':airbyte-config:init') implementation project(':airbyte-db') implementation project(":airbyte-json-validation") + implementation project(':airbyte-migration') implementation project(':airbyte-notification') implementation project(':airbyte-protocol:models') implementation project(':airbyte-scheduler:client') @@ -61,3 +62,42 @@ run { environment "TEMPORAL_HOST", "localhost:7233" } + +// generate seed for each yaml file. +task generateSeed { + def seeds = [ + [ + "sourceDefinitionId", + new File(project.projectDir, '/src/main/resources/seed/source_definitions.yaml'), + new File(project.projectDir, '/src/main/resources/config/STANDARD_SOURCE_DEFINITION') + ], + [ + "destinationDefinitionId", + new File(project.projectDir, '/src/main/resources/seed/destination_definitions.yaml'), + new File(project.projectDir, '/src/main/resources/config/STANDARD_DESTINATION_DEFINITION') + ], + ] + seeds.each{val -> + def name = val[0] + def taskName = "generateSeed$name" + dependsOn taskName + task "$taskName"(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + + main = 'io.airbyte.server.SeedRepository' + + // arguments to pass to the application + args '--id-name' + args val[0] + args '--input-path' + args val[1] + args '--output-path' + args val[2] + } + } +} + +// we only want to attempt generateSeed if tests have passed. +generateSeed.dependsOn(check) +generateSeed.dependsOn(assemble) +build.dependsOn(generateSeed) diff --git a/airbyte-server/seed.Dockerfile b/airbyte-server/seed.Dockerfile new file mode 100644 index 0000000000000..625b159189312 --- /dev/null +++ b/airbyte-server/seed.Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3.4 AS seed + +WORKDIR / + +# the sole purpose of this image is to seed the data volume with the default data +# that the app should have when it is first installed. +COPY build/resources/main/config latest_seeds/config diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExport.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExport.java new file mode 100644 index 0000000000000..d222619cc728d --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExport.java @@ -0,0 +1,151 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.io.Archives; +import io.airbyte.commons.lang.CloseableConsumer; +import io.airbyte.commons.lang.Exceptions; +import io.airbyte.commons.yaml.Yamls; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.scheduler.persistence.JobPersistence; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.io.FileUtils; + +// TODO: Write a test case which compares the output dump with the output of ArchiveHandler export +// for the same data + +/** + * This class acts like export method of ArchiveHandler but the difference is 1. It takes a full + * dump of whatever is available in the config directory without any schema validation. We dont want + * schema validation because in case of automatic migration, the code that is going to do the schema + * validation is from new version but the data in the config files is old. Thus schema validation + * would fail. 2. Unlike ArchiveHandler, this doesn't take the dump of specific files but looks at + * the config directory and takes the full dump of whatever is available + */ +public class ConfigDumpExport { + + private static final String ARCHIVE_FILE_NAME = "airbyte_config_dump"; + private static final String CONFIG_FOLDER_NAME = "airbyte_config"; + private static final String DB_FOLDER_NAME = "airbyte_db"; + private static final String VERSION_FILE_NAME = "VERSION"; + private final ConfigRepository configRepository; + private final JobPersistence jobPersistence; + private final String version; + + public ConfigDumpExport(ConfigRepository configRepository, JobPersistence jobPersistence, String version) { + this.configRepository = configRepository; + this.jobPersistence = jobPersistence; + this.version = version; + } + + public File dump() { + try { + final Path tempFolder = Files.createTempDirectory(Path.of("/tmp"), ARCHIVE_FILE_NAME); + final File dump = Files.createTempFile(ARCHIVE_FILE_NAME, ".tar.gz").toFile(); + exportVersionFile(tempFolder); + dumpConfigs(tempFolder); + dumpDatabase(tempFolder); + + Archives.createArchive(tempFolder, dump.toPath()); + return dump; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void exportVersionFile(Path tempFolder) throws IOException { + final File versionFile = Files.createFile(tempFolder.resolve(VERSION_FILE_NAME)).toFile(); + FileUtils.writeStringToFile(versionFile, version, Charset.defaultCharset()); + } + + private void dumpDatabase(Path parentFolder) throws Exception { + final Map> tables = jobPersistence.dump(); + Files.createDirectories(parentFolder.resolve(DB_FOLDER_NAME)); + for (Map.Entry> table : tables.entrySet()) { + final Path tablePath = buildTablePath(parentFolder, table.getKey()); + writeTableToArchive(tablePath, table.getValue()); + } + } + + private void writeTableToArchive(final Path tablePath, final Stream tableStream) + throws Exception { + Files.createDirectories(tablePath.getParent()); + final BufferedWriter recordOutputWriter = new BufferedWriter( + new FileWriter(tablePath.toFile())); + final CloseableConsumer recordConsumer = Yamls.listWriter(recordOutputWriter); + tableStream.forEach(row -> Exceptions.toRuntime(() -> { + recordConsumer.accept(row); + })); + recordConsumer.close(); + } + + protected static Path buildTablePath(final Path storageRoot, final String tableName) { + return storageRoot + .resolve(DB_FOLDER_NAME) + .resolve(String.format("%s.yaml", tableName.toUpperCase())); + } + + private void dumpConfigs(Path parentFolder) throws IOException { + for (Map.Entry> configEntry : configRepository.dumpConfigs().entrySet()) { + writeConfigsToArchive(parentFolder, configEntry.getKey(), configEntry.getValue()); + } + } + + private void writeConfigsToArchive(final Path storageRoot, + final String schemaType, + final Stream configs) + throws IOException { + final Path configPath = buildConfigPath(storageRoot, schemaType); + Files.createDirectories(configPath.getParent()); + final List configList = configs.collect(Collectors.toList()); + if (!configList.isEmpty()) { + final List sortedConfigs = configList.stream() + .sorted(Comparator.comparing(JsonNode::toString)).collect( + Collectors.toList()); + Files.writeString(configPath, Yamls.serialize(sortedConfigs)); + } else { + // Create empty file + Files.createFile(configPath); + } + } + + private static Path buildConfigPath(final Path storageRoot, final String schemaType) { + return storageRoot.resolve(CONFIG_FOLDER_NAME) + .resolve(String.format("%s.yaml", schemaType)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java new file mode 100644 index 0000000000000..19d0f476d64f2 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java @@ -0,0 +1,364 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Streams; +import io.airbyte.analytics.TrackingClientSingleton; +import io.airbyte.api.model.ImportRead; +import io.airbyte.api.model.ImportRead.StatusEnum; +import io.airbyte.commons.io.Archives; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.stream.MoreStreams; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.yaml.Yamls; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.PersistenceConstants; +import io.airbyte.scheduler.persistence.DatabaseSchema; +import io.airbyte.scheduler.persistence.JobPersistence; +import io.airbyte.validation.json.JsonSchemaValidator; +import io.airbyte.validation.json.JsonValidationException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigDumpImport { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigDumpImport.class); + private static final String CONFIG_FOLDER_NAME = "airbyte_config"; + private static final String DB_FOLDER_NAME = "airbyte_db"; + private static final String VERSION_FILE_NAME = "VERSION"; + private final ConfigRepository configRepository; + private final JsonSchemaValidator jsonSchemaValidator; + private final JobPersistence postgresPersistence; + private final String targetVersion; + private final String initialVersion; + private final Path latestSeed; + + public ConfigDumpImport(String initialVersion, + String targetVersion, + Path latestSeed, + JobPersistence postgresPersistence, + ConfigRepository configRepository) { + this.targetVersion = targetVersion; + this.initialVersion = initialVersion; + this.latestSeed = latestSeed; + this.jsonSchemaValidator = new JsonSchemaValidator(); + this.postgresPersistence = postgresPersistence; + this.configRepository = configRepository; + } + + @VisibleForTesting + public Optional getCurrentCustomerId() { + try { + return Optional.of(configRepository + .getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID, true).getCustomerId()); + } catch (Exception e) { + // because this is used for tracking we prefer to log instead of killing the import. + LOGGER.error("failed to fetch current customerId.", e); + return Optional.empty(); + } + } + + public ImportRead importData(File archive) { + + final Optional previousCustomerIdOptional = getCurrentCustomerId(); + ImportRead result; + try { + final Path sourceRoot = Files.createTempDirectory(Path.of("/tmp"), "airbyte_archive"); + try { + // 1. Unzip source + Archives.extractArchive(archive.toPath(), sourceRoot); + + // 2. Set DB version + LOGGER.info("Setting the DB Airbyte version to : " + targetVersion); + postgresPersistence.setVersion(targetVersion); + + // 3. dry run + try { + checkImport(sourceRoot); + } catch (Exception e) { + LOGGER.warn("Dry run failed, setting DB version back to initial version"); + postgresPersistence.setVersion(initialVersion); + throw e; + } + + // 4. Import Postgres content + importDatabaseFromArchive(sourceRoot, targetVersion); + + // 5. Import Configs + importConfigsFromArchive(sourceRoot, false); + result = new ImportRead().status(StatusEnum.SUCCEEDED); + } finally { + FileUtils.deleteDirectory(sourceRoot.toFile()); + FileUtils.deleteQuietly(archive); + } + + // identify this instance as the new customer id. + TrackingClientSingleton.get().identify(); + // report that the previous customer id is now superseded by the imported one. + previousCustomerIdOptional.ifPresent( + previousCustomerId -> TrackingClientSingleton.get().alias(previousCustomerId.toString())); + } catch (IOException | JsonValidationException | RuntimeException e) { + LOGGER.error("Import failed", e); + result = new ImportRead().status(StatusEnum.FAILED).reason(e.getMessage()); + } + + return result; + } + + private void checkImport(Path tempFolder) throws IOException, JsonValidationException { + final Path versionFile = tempFolder.resolve(VERSION_FILE_NAME); + final String importVersion = Files.readString(versionFile, Charset.defaultCharset()) + .replace("\n", "").strip(); + LOGGER.info(String.format("Checking Airbyte Version to import %s", importVersion)); + if (!AirbyteVersion.isCompatible(targetVersion, importVersion)) { + throw new IOException(String + .format("Imported VERSION (%s) is incompatible with current Airbyte version (%s).\n" + + "Please upgrade your Airbyte Archive, see more at https://docs.airbyte.io/tutorials/upgrading-airbyte\n", + importVersion, targetVersion)); + } + checkDBVersion(targetVersion); + importConfigsFromArchive(tempFolder, true); + } + + // Config + private List listDirectories(Path sourceRoot) throws IOException { + try (Stream files = Files.list(sourceRoot.resolve(CONFIG_FOLDER_NAME))) { + return files.map(c -> c.getFileName().toString()) + .collect(Collectors.toList()); + } + } + + private void importConfigsFromArchive(final Path sourceRoot, final boolean dryRun) throws IOException, JsonValidationException { + List sourceDefinitionsToMigrate = new ArrayList<>(); + List destinationDefinitionsToMigrate = new ArrayList<>(); + final boolean[] sourceProcessed = {false}; + final boolean[] destinationProcessed = {false}; + List directories = listDirectories(sourceRoot); + // We sort the directories cause we want to process SOURCE_CONNECTION before + // STANDARD_SOURCE_DEFINITION and DESTINATION_CONNECTION before STANDARD_DESTINATION_DEFINITION + // so that we can identify which definitions should not be upgraded to the latest version + Collections.sort(directories); + Map> data = new LinkedHashMap<>(); + Map> latestSeeds = latestSeeds(); + for (String directory : directories) { + ConfigSchema configSchema = ConfigSchema.valueOf(directory.replace(".yaml", "")); + Stream configs = readConfigsFromArchive(sourceRoot, configSchema); + configs = streamWithAdditionalOperation(sourceDefinitionsToMigrate, destinationDefinitionsToMigrate, sourceProcessed, destinationProcessed, + configSchema, configs, latestSeeds); + data.put(configSchema, configs); + } + configRepository.replaceAllConfigs(data, dryRun); + } + + private Map> latestSeeds() throws IOException { + List configSchemas = Files.list(latestSeed).map(c -> ConfigSchema.valueOf(c.getFileName().toString())).collect(Collectors.toList()); + Map> allData = new HashMap<>(); + for (ConfigSchema configSchema : configSchemas) { + Map data = readLatestSeed(configSchema); + allData.put(configSchema, data); + } + return allData; + } + + private Map readLatestSeed(ConfigSchema configSchema) throws IOException { + try (Stream files = Files.list(latestSeed.resolve(configSchema.toString()))) { + final List ids = files + .filter(p -> !p.endsWith(".json")) + .map(p -> p.getFileName().toString().replace(".json", "")) + .collect(Collectors.toList()); + + final Map configData = new HashMap<>(); + for (String id : ids) { + try { + final Path configPath = latestSeed.resolve(configSchema.toString()).resolve(String.format("%s.json", id)); + if (!Files.exists(configPath)) { + throw new RuntimeException("Config NotFound"); + } + + T config = Jsons.deserialize(Files.readString(configPath), configSchema.getClassName()); + configData.put(id, config); + } catch (RuntimeException e) { + throw new IOException(e); + } + } + + return configData; + } + } + + private Stream streamWithAdditionalOperation(List sourceDefinitionsToMigrate, + List destinationDefinitionsToMigrate, + boolean[] sourceProcessed, + boolean[] destinationProcessed, + ConfigSchema configSchema, + Stream configs, + Map> latestSeeds) { + if (configSchema == ConfigSchema.SOURCE_CONNECTION) { + sourceProcessed[0] = true; + configs = configs.peek(config -> sourceDefinitionsToMigrate.add(((SourceConnection) config).getSourceDefinitionId().toString())); + } else if (configSchema == ConfigSchema.DESTINATION_CONNECTION) { + destinationProcessed[0] = true; + configs = configs.peek(config -> destinationDefinitionsToMigrate.add(((DestinationConnection) config).getDestinationDefinitionId().toString())); + } else if (configSchema == ConfigSchema.STANDARD_SOURCE_DEFINITION) { + configs = getDefinitionStream(sourceDefinitionsToMigrate, sourceProcessed[0], configSchema, configs, latestSeeds); + } else if (configSchema == ConfigSchema.STANDARD_DESTINATION_DEFINITION) { + configs = getDefinitionStream(destinationDefinitionsToMigrate, destinationProcessed[0], configSchema, configs, latestSeeds); + } + return configs; + } + + /** + * This method combines latest definitions, with existing definition. If a connector is being used + * by user, it will continue to be at the same version, otherwise it will be migrated to the latest + * version + */ + private Stream getDefinitionStream(List definitionsToMigrate, + boolean definitionsPopulated, + ConfigSchema configSchema, + Stream configs, + Map> latestSeeds) { + if (!definitionsPopulated) { + throw new RuntimeException("Trying to process " + configSchema + " without populating the definitions to migrate"); + } + + return Streams.concat(configs.filter(c -> definitionsToMigrate.contains(configSchema.getId(c))), + latestSeeds.get(configSchema).entrySet().stream().filter(c -> !definitionsToMigrate.contains(c.getKey())) + .map(Entry::getValue)); + } + + private Stream readConfigsFromArchive(final Path storageRoot, final ConfigSchema schemaType) + throws IOException { + + final Path configPath = buildConfigPath(storageRoot, schemaType); + if (configPath.toFile().exists()) { + final String configStr = Files.readString(configPath); + final JsonNode node = Yamls.deserialize(configStr); + return StreamSupport + .stream(Spliterators.spliteratorUnknownSize(node.elements(), Spliterator.ORDERED), false) + .map(element -> { + final T config = Jsons.object(element, schemaType.getClassName()); + try { + validateJson(config, schemaType); + return config; + } catch (JsonValidationException e) { + throw new RuntimeException(e); + } + }); + + } else { + throw new FileNotFoundException( + String.format("Airbyte Configuration %s was not found in the archive", schemaType)); + } + } + + private void validateJson(final T config, final ConfigSchema configType) + throws JsonValidationException { + JsonNode schema = JsonSchemaValidator.getSchema(configType.getFile()); + jsonSchemaValidator.ensure(schema, Jsons.jsonNode(config)); + } + + protected static Path buildConfigPath(final Path storageRoot, final ConfigSchema schemaType) { + return storageRoot.resolve(CONFIG_FOLDER_NAME) + .resolve(String.format("%s.yaml", schemaType.name())); + } + + // Postgres Portion + public void importDatabaseFromArchive(final Path storageRoot, final String airbyteVersion) + throws IOException { + try { + final Map> data = new HashMap<>(); + for (DatabaseSchema tableType : DatabaseSchema.values()) { + final Path tablePath = buildTablePath(storageRoot, tableType.name()); + data.put(tableType, readTableFromArchive(tableType, tablePath)); + } + postgresPersistence.importDatabase(airbyteVersion, data); + LOGGER.info("Successful upgrade of airbyte postgres database from archive"); + } catch (Exception e) { + LOGGER.warn("Postgres database version upgrade failed, setting DB version back to initial version"); + postgresPersistence.setVersion(initialVersion); + throw e; + } + } + + protected static Path buildTablePath(final Path storageRoot, final String tableName) { + return storageRoot + .resolve(DB_FOLDER_NAME) + .resolve(String.format("%s.yaml", tableName.toUpperCase())); + } + + private Stream readTableFromArchive(final DatabaseSchema tableSchema, + final Path tablePath) + throws FileNotFoundException { + final JsonNode schema = tableSchema.toJsonNode(); + if (schema != null) { + return MoreStreams.toStream(Yamls.deserialize(IOs.readFile(tablePath)).elements()) + .peek(r -> { + try { + jsonSchemaValidator.ensure(schema, r); + } catch (JsonValidationException e) { + throw new IllegalArgumentException( + "Archived Data Schema does not match current Airbyte Data Schemas", e); + } + }); + } else { + throw new FileNotFoundException(String + .format("Airbyte Database table %s was not found in the archive", tableSchema.name())); + } + } + + private void checkDBVersion(final String airbyteVersion) throws IOException { + final Optional airbyteDatabaseVersion = postgresPersistence.getVersion(); + airbyteDatabaseVersion + .ifPresent(dbversion -> AirbyteVersion.assertIsCompatible(airbyteVersion, dbversion)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/RunMigration.java b/airbyte-server/src/main/java/io/airbyte/server/RunMigration.java new file mode 100644 index 0000000000000..7a8a8d4b4952a --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/RunMigration.java @@ -0,0 +1,105 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server; + +import io.airbyte.api.model.ImportRead; +import io.airbyte.api.model.ImportRead.StatusEnum; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.migrate.MigrateConfig; +import io.airbyte.migrate.MigrationRunner; +import io.airbyte.scheduler.persistence.JobPersistence; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RunMigration implements Runnable, AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(RunMigration.class); + private final String targetVersion; + private final ConfigDumpExport configDumpExport; + private final ConfigDumpImport configDumpImport; + private final List filesToBeCleanedUp = new ArrayList<>(); + + public RunMigration(String initialVersion, + JobPersistence jobPersistence, + ConfigRepository configRepository, + String targetVersion, + Path latestSeeds) { + this.targetVersion = targetVersion; + this.configDumpExport = new ConfigDumpExport(configRepository, jobPersistence, initialVersion); + this.configDumpImport = new ConfigDumpImport(initialVersion, targetVersion, latestSeeds, jobPersistence, configRepository); + } + + @Override + public void run() { + try { + // Export data + File exportData = configDumpExport.dump(); + filesToBeCleanedUp.add(exportData); + + // Define output target + final Path tempFolder = Files.createTempDirectory(Path.of("/tmp"), "airbyte_archive_output"); + final File output = Files.createTempFile(tempFolder, "airbyte_archive_output", ".tar.gz") + .toFile(); + filesToBeCleanedUp.add(output); + filesToBeCleanedUp.add(tempFolder.toFile()); + + // Run Migration + MigrateConfig migrateConfig = new MigrateConfig(exportData.toPath(), output.toPath(), + targetVersion); + MigrationRunner.run(migrateConfig); + + // Import data + ImportRead importRead = configDumpImport.importData(output); + if (importRead.getStatus() == StatusEnum.FAILED) { + throw new RuntimeException("Automatic migration failed : " + importRead.getReason()); + } + + } catch (IOException e) { + throw new RuntimeException("Automatic migration failed", e); + } + } + + @Override + public void close() throws IOException { + for (File file : filesToBeCleanedUp) { + if (file.exists()) { + LOGGER.info("Deleting " + file.getName()); + if (file.isDirectory()) { + FileUtils.deleteDirectory(file); + } else { + Files.delete(file.toPath()); + } + } + } + } + +} diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedRepository.java b/airbyte-server/src/main/java/io/airbyte/server/SeedRepository.java similarity index 99% rename from airbyte-config/init/src/main/java/io/airbyte/config/init/SeedRepository.java rename to airbyte-server/src/main/java/io/airbyte/server/SeedRepository.java index 0955b56c911ac..b815dd1937185 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedRepository.java +++ b/airbyte-server/src/main/java/io/airbyte/server/SeedRepository.java @@ -22,7 +22,7 @@ * SOFTWARE. */ -package io.airbyte.config.init; +package io.airbyte.server; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.io.IOs; diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 2517cc301375a..2a5293c5c7caa 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -29,6 +29,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; +import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.EnvConfigs; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.helpers.LogHelpers; @@ -210,15 +211,50 @@ public static void main(String[] args) throws Exception { LOGGER.info(String.format("Setting Database version to %s...", airbyteVersion)); jobPersistence.setVersion(airbyteVersion); } - final Optional airbyteDatabaseVersion = jobPersistence.getVersion(); + + Optional airbyteDatabaseVersion = jobPersistence.getVersion(); + boolean isKubernetes = configs.getWorkerEnvironment() == WorkerEnvironment.KUBERNETES; + if (!isKubernetes && airbyteDatabaseVersion.isPresent() && !AirbyteVersion.isCompatible(airbyteVersion, airbyteDatabaseVersion.get()) + && !isDatabaseVersionAheadOfAppVersion(airbyteVersion, airbyteDatabaseVersion.get())) { + runAutomaticMigration(configRepository, jobPersistence, airbyteVersion, airbyteDatabaseVersion.get()); + // After migration, upgrade the DB version + airbyteDatabaseVersion = jobPersistence.getVersion(); + } if (airbyteDatabaseVersion.isPresent() && AirbyteVersion.isCompatible(airbyteVersion, airbyteDatabaseVersion.get())) { LOGGER.info("Starting server..."); new ServerApp(configRepository, jobPersistence, configs).start(); } else { - LOGGER.info("Start serving version mismatch errors..."); + LOGGER.info("Start serving version mismatch errors. " + + (isKubernetes ? " Automatic migration can't run on KUBERNETES." : " Automatic migration must have failed")); new VersionMismatchServer(airbyteVersion, airbyteDatabaseVersion.get(), PORT).start(); } } + private static void runAutomaticMigration(ConfigRepository configRepository, + JobPersistence jobPersistence, + String airbyteVersion, + String airbyteDatabaseVersion) { + LOGGER.info("Running Automatic Migration from version : " + airbyteDatabaseVersion + " to version : " + airbyteVersion); + final Path latestSeedsPath = Path.of(System.getProperty("user.dir")).resolve("latest_seeds"); + LOGGER.info("Last seeds dir: {}", latestSeedsPath); + try (RunMigration runMigration = new RunMigration(airbyteDatabaseVersion, + jobPersistence, configRepository, airbyteVersion, latestSeedsPath)) { + runMigration.run(); + } catch (Exception e) { + LOGGER.error("Automatic Migration failed ", e); + } + } + + public static boolean isDatabaseVersionAheadOfAppVersion(String airbyteVersion, String airbyteDatabaseVersion) { + AirbyteVersion serverVersion = new AirbyteVersion(airbyteVersion); + AirbyteVersion databaseVersion = new AirbyteVersion(airbyteDatabaseVersion); + + if (databaseVersion.getMajorVersion().compareTo(serverVersion.getMajorVersion()) > 0) { + return true; + } + + return databaseVersion.getMinorVersion().compareTo(serverVersion.getMinorVersion()) > 0; + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java index 82347325416ff..16e74d3cd94e7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java +++ b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java @@ -43,9 +43,9 @@ public class AirbyteGithubStore { private static final String GITHUB_BASE_URL = "https://raw.githubusercontent.com"; private static final String SOURCE_DEFINITION_LIST_LOCATION_PATH = - "/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/seed/source_definitions.yaml"; + "/airbytehq/airbyte/master/airbyte-server/src/main/resources/seed/source_definitions.yaml"; private static final String DESTINATION_DEFINITION_LIST_LOCATION_PATH = - "/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml"; + "/airbytehq/airbyte/master/airbyte-server/src/main/resources/seed/destination_definitions.yaml"; private static final HttpClient httpClient = HttpClient.newHttpClient(); diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/356668e2-7e34-47f3-a3b0-67a8a481b692.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/3986776d-2319-4de9-8af8-db14c0996e72.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/d4353156-9217-4cad-8dd7-c108fd4f74cf.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/d4353156-9217-4cad-8dd7-c108fd4f74cf.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/d4353156-9217-4cad-8dd7-c108fd4f74cf.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/d4353156-9217-4cad-8dd7-c108fd4f74cf.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json rename to airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/29b409d9-30a5-4cc8-ad50-886eb846fea3.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/29b409d9-30a5-4cc8-ad50-886eb846fea3.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/29b409d9-30a5-4cc8-ad50-886eb846fea3.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/29b409d9-30a5-4cc8-ad50-886eb846fea3.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2e875208-0c0b-4ee4-9e92-1cb3156ea799.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2e875208-0c0b-4ee4-9e92-1cb3156ea799.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2e875208-0c0b-4ee4-9e92-1cb3156ea799.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2e875208-0c0b-4ee4-9e92-1cb3156ea799.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/374ebc65-6636-4ea0-925c-7d35999a8ffc.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/374ebc65-6636-4ea0-925c-7d35999a8ffc.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/374ebc65-6636-4ea0-925c-7d35999a8ffc.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/374ebc65-6636-4ea0-925c-7d35999a8ffc.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5a1d14c2-d829-49cd-8437-1e87dc9f5368.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5a1d14c2-d829-49cd-8437-1e87dc9f5368.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5a1d14c2-d829-49cd-8437-1e87dc9f5368.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5a1d14c2-d829-49cd-8437-1e87dc9f5368.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6acf6b55-4f1e-4fca-944e-1a3caef8aba8.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6acf6b55-4f1e-4fca-944e-1a3caef8aba8.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6acf6b55-4f1e-4fca-944e-1a3caef8aba8.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6acf6b55-4f1e-4fca-944e-1a3caef8aba8.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6ff047c0-f5d5-4ce5-8c81-204a830fa7e1.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6ff047c0-f5d5-4ce5-8c81-204a830fa7e1.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6ff047c0-f5d5-4ce5-8c81-204a830fa7e1.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6ff047c0-f5d5-4ce5-8c81-204a830fa7e1.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bad83517-5e54-4a3d-9b53-63e85fbd4d7c.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c8630570-086d-4a40-99ae-ea5b18673071.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c8630570-086d-4a40-99ae-ea5b18673071.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c8630570-086d-4a40-99ae-ea5b18673071.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c8630570-086d-4a40-99ae-ea5b18673071.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d0243522-dccf-4978-8ba0-37ed47a0bdbf.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d0243522-dccf-4978-8ba0-37ed47a0bdbf.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d0243522-dccf-4978-8ba0-37ed47a0bdbf.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d0243522-dccf-4978-8ba0-37ed47a0bdbf.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d1aa448b-7c54-498e-ad95-263cbebcd2db.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d1aa448b-7c54-498e-ad95-263cbebcd2db.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d1aa448b-7c54-498e-ad95-263cbebcd2db.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d1aa448b-7c54-498e-ad95-263cbebcd2db.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2b40e36-aa0e-4bed-b41b-bcea6fa348b1.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2b40e36-aa0e-4bed-b41b-bcea6fa348b1.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2b40e36-aa0e-4bed-b41b-bcea6fa348b1.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2b40e36-aa0e-4bed-b41b-bcea6fa348b1.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e55879a8-0ef8-4557-abcf-ab34c53ec460.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json b/airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json rename to airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fe2b4084-3386-4d3b-9ad6-308f61a6f1e6.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json b/airbyte-server/src/main/resources/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json similarity index 100% rename from airbyte-config/init/src/main/resources/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json rename to airbyte-server/src/main/resources/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json diff --git a/airbyte-config/init/src/main/resources/icons/appstore.svg b/airbyte-server/src/main/resources/icons/appstore.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/appstore.svg rename to airbyte-server/src/main/resources/icons/appstore.svg diff --git a/airbyte-config/init/src/main/resources/icons/asana.svg b/airbyte-server/src/main/resources/icons/asana.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/asana.svg rename to airbyte-server/src/main/resources/icons/asana.svg diff --git a/airbyte-config/init/src/main/resources/icons/braintree.svg b/airbyte-server/src/main/resources/icons/braintree.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/braintree.svg rename to airbyte-server/src/main/resources/icons/braintree.svg diff --git a/airbyte-config/init/src/main/resources/icons/db2.svg b/airbyte-server/src/main/resources/icons/db2.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/db2.svg rename to airbyte-server/src/main/resources/icons/db2.svg diff --git a/airbyte-config/init/src/main/resources/icons/drift.svg b/airbyte-server/src/main/resources/icons/drift.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/drift.svg rename to airbyte-server/src/main/resources/icons/drift.svg diff --git a/airbyte-config/init/src/main/resources/icons/exchangeratesapi.svg b/airbyte-server/src/main/resources/icons/exchangeratesapi.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/exchangeratesapi.svg rename to airbyte-server/src/main/resources/icons/exchangeratesapi.svg diff --git a/airbyte-config/init/src/main/resources/icons/facebook.svg b/airbyte-server/src/main/resources/icons/facebook.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/facebook.svg rename to airbyte-server/src/main/resources/icons/facebook.svg diff --git a/airbyte-config/init/src/main/resources/icons/file.svg b/airbyte-server/src/main/resources/icons/file.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/file.svg rename to airbyte-server/src/main/resources/icons/file.svg diff --git a/airbyte-config/init/src/main/resources/icons/freshdesk.svg b/airbyte-server/src/main/resources/icons/freshdesk.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/freshdesk.svg rename to airbyte-server/src/main/resources/icons/freshdesk.svg diff --git a/airbyte-config/init/src/main/resources/icons/github.svg b/airbyte-server/src/main/resources/icons/github.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/github.svg rename to airbyte-server/src/main/resources/icons/github.svg diff --git a/airbyte-config/init/src/main/resources/icons/google-adwords.svg b/airbyte-server/src/main/resources/icons/google-adwords.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/google-adwords.svg rename to airbyte-server/src/main/resources/icons/google-adwords.svg diff --git a/airbyte-config/init/src/main/resources/icons/google-analytics.svg b/airbyte-server/src/main/resources/icons/google-analytics.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/google-analytics.svg rename to airbyte-server/src/main/resources/icons/google-analytics.svg diff --git a/airbyte-config/init/src/main/resources/icons/google-sheets.svg b/airbyte-server/src/main/resources/icons/google-sheets.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/google-sheets.svg rename to airbyte-server/src/main/resources/icons/google-sheets.svg diff --git a/airbyte-config/init/src/main/resources/icons/greenhouse.svg b/airbyte-server/src/main/resources/icons/greenhouse.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/greenhouse.svg rename to airbyte-server/src/main/resources/icons/greenhouse.svg diff --git a/airbyte-config/init/src/main/resources/icons/http.svg b/airbyte-server/src/main/resources/icons/http.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/http.svg rename to airbyte-server/src/main/resources/icons/http.svg diff --git a/airbyte-config/init/src/main/resources/icons/hubspot.svg b/airbyte-server/src/main/resources/icons/hubspot.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/hubspot.svg rename to airbyte-server/src/main/resources/icons/hubspot.svg diff --git a/airbyte-config/init/src/main/resources/icons/intercom.svg b/airbyte-server/src/main/resources/icons/intercom.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/intercom.svg rename to airbyte-server/src/main/resources/icons/intercom.svg diff --git a/airbyte-config/init/src/main/resources/icons/jira.svg b/airbyte-server/src/main/resources/icons/jira.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/jira.svg rename to airbyte-server/src/main/resources/icons/jira.svg diff --git a/airbyte-config/init/src/main/resources/icons/looker.svg b/airbyte-server/src/main/resources/icons/looker.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/looker.svg rename to airbyte-server/src/main/resources/icons/looker.svg diff --git a/airbyte-config/init/src/main/resources/icons/mailchimp.svg b/airbyte-server/src/main/resources/icons/mailchimp.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/mailchimp.svg rename to airbyte-server/src/main/resources/icons/mailchimp.svg diff --git a/airbyte-config/init/src/main/resources/icons/marketo.svg b/airbyte-server/src/main/resources/icons/marketo.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/marketo.svg rename to airbyte-server/src/main/resources/icons/marketo.svg diff --git a/airbyte-config/init/src/main/resources/icons/microsoft-teams.svg b/airbyte-server/src/main/resources/icons/microsoft-teams.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/microsoft-teams.svg rename to airbyte-server/src/main/resources/icons/microsoft-teams.svg diff --git a/airbyte-config/init/src/main/resources/icons/mixpanel.svg b/airbyte-server/src/main/resources/icons/mixpanel.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/mixpanel.svg rename to airbyte-server/src/main/resources/icons/mixpanel.svg diff --git a/airbyte-config/init/src/main/resources/icons/mongodb.svg b/airbyte-server/src/main/resources/icons/mongodb.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/mongodb.svg rename to airbyte-server/src/main/resources/icons/mongodb.svg diff --git a/airbyte-config/init/src/main/resources/icons/mssql.svg b/airbyte-server/src/main/resources/icons/mssql.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/mssql.svg rename to airbyte-server/src/main/resources/icons/mssql.svg diff --git a/airbyte-config/init/src/main/resources/icons/mysql.svg b/airbyte-server/src/main/resources/icons/mysql.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/mysql.svg rename to airbyte-server/src/main/resources/icons/mysql.svg diff --git a/airbyte-config/init/src/main/resources/icons/plaid.svg b/airbyte-server/src/main/resources/icons/plaid.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/plaid.svg rename to airbyte-server/src/main/resources/icons/plaid.svg diff --git a/airbyte-config/init/src/main/resources/icons/postgresql.svg b/airbyte-server/src/main/resources/icons/postgresql.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/postgresql.svg rename to airbyte-server/src/main/resources/icons/postgresql.svg diff --git a/airbyte-config/init/src/main/resources/icons/recurly.svg b/airbyte-server/src/main/resources/icons/recurly.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/recurly.svg rename to airbyte-server/src/main/resources/icons/recurly.svg diff --git a/airbyte-config/init/src/main/resources/icons/redshift.svg b/airbyte-server/src/main/resources/icons/redshift.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/redshift.svg rename to airbyte-server/src/main/resources/icons/redshift.svg diff --git a/airbyte-config/init/src/main/resources/icons/s3.svg b/airbyte-server/src/main/resources/icons/s3.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/s3.svg rename to airbyte-server/src/main/resources/icons/s3.svg diff --git a/airbyte-config/init/src/main/resources/icons/salesforce.svg b/airbyte-server/src/main/resources/icons/salesforce.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/salesforce.svg rename to airbyte-server/src/main/resources/icons/salesforce.svg diff --git a/airbyte-config/init/src/main/resources/icons/sendgrid.svg b/airbyte-server/src/main/resources/icons/sendgrid.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/sendgrid.svg rename to airbyte-server/src/main/resources/icons/sendgrid.svg diff --git a/airbyte-config/init/src/main/resources/icons/shopify.svg b/airbyte-server/src/main/resources/icons/shopify.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/shopify.svg rename to airbyte-server/src/main/resources/icons/shopify.svg diff --git a/airbyte-config/init/src/main/resources/icons/slack.svg b/airbyte-server/src/main/resources/icons/slack.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/slack.svg rename to airbyte-server/src/main/resources/icons/slack.svg diff --git a/airbyte-config/init/src/main/resources/icons/snowflake.svg b/airbyte-server/src/main/resources/icons/snowflake.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/snowflake.svg rename to airbyte-server/src/main/resources/icons/snowflake.svg diff --git a/airbyte-config/init/src/main/resources/icons/stripe.svg b/airbyte-server/src/main/resources/icons/stripe.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/stripe.svg rename to airbyte-server/src/main/resources/icons/stripe.svg diff --git a/airbyte-config/init/src/main/resources/icons/twilio.svg b/airbyte-server/src/main/resources/icons/twilio.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/twilio.svg rename to airbyte-server/src/main/resources/icons/twilio.svg diff --git a/airbyte-config/init/src/main/resources/icons/zendesk.svg b/airbyte-server/src/main/resources/icons/zendesk.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/zendesk.svg rename to airbyte-server/src/main/resources/icons/zendesk.svg diff --git a/airbyte-config/init/src/main/resources/icons/zoom.svg b/airbyte-server/src/main/resources/icons/zoom.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/zoom.svg rename to airbyte-server/src/main/resources/icons/zoom.svg diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-server/src/main/resources/seed/destination_definitions.yaml similarity index 100% rename from airbyte-config/init/src/main/resources/seed/destination_definitions.yaml rename to airbyte-server/src/main/resources/seed/destination_definitions.yaml diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-server/src/main/resources/seed/source_definitions.yaml similarity index 100% rename from airbyte-config/init/src/main/resources/seed/source_definitions.yaml rename to airbyte-server/src/main/resources/seed/source_definitions.yaml diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/SeedRepositoryTest.java b/airbyte-server/src/test/java/io/airbyte/server/SeedRepositoryTest.java similarity index 96% rename from airbyte-config/init/src/test/java/io/airbyte/config/init/SeedRepositoryTest.java rename to airbyte-server/src/test/java/io/airbyte/server/SeedRepositoryTest.java index 305d0c0ee455d..e03455038b02b 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/SeedRepositoryTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/SeedRepositoryTest.java @@ -22,9 +22,10 @@ * SOFTWARE. */ -package io.airbyte.config.init; +package io.airbyte.server; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java b/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java new file mode 100644 index 0000000000000..e2610d79315e5 --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java @@ -0,0 +1,311 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server.migration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.google.common.io.Resources; +import io.airbyte.commons.io.Archives; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.OperatorNormalization.Option; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncOperation.OperatorType; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.DefaultConfigPersistence; +import io.airbyte.db.Database; +import io.airbyte.migrate.Migrations; +import io.airbyte.scheduler.persistence.DefaultJobPersistence; +import io.airbyte.scheduler.persistence.JobPersistence; +import io.airbyte.server.RunMigration; +import io.airbyte.server.converters.DatabaseArchiver; +import io.airbyte.validation.json.JsonValidationException; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.jupiter.api.Test; + +public class RunMigrationTest { + + private static final String INITIAL_VERSION = "0.17.0-alpha"; + private static final String TARGET_VERSION = Migrations.MIGRATIONS + .get(Migrations.MIGRATIONS.size() - 1).getVersion(); + private final List resourceToBeCleanedUp = new ArrayList<>(); + + @After + public void cleanup() throws IOException { + for (File file : resourceToBeCleanedUp) { + if (file.exists()) { + if (file.isDirectory()) { + FileUtils.deleteDirectory(file); + } else { + Files.delete(file.toPath()); + } + } + } + } + + @Test + public void testRunMigration() { + try (StubAirbyteDB stubAirbyteDB = new StubAirbyteDB()) { + final File file = Path + .of(Resources.getResource("migration/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz").toURI()) + .toFile(); + + Path dummyDataSource = Path.of(Resources.getResource("migration/dummy_data").toURI()); + + Path configRoot = Files.createTempDirectory("dummy_data"); + FileUtils.copyDirectory(dummyDataSource.toFile(), configRoot.toFile()); + resourceToBeCleanedUp.add(configRoot.toFile()); + JobPersistence jobPersistence = getJobPersistence(stubAirbyteDB.getDatabase(), file, + INITIAL_VERSION); + assertDatabaseVersion(jobPersistence, INITIAL_VERSION); + + runMigration(jobPersistence, configRoot); + + assertDatabaseVersion(jobPersistence, TARGET_VERSION); + assertPostMigrationConfigs(configRoot); + FileUtils.deleteDirectory(configRoot.toFile()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void assertDatabaseVersion(JobPersistence jobPersistence, String version) + throws IOException { + Optional versionFromDb = jobPersistence.getVersion(); + assertTrue(versionFromDb.isPresent()); + assertEquals(versionFromDb.get(), version); + } + + private void assertPostMigrationConfigs(Path importRoot) + throws IOException, JsonValidationException, ConfigNotFoundException { + ConfigRepository configRepository = new ConfigRepository( + new DefaultConfigPersistence(importRoot)); + StandardSyncOperation standardSyncOperation = assertSyncOperations(configRepository); + assertStandardSyncs(configRepository, standardSyncOperation); + assertWorkspace(configRepository); + assertSources(configRepository); + assertDestinations(configRepository); + assertSourceDefinitions(configRepository); + assertDestinationDefinitions(configRepository); + } + + private void assertSourceDefinitions(ConfigRepository configRepository) throws JsonValidationException, IOException { + Map sourceDefinitions = + configRepository.listStandardSources().stream().collect(Collectors.toMap(c -> c.getSourceDefinitionId().toString(), c -> c)); + assertTrue(sourceDefinitions.size() >= 59); + StandardSourceDefinition mysqlDefinition = sourceDefinitions.get("435bb9a5-7887-4809-aa58-28c27df0d7ad"); + assertEquals("0.2.0", mysqlDefinition.getDockerImageTag()); + assertEquals("MySQL", mysqlDefinition.getName()); + + StandardSourceDefinition postgresDefinition = sourceDefinitions.get("decd338e-5647-4c0b-adf4-da0e75f5a750"); + assertTrue(postgresDefinition.getDockerImageTag().compareTo("0.3.4") >= 0); + assertTrue(postgresDefinition.getName().contains("Postgres")); + } + + private void assertDestinationDefinitions(ConfigRepository configRepository) throws JsonValidationException, IOException { + Map sourceDefinitions = configRepository.listStandardDestinationDefinitions().stream() + .collect(Collectors.toMap(c -> c.getDestinationDefinitionId().toString(), c -> c)); + assertTrue(sourceDefinitions.size() >= 11); + StandardDestinationDefinition postgresDefinition = sourceDefinitions.get("25c5221d-dce2-4163-ade9-739ef790f503"); + assertEquals("0.2.0", postgresDefinition.getDockerImageTag()); + assertEquals(postgresDefinition.getName(), "Postgres"); + + StandardDestinationDefinition localCsvDefinition = sourceDefinitions.get("8be1cf83-fde1-477f-a4ad-318d23c9f3c6"); + assertTrue(localCsvDefinition.getName().contains("Local CSV")); + assertEquals("0.2.0", localCsvDefinition.getDockerImageTag()); + + StandardDestinationDefinition snowflakeDefinition = sourceDefinitions.get("424892c4-daac-4491-b35d-c6688ba547ba"); + assertTrue(snowflakeDefinition.getDockerImageTag().compareTo("0.3.9") >= 0); + assertTrue(snowflakeDefinition.getName().contains("Snowflake")); + } + + private void assertStandardSyncs(ConfigRepository configRepository, + StandardSyncOperation standardSyncOperation) + throws ConfigNotFoundException, IOException, JsonValidationException { + List standardSyncs = configRepository.listStandardSyncs(); + assertEquals(standardSyncs.size(), 2); + for (StandardSync standardSync : standardSyncs) { + if (standardSync.getConnectionId().toString() + .equals("a294256f-1abe-4837-925f-91602c7207b4")) { + assertEquals(standardSync.getPrefix(), ""); + assertEquals(standardSync.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(standardSync.getDestinationId().toString(), + "4e00862d-5484-4f50-9860-f3bbb4317397"); + assertEquals(standardSync.getOperationIds().size(), 1); + assertEquals(standardSync.getOperationIds().get(0).toString(), + standardSyncOperation.getOperationId().toString()); + assertEquals(standardSync.getName(), "default"); + assertEquals(standardSync.getStatus().value(), "active"); + assertNull(standardSync.getSchedule()); + assertTrue(standardSync.getManual()); + } else if (standardSync.getConnectionId().toString() + .equals("49dae3f0-158b-4737-b6e4-0eed77d4b74e")) { + assertEquals(standardSync.getPrefix(), ""); + assertEquals(standardSync.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(standardSync.getDestinationId().toString(), + "5434615d-a3b7-4351-bc6b-a9a695555a30"); + assertEquals(standardSync.getOperationIds().size(), 0); + assertEquals(standardSync.getName(), "default"); + assertEquals(standardSync.getStatus().value(), "active"); + assertNull(standardSync.getSchedule()); + assertTrue(standardSync.getManual()); + } else { + fail("Unknown sync " + standardSync.getConnectionId().toString()); + } + } + } + + @NotNull + private StandardSyncOperation assertSyncOperations(ConfigRepository configRepository) + throws IOException, JsonValidationException { + List standardSyncOperations = configRepository + .listStandardSyncOperations(); + assertEquals(standardSyncOperations.size(), 1); + StandardSyncOperation standardSyncOperation = standardSyncOperations.get(0); + assertEquals(standardSyncOperation.getName(), "default-normalization"); + assertEquals(standardSyncOperation.getOperatorType(), OperatorType.NORMALIZATION); + assertEquals(standardSyncOperation.getOperatorNormalization().getOption(), Option.BASIC); + assertNull(standardSyncOperation.getOperatorDbt()); + assertFalse(standardSyncOperation.getTombstone()); + return standardSyncOperation; + } + + private void assertSources(ConfigRepository configRepository) + throws JsonValidationException, IOException { + List sourceConnections = configRepository.listSourceConnection(); + assertEquals(sourceConnections.size(), 1); + SourceConnection sourceConnection = sourceConnections.get(0); + assertEquals(sourceConnection.getName(), "MySQL localhost"); + assertEquals(sourceConnection.getSourceDefinitionId().toString(), + "435bb9a5-7887-4809-aa58-28c27df0d7ad"); + assertEquals(sourceConnection.getWorkspaceId().toString(), + "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(sourceConnection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(sourceConnection.getConfiguration().get("username").asText(), "root"); + assertEquals(sourceConnection.getConfiguration().get("password").asText(), "password"); + assertEquals(sourceConnection.getConfiguration().get("database").asText(), "localhost_test"); + assertEquals(sourceConnection.getConfiguration().get("port").asInt(), 3306); + assertEquals(sourceConnection.getConfiguration().get("host").asText(), "host.docker.internal"); + } + + private void assertWorkspace(ConfigRepository configRepository) + throws JsonValidationException, IOException { + List standardWorkspaces = configRepository.listStandardWorkspaces(true); + assertEquals(standardWorkspaces.size(), 1); + StandardWorkspace workspace = standardWorkspaces.get(0); + assertEquals(workspace.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(workspace.getCustomerId().toString(), "17f90b72-5ae4-40b7-bc49-d6c2943aea57"); + assertEquals(workspace.getName(), "default"); + assertEquals(workspace.getSlug(), "default"); + assertEquals(workspace.getInitialSetupComplete(), true); + assertEquals(workspace.getAnonymousDataCollection(), false); + assertEquals(workspace.getNews(), false); + assertEquals(workspace.getSecurityUpdates(), false); + assertEquals(workspace.getDisplaySetupWizard(), false); + } + + private void assertDestinations(ConfigRepository configRepository) + throws JsonValidationException, IOException { + List destinationConnections = configRepository + .listDestinationConnection(); + assertEquals(destinationConnections.size(), 2); + for (DestinationConnection destination : destinationConnections) { + if (destination.getDestinationId().toString() + .equals("4e00862d-5484-4f50-9860-f3bbb4317397")) { + assertEquals(destination.getName(), "Postgres Docker"); + assertEquals(destination.getDestinationDefinitionId().toString(), + "25c5221d-dce2-4163-ade9-739ef790f503"); + assertEquals(destination.getWorkspaceId().toString(), + "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(destination.getConfiguration().get("username").asText(), "postgres"); + assertEquals(destination.getConfiguration().get("password").asText(), "password"); + assertEquals(destination.getConfiguration().get("database").asText(), "postgres"); + assertEquals(destination.getConfiguration().get("schema").asText(), "public"); + assertEquals(destination.getConfiguration().get("port").asInt(), 3000); + assertEquals(destination.getConfiguration().get("host").asText(), "localhost"); + assertNull(destination.getConfiguration().get("basic_normalization")); + } else if (destination.getDestinationId().toString() + .equals("5434615d-a3b7-4351-bc6b-a9a695555a30")) { + assertEquals(destination.getName(), "CSV"); + assertEquals(destination.getDestinationDefinitionId().toString(), + "8be1cf83-fde1-477f-a4ad-318d23c9f3c6"); + assertEquals(destination.getWorkspaceId().toString(), + "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(destination.getConfiguration().get("destination_path").asText(), "csv_data"); + } else { + fail("Unknown destination found with destination id : " + destination.getDestinationId() + .toString()); + } + } + } + + private void runMigration(JobPersistence jobPersistence, Path exportConfigRoot) + throws IOException, URISyntaxException { + try (RunMigration runMigration = new RunMigration( + INITIAL_VERSION, + jobPersistence, + new ConfigRepository(new DefaultConfigPersistence(exportConfigRoot)), + TARGET_VERSION, + Path.of(Resources.getResource("config").toURI()))) { + runMigration.run(); + } + } + + private JobPersistence getJobPersistence(Database database, + File file, + String version) + throws IOException { + DefaultJobPersistence jobPersistence = new DefaultJobPersistence(database); + final Path tempFolder = Files.createTempDirectory(Path.of("/tmp"), "db_init"); + resourceToBeCleanedUp.add(tempFolder.toFile()); + + Archives.extractArchive(file.toPath(), tempFolder); + DatabaseArchiver databaseArchiver = new DatabaseArchiver(jobPersistence); + databaseArchiver.importDatabaseFromArchive(tempFolder, version); + return jobPersistence; + } + +} diff --git a/airbyte-server/src/test/java/io/airbyte/server/migration/StubAirbyteDB.java b/airbyte-server/src/test/java/io/airbyte/server/migration/StubAirbyteDB.java new file mode 100644 index 0000000000000..36bda7fa6fde5 --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/migration/StubAirbyteDB.java @@ -0,0 +1,67 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server.migration; + +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import java.io.IOException; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.MountableFile; + +public class StubAirbyteDB implements AutoCloseable { + + private final PostgreSQLContainer container; + private final Database database; + + public Database getDatabase() { + return database; + } + + public StubAirbyteDB() throws IOException, InterruptedException { + container = + new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName("airbyte") + .withUsername("docker") + .withPassword("docker"); + container.start(); + + container + .copyFileToContainer(MountableFile.forClasspathResource("migration/schema.sql"), "/etc/init.sql"); + // execInContainer uses Docker's EXEC so it needs to be split up like this + container.execInContainer("psql", "-d", "airbyte", "-U", "docker", "-a", "-f", "/etc/init.sql"); + + database = Databases + .createPostgresDatabase(container.getUsername(), container.getPassword(), + container.getJdbcUrl()); + + } + + @Override + public void close() throws Exception { + database.close(); + container.close(); + } + +} diff --git a/airbyte-server/src/test/resources/migration/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz b/airbyte-server/src/test/resources/migration/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz new file mode 100644 index 0000000000000..0e9f6b2c40691 Binary files /dev/null and b/airbyte-server/src/test/resources/migration/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz differ diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/4e00862d-5484-4f50-9860-f3bbb4317397.json b/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/4e00862d-5484-4f50-9860-f3bbb4317397.json new file mode 100644 index 0000000000000..b88d0a44331d4 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/4e00862d-5484-4f50-9860-f3bbb4317397.json @@ -0,0 +1,16 @@ +{ + "name": "Postgres Docker", + "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", + "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", + "destinationId": "4e00862d-5484-4f50-9860-f3bbb4317397", + "configuration": { + "basic_normalization": true, + "username": "postgres", + "password": "password", + "database": "postgres", + "schema": "public", + "port": 3000, + "host": "localhost" + }, + "tombstone": false +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/5434615d-a3b7-4351-bc6b-a9a695555a30.json b/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/5434615d-a3b7-4351-bc6b-a9a695555a30.json new file mode 100644 index 0000000000000..1820bef139d33 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/DESTINATION_CONNECTION/5434615d-a3b7-4351-bc6b-a9a695555a30.json @@ -0,0 +1,8 @@ +{ + "name": "CSV", + "destinationDefinitionId": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6", + "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", + "destinationId": "5434615d-a3b7-4351-bc6b-a9a695555a30", + "configuration": { "destination_path": "csv_data" }, + "tombstone": false +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/28ffee2b-372a-4f72-9b95-8ed56a8b99c5.json b/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/28ffee2b-372a-4f72-9b95-8ed56a8b99c5.json new file mode 100644 index 0000000000000..fc15d49eb16e0 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/28ffee2b-372a-4f72-9b95-8ed56a8b99c5.json @@ -0,0 +1,14 @@ +{ + "name": "MySQL localhost", + "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", + "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", + "sourceId": "28ffee2b-372a-4f72-9b95-8ed56a8b99c5", + "configuration": { + "username": "root", + "password": "password", + "database": "localhost_test", + "port": 3306, + "host": "host.docker.internal" + }, + "tombstone": false +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json new file mode 100644 index 0000000000000..7e7a55eacace1 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", + "name": "BigQuery", + "dockerRepository": "airbyte/destination-bigquery", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json new file mode 100644 index 0000000000000..2bd62e324b945 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/25c5221d-dce2-4163-ade9-739ef790f503.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", + "name": "Postgres", + "dockerRepository": "airbyte/destination-postgres", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json new file mode 100644 index 0000000000000..8d679bb252c87 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", + "name": "Snowflake", + "dockerRepository": "airbyte/destination-snowflake", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json new file mode 100644 index 0000000000000..3d1a8ea6f8484 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6", + "name": "Local CSV", + "dockerRepository": "airbyte/destination-csv", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-csv" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json new file mode 100644 index 0000000000000..d7ecf5e27e83f --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/a625d593-bba5-4a1c-a53d-2d246268a816.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "a625d593-bba5-4a1c-a53d-2d246268a816", + "name": "Local JSON", + "dockerRepository": "airbyte/destination-local-json", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-json" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json new file mode 100644 index 0000000000000..21018601a7130 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", + "name": "MeiliSearch", + "dockerRepository": "airbyte/destination-meilisearch", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json new file mode 100644 index 0000000000000..f686204fee6b4 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", + "name": "Redshift", + "dockerRepository": "airbyte/destination-redshift", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json new file mode 100644 index 0000000000000..771a38d386fd5 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", + "name": "Looker", + "dockerRepository": "airbyte/source-looker", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-looker" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json new file mode 100644 index 0000000000000..df7ea85dc259c --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77", + "name": "Salesforce", + "dockerRepository": "airbyte/source-salesforce-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-salesforce-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json new file mode 100644 index 0000000000000..4ec108f990c2e --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/2af123bf-0aaf-4e0d-9784-cb497f23741a.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "2af123bf-0aaf-4e0d-9784-cb497f23741a", + "name": "Appstore", + "dockerRepository": "airbyte/source-appstore-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-appstore-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json new file mode 100644 index 0000000000000..0d62416473a55 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992", + "name": "Braintree", + "dockerRepository": "airbyte/source-braintree-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-braintree-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json new file mode 100644 index 0000000000000..e496f936f69ee --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1", + "name": "Google Analytics", + "dockerRepository": "airbyte/source-googleanalytics-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-googleanalytics-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json new file mode 100644 index 0000000000000..40eb6405be223 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/41375467-61ae-4204-8e38-e2b8b7365f23.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "41375467-61ae-4204-8e38-e2b8b7365f23", + "name": "Slack", + "dockerRepository": "airbyte/source-slack-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-slack-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json new file mode 100644 index 0000000000000..a6b3ef3f1b76d --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", + "name": "MySQL", + "dockerRepository": "airbyte/source-mysql", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json new file mode 100644 index 0000000000000..f12b4cecfa999 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "445831eb-78db-4b1f-8f1f-0d96ad8739e2", + "name": "Drift", + "dockerRepository": "airbyte/source-drift", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-drift" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json new file mode 100644 index 0000000000000..7995d066a56d8 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/487b930d-7f6a-43ce-8bac-46e6b2de0a55.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "487b930d-7f6a-43ce-8bac-46e6b2de0a55", + "name": "Mongo DB", + "dockerRepository": "airbyte/source-mongodb", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-mongodb" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json new file mode 100644 index 0000000000000..bf5fbf68969bd --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "57eb1576-8f52-463d-beb6-2e107cdf571d", + "name": "Hubspot", + "dockerRepository": "airbyte/source-hubspot-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json new file mode 100644 index 0000000000000..3e3cf22a56d75 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", + "name": "Greenhouse", + "dockerRepository": "airbyte/source-greenhouse", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/greenhouse" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json new file mode 100644 index 0000000000000..19c37f45bd7ab --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/68e63de2-bb83-4c7e-93fa-a8a9051e3993.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "68e63de2-bb83-4c7e-93fa-a8a9051e3993", + "name": "Jira", + "dockerRepository": "airbyte/source-jira", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-jira" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json new file mode 100644 index 0000000000000..9239ec3348d6a --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/71607ba1-c0ac-4799-8049-7f4b90dd50f7.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7", + "name": "Google Sheets", + "dockerRepository": "airbyte/source-google-sheets", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-google-sheets" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json new file mode 100644 index 0000000000000..70cb8a722c71f --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/778daa7c-feaf-4db6-96f3-70fd645acc77.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", + "name": "File", + "dockerRepository": "airbyte/source-file", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-file" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json new file mode 100644 index 0000000000000..e57ba78e2758d --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/859e501d-2b67-471f-91bb-1c801414d28f.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "859e501d-2b67-471f-91bb-1c801414d28f", + "name": "Mixpanel", + "dockerRepository": "airbyte/source-mixpanel-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-mixpanel-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json new file mode 100644 index 0000000000000..3d365c2a57e8b --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/932e6363-d006-4464-a9f5-102b82e07c06.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c06", + "name": "Twilio", + "dockerRepository": "airbyte/source-twilio-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-twilio-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json new file mode 100644 index 0000000000000..43fe058298656 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b", + "name": "HTTP Request", + "dockerRepository": "airbyte/source-http-request", + "dockerImageTag": "0.2.1", + "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json new file mode 100644 index 0000000000000..710d901172405 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", + "name": "Marketo", + "dockerRepository": "airbyte/source-marketo-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-marketo-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9fed261d-d107-47fd-8c8b-323023db6e20.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9fed261d-d107-47fd-8c8b-323023db6e20.json new file mode 100644 index 0000000000000..7134b24eed16a --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/9fed261d-d107-47fd-8c8b-323023db6e20.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "9fed261d-d107-47fd-8c8b-323023db6e20", + "name": "Exchange Rates Api", + "dockerRepository": "airbyte/source-exchangeratesapi-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-exchangeratesapi_io-source" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json new file mode 100644 index 0000000000000..1bbb2570076ec --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/aea2fd0d-377d-465e-86c0-4fdc4f688e51.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "aea2fd0d-377d-465e-86c0-4fdc4f688e51", + "name": "Zoom", + "dockerRepository": "airbyte/source-zoom-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-zoom-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json new file mode 100644 index 0000000000000..b2017c8322096 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b03a9f3e-22a5-11eb-adc1-0242ac120002.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002", + "name": "Mailchimp", + "dockerRepository": "airbyte/source-mailchimp", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-mailchimp" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json new file mode 100644 index 0000000000000..f94e9c4f6f48e --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b1892b11-788d-44bd-b9ec-3a436f7b54ce.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "b1892b11-788d-44bd-b9ec-3a436f7b54ce", + "name": "Shopify", + "dockerRepository": "airbyte/source-shopify-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-shopify-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json new file mode 100644 index 0000000000000..a6d4d6620c813 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", + "name": "Microsoft SQL Server (MSSQL)", + "dockerRepository": "airbyte/source-mssql", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-mssql" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json new file mode 100644 index 0000000000000..c4bd3064a9a4d --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/cd42861b-01fc-4658-a8ab-5d11d0510f01.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01", + "name": "Recurly", + "dockerRepository": "airbyte/source-recurly", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-recurly" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json new file mode 100644 index 0000000000000..5bc2bdc6ad62b --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d29764f8-80d7-4dd7-acbe-1a42005ee5aa.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d29764f8-80d7-4dd7-acbe-1a42005ee5aa", + "name": "Zendesk Support", + "dockerRepository": "airbyte/source-zendesk-support-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-zendesk-support-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json new file mode 100644 index 0000000000000..81893900811f9 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", + "name": "Intercom", + "dockerRepository": "airbyte/source-intercom-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-intercom-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json new file mode 100644 index 0000000000000..b994bec5ae3f5 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", + "name": "Postgres", + "dockerRepository": "airbyte/source-postgres", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json new file mode 100644 index 0000000000000..88434c56845d1 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", + "name": "Stripe", + "dockerRepository": "airbyte/source-stripe-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-stripe-source" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json new file mode 100644 index 0000000000000..d8a46e6a26514 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", + "name": "Facebook Marketing", + "dockerRepository": "airbyte/source-facebook-marketing", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json new file mode 100644 index 0000000000000..d402393168cd2 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", + "name": "Redshift", + "dockerRepository": "airbyte/source-redshift", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-redshift" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json new file mode 100644 index 0000000000000..ce4f5fdb31dd1 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227", + "name": "Microsoft teams", + "dockerRepository": "airbyte/source-microsoft-teams", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-microsoft-teams" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json new file mode 100644 index 0000000000000..b4dbba4fd0e4c --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ec4b9503-13cb-48ab-a4ab-6ade4be46567.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", + "name": "Freshdesk", + "dockerRepository": "airbyte/source-freshdesk", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-freshdesk" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json new file mode 100644 index 0000000000000..15dd7c90595db --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ed799e2b-2158-4c66-8da4-b40fe63bc72a.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "ed799e2b-2158-4c66-8da4-b40fe63bc72a", + "name": "Plaid", + "dockerRepository": "airbyte/source-plaid", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-plaid" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json new file mode 100644 index 0000000000000..3667ff4b9af12 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", + "name": "GitHub", + "dockerRepository": "airbyte/source-github-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-github-singer" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json new file mode 100644 index 0000000000000..af5a8a8009b0b --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", + "name": "Sendgrid", + "dockerRepository": "airbyte/source-sendgrid", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-sendgrid" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json new file mode 100644 index 0000000000000..fc7b172db4769 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4", + "name": "Google Adwords", + "dockerRepository": "airbyte/source-google-adwords-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json new file mode 100644 index 0000000000000..281271e9b9404 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json @@ -0,0 +1,45 @@ +{ + "prefix": "", + "sourceId": "28ffee2b-372a-4f72-9b95-8ed56a8b99c5", + "destinationId": "5434615d-a3b7-4351-bc6b-a9a695555a30", + "connectionId": "49dae3f0-158b-4737-b6e4-0eed77d4b74e", + "name": "default", + "catalog": { + "streams": [ + { + "stream": { + "name": "localhost_test.new_table", + "json_schema": { + "type": "object", + "properties": { + "id": { "type": "number" }, + "val": { "type": "string" } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [] + }, + "sync_mode": "full_refresh", + "cursor_field": [] + }, + { + "stream": { + "name": "localhost_test.test_table", + "json_schema": { + "type": "object", + "properties": { + "id": { "type": "number" }, + "val": { "type": "string" }, + "updated_at": { "type": "string" } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + } + ] + }, + "status": "active" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/a294256f-1abe-4837-925f-91602c7207b4.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/a294256f-1abe-4837-925f-91602c7207b4.json new file mode 100644 index 0000000000000..9a8b5318a58f5 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC/a294256f-1abe-4837-925f-91602c7207b4.json @@ -0,0 +1,45 @@ +{ + "prefix": "", + "sourceId": "28ffee2b-372a-4f72-9b95-8ed56a8b99c5", + "destinationId": "4e00862d-5484-4f50-9860-f3bbb4317397", + "connectionId": "a294256f-1abe-4837-925f-91602c7207b4", + "name": "default", + "catalog": { + "streams": [ + { + "stream": { + "name": "localhost_test.new_table", + "json_schema": { + "type": "object", + "properties": { + "id": { "type": "number" }, + "val": { "type": "string" } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [] + }, + "sync_mode": "full_refresh", + "cursor_field": [] + }, + { + "stream": { + "name": "localhost_test.test_table", + "json_schema": { + "type": "object", + "properties": { + "id": { "type": "number" }, + "val": { "type": "string" }, + "updated_at": { "type": "string" } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"] + } + ] + }, + "status": "active" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json new file mode 100644 index 0000000000000..3c9c3fb1825b4 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/49dae3f0-158b-4737-b6e4-0eed77d4b74e.json @@ -0,0 +1 @@ +{ "connectionId": "49dae3f0-158b-4737-b6e4-0eed77d4b74e", "manual": true } diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/a294256f-1abe-4837-925f-91602c7207b4.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/a294256f-1abe-4837-925f-91602c7207b4.json new file mode 100644 index 0000000000000..0ed8884c4750a --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SYNC_SCHEDULE/a294256f-1abe-4837-925f-91602c7207b4.json @@ -0,0 +1 @@ +{ "connectionId": "a294256f-1abe-4837-925f-91602c7207b4", "manual": true } diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json new file mode 100644 index 0000000000000..864257ed89661 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_WORKSPACE/5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6.json @@ -0,0 +1,11 @@ +{ + "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", + "customerId": "17f90b72-5ae4-40b7-bc49-d6c2943aea57", + "name": "default", + "slug": "default", + "initialSetupComplete": true, + "anonymousDataCollection": false, + "news": false, + "securityUpdates": false, + "displaySetupWizard": false +} diff --git a/airbyte-server/src/test/resources/migration/schema.sql b/airbyte-server/src/test/resources/migration/schema.sql new file mode 100644 index 0000000000000..39ac4da663287 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/schema.sql @@ -0,0 +1,92 @@ +-- database + CREATE + DATABASE airbyte; + +\connect airbyte; + +-- extensions + CREATE + EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- types + CREATE + TYPE JOB_STATUS AS ENUM( + 'pending', + 'running', + 'incomplete', + 'failed', + 'succeeded', + 'cancelled' + ); + +CREATE + TYPE ATTEMPT_STATUS AS ENUM( + 'running', + 'failed', + 'succeeded' + ); + +CREATE + TYPE JOB_CONFIG_TYPE AS ENUM( + 'check_connection_source', + 'check_connection_destination', + 'discover_schema', + 'get_spec', + 'sync', + 'reset_connection' + ); + +-- tables + CREATE + TABLE + AIRBYTE_METADATA( + KEY VARCHAR(255) PRIMARY KEY, + value VARCHAR(255) + ); + +CREATE + TABLE + JOBS( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + config_type JOB_CONFIG_TYPE, + SCOPE VARCHAR(255), + config JSONB, + status JOB_STATUS, + started_at TIMESTAMPTZ, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ + ); + +CREATE + TABLE + ATTEMPTS( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + job_id BIGINT, + attempt_number INTEGER, + log_path VARCHAR(255), + OUTPUT JSONB, + status ATTEMPT_STATUS, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ, + ended_at TIMESTAMPTZ + ); + +CREATE + UNIQUE INDEX job_attempt_idx ON + ATTEMPTS( + job_id, + attempt_number + ); + +-- entries + INSERT + INTO + AIRBYTE_METADATA + VALUES( + 'server_uuid', + uuid_generate_v4() + ); + +-- grants + GRANT ALL ON +DATABASE airbyte TO docker; diff --git a/airbyte-tests/build.gradle b/airbyte-tests/build.gradle index 6ab443228180c..b1413b16127be 100644 --- a/airbyte-tests/build.gradle +++ b/airbyte-tests/build.gradle @@ -14,6 +14,14 @@ sourceSets { srcDir("src/acceptanceTests/resources") } } + automaticMigrationAcceptanceTest { + java { + srcDir("src/automaticMigrationAcceptanceTest/java") + } + resources { + srcDir("src/automaticMigrationAcceptanceTest/resources") + } + } } // Gradle links configurations with the name xImplementation or xRuntimeOnly etc.. to the source set named x. Therefore, any deps specified @@ -21,6 +29,9 @@ sourceSets { configurations { acceptanceTestsImplementation.extendsFrom testImplementation acceptanceTestsRuntimeOnly.extendsFrom testRuntimeOnly + + automaticMigrationAcceptanceTestImplementation.extendsFrom testImplementation + automaticMigrationAcceptanceTestRuntimeOnly.extendsFrom testRuntimeOnly } dependencies { @@ -34,6 +45,10 @@ dependencies { acceptanceTestsImplementation "org.testcontainers:postgresql:1.15.1" acceptanceTestsImplementation "org.postgresql:postgresql:42.2.18" acceptanceTestsImplementation "com.fasterxml.jackson.core:jackson-databind" + + automaticMigrationAcceptanceTestImplementation project(':airbyte-api') + automaticMigrationAcceptanceTestImplementation project(':airbyte-commons') + automaticMigrationAcceptanceTestImplementation "org.testcontainers:testcontainers:1.15.3" } task acceptanceTests(type: Test) { @@ -53,3 +68,15 @@ task acceptanceTests(type: Test) { dependsOn ':airbyte-integrations:connectors:source-e2e-test:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-e2e-test:airbyteDocker' } + +task automaticMigrationAcceptanceTest(type: Test) { + testClassesDirs += sourceSets.automaticMigrationAcceptanceTest.output.classesDirs + classpath += sourceSets.automaticMigrationAcceptanceTest.runtimeClasspath + useJUnitPlatform() + failFast = true + testLogging() { + events "passed", "failed" + exceptionFormat "full" + } + mustRunAfter test +} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/CustomDockerComposeContainer.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/CustomDockerComposeContainer.java new file mode 100644 index 0000000000000..17d693cab0f89 --- /dev/null +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/CustomDockerComposeContainer.java @@ -0,0 +1,86 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.test.automaticMigrationAcceptance; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.SocatContainer; + +/** + * we need this class to hack the method + * {@link org.testcontainers.containers.DockerComposeContainer#stop()} so that we can do a docker + * compose down without removing the volumes + */ +public class CustomDockerComposeContainer { + + private final DockerComposeContainer dockerComposeContainer; + + public CustomDockerComposeContainer(DockerComposeContainer dockerComposeContainer) { + this.dockerComposeContainer = dockerComposeContainer; + } + + public void start() { + dockerComposeContainer.start(); + } + + /** + * This method is hacked from {@link org.testcontainers.containers.DockerComposeContainer#stop()} We + * needed to do this to avoid removing the volumes when the container is stopped so that the data + * persists and can be tested against in the second run + */ + public void stop() + throws NoSuchFieldException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + Class dockerComposeContainerClass = dockerComposeContainer + .getClass(); + try { + Field ambassadorContainerField = dockerComposeContainerClass.getDeclaredField("ambassadorContainer"); + ambassadorContainerField.setAccessible(true); + SocatContainer ambassadorContainer = (SocatContainer) ambassadorContainerField + .get(dockerComposeContainer); + ambassadorContainer.stop(); + + String cmd = "down "; + + Method runWithComposeMethod = dockerComposeContainerClass + .getDeclaredMethod("runWithCompose", String.class); + runWithComposeMethod.setAccessible(true); + runWithComposeMethod.invoke(dockerComposeContainer, cmd); + + } finally { + Field projectField = dockerComposeContainerClass.getDeclaredField("project"); + projectField.setAccessible(true); + + Method randomProjectId = dockerComposeContainerClass + .getDeclaredMethod("randomProjectId"); + randomProjectId.setAccessible(true); + String newProjectValue = (String) randomProjectId.invoke(dockerComposeContainer); + + projectField.set(dockerComposeContainer, newProjectValue); + } + } + +} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java new file mode 100644 index 0000000000000..b49888f4c42c5 --- /dev/null +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java @@ -0,0 +1,133 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.test.automaticMigrationAcceptance; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.api.client.invoker.ApiClient; +import io.airbyte.api.client.invoker.ApiException; +import io.airbyte.api.client.invoker.ApiResponse; +import io.airbyte.api.client.model.ImportRead; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.Builder; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.function.Consumer; + +/** + * The reason we are using this class instead of {@link io.airbyte.api.client.DeploymentApi is cause + * there is a bug in the the method + * {@link io.airbyte.api.client.DeploymentApi#importArchiveRequestBuilder(File)}, The method + * specifies the content type as `localVarRequestBuilder.header("Content-Type", + * "application/json");` but its supposed to be localVarRequestBuilder.header("Content-Type", + * "application/x-gzip"); + */ +public class ImportApi { + + private final HttpClient memberVarHttpClient; + private final ObjectMapper memberVarObjectMapper; + private final String memberVarBaseUri; + private final Consumer memberVarInterceptor; + private final Duration memberVarReadTimeout; + private final Consumer> memberVarResponseInterceptor; + + public ImportApi(ApiClient apiClient) { + memberVarHttpClient = apiClient.getHttpClient(); + memberVarObjectMapper = apiClient.getObjectMapper(); + memberVarBaseUri = apiClient.getBaseUri(); + memberVarInterceptor = apiClient.getRequestInterceptor(); + memberVarReadTimeout = apiClient.getReadTimeout(); + memberVarResponseInterceptor = apiClient.getResponseInterceptor(); + } + + public ImportRead importArchive(File body) throws ApiException { + ApiResponse localVarResponse = importArchiveWithHttpInfo(body); + return localVarResponse.getData(); + } + + public ApiResponse importArchiveWithHttpInfo(File body) throws ApiException { + HttpRequest.Builder localVarRequestBuilder = importArchiveRequestBuilder(body); + try { + HttpResponse localVarResponse = memberVarHttpClient.send( + localVarRequestBuilder.build(), + HttpResponse.BodyHandlers.ofInputStream()); + if (memberVarResponseInterceptor != null) { + memberVarResponseInterceptor.accept(localVarResponse); + } + if (localVarResponse.statusCode() / 100 != 2) { + throw new ApiException(localVarResponse.statusCode(), + "importArchive call received non-success response", + localVarResponse.headers(), + localVarResponse.body() == null ? null + : new String(localVarResponse.body().readAllBytes())); + } + return new ApiResponse( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + memberVarObjectMapper.readValue(localVarResponse.body(), new TypeReference() {})); + } catch (IOException e) { + throw new ApiException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ApiException(e); + } + } + + private HttpRequest.Builder importArchiveRequestBuilder(File body) throws ApiException { + // verify the required parameter 'body' is set + if (body == null) { + throw new ApiException(400, + "Missing the required parameter 'body' when calling importArchive"); + } + + HttpRequest.Builder localVarRequestBuilder = HttpRequest.newBuilder(); + + String localVarPath = "/v1/deployment/import"; + + localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath)); + + localVarRequestBuilder.header("Content-Type", "application/x-gzip"); + localVarRequestBuilder.header("Accept", "application/json"); + + try { + localVarRequestBuilder.method("POST", HttpRequest.BodyPublishers.ofFile(body.toPath())); + } catch (IOException e) { + throw new ApiException(e); + } + if (memberVarReadTimeout != null) { + localVarRequestBuilder.timeout(memberVarReadTimeout); + } + if (memberVarInterceptor != null) { + memberVarInterceptor.accept(localVarRequestBuilder); + } + return localVarRequestBuilder; + } + +} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java new file mode 100644 index 0000000000000..aaa387793077c --- /dev/null +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -0,0 +1,346 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.test.automaticMigrationAcceptance; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.google.common.io.Resources; +import io.airbyte.api.client.ConnectionApi; +import io.airbyte.api.client.DestinationDefinitionApi; +import io.airbyte.api.client.HealthApi; +import io.airbyte.api.client.SourceDefinitionApi; +import io.airbyte.api.client.WorkspaceApi; +import io.airbyte.api.client.invoker.ApiClient; +import io.airbyte.api.client.invoker.ApiException; +import io.airbyte.api.client.model.ConnectionRead; +import io.airbyte.api.client.model.ConnectionStatus; +import io.airbyte.api.client.model.DestinationDefinitionRead; +import io.airbyte.api.client.model.HealthCheckRead; +import io.airbyte.api.client.model.ImportRead; +import io.airbyte.api.client.model.ImportRead.StatusEnum; +import io.airbyte.api.client.model.SourceDefinitionRead; +import io.airbyte.api.client.model.WorkspaceIdRequestBody; +import io.airbyte.api.client.model.WorkspaceRead; +import io.airbyte.commons.version.AirbyteVersion; +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.output.OutputFrame; + +/** + * In order to run this test from intellij, build the docker images via ./gradlew composeBuild and + * replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env file + */ +public class MigrationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(MigrationAcceptanceTest.class); + + @Test + public void testAutomaticMigration() + throws URISyntaxException, NoSuchFieldException, InvocationTargetException, IllegalAccessException, NoSuchMethodException, ApiException, + InterruptedException { + String targetVersion = System.getenv("MIGRATION_TEST_VERSION"); + firstRun(); + secondRun(targetVersion); + } + + private Consumer logConsumerForServer(Set logs) { + return c -> { + if (c != null && c.getBytes() != null) { + String log = new String(c.getBytes()); + + String keyToRemove = ""; + for (String expected : logs) { + if (log.contains(expected)) { + keyToRemove = expected; + } + } + if (!keyToRemove.isEmpty()) { + logs.remove(keyToRemove); + } + + LOGGER.info(log); + } + }; + } + + private void firstRun() + throws URISyntaxException, InterruptedException, ApiException, NoSuchFieldException, IllegalAccessException, NoSuchMethodException, + InvocationTargetException { + Map environmentVariables = getEnvironmentVariables("0.17.0-alpha"); + final File firstRun = Path + .of(Resources.getResource("docker-compose-migration-test-first-run.yaml").toURI()) + .toFile(); + + Set logsToExpect = new HashSet<>(); + logsToExpect.add("Version: 0.17.0-alpha"); + + DockerComposeContainer dockerComposeContainer = new DockerComposeContainer(firstRun) + .withLogConsumer("server", logConsumerForServer(logsToExpect)) + .withEnv(environmentVariables); + try { + /** + * We are using CustomDockerComposeContainer cause the + * {@link org.testcontainers.containers.DockerComposeContainer#stop()} method also deletes the + * volume but we dont want to delete the volume to test the automatic migration + */ + CustomDockerComposeContainer customDockerComposeContainer = new CustomDockerComposeContainer( + dockerComposeContainer); + + customDockerComposeContainer.start(); + + Thread.sleep(20000); + + assertTrue(logsToExpect.isEmpty()); + ApiClient apiClient = getApiClient(); + healthCheck(apiClient); + populateDataForFirstRun(apiClient); + customDockerComposeContainer.stop(); + } catch (Exception e) { + dockerComposeContainer.stop(); + throw e; + } + } + + private String targetVersionWithoutPatch(String targetVersion) { + return AirbyteVersion.versionWithoutPatch(targetVersion).getVersion(); + } + + private void secondRun(String targetVersion) + throws URISyntaxException, InterruptedException, ApiException { + + Map environmentVariables = getEnvironmentVariables(targetVersion); + final File secondRun = Path + .of(Resources.getResource("docker-compose-migration-test-second-run.yaml") + .toURI()) + .toFile(); + + Set logsToExpect = new HashSet<>(); + logsToExpect.add("Version: " + targetVersion); + logsToExpect.add("Starting migrations. Current version: 0.17.0-alpha, Target version: " + + targetVersionWithoutPatch(targetVersion)); + logsToExpect.add("Migrating from version: 0.17.0-alpha to version 0.18.0-alpha."); + logsToExpect.add("Migrating from version: 0.18.0-alpha to version 0.19.0-alpha."); + logsToExpect.add("Migrating from version: 0.19.0-alpha to version 0.20.0-alpha."); + logsToExpect.add("Migrating from version: 0.20.0-alpha to version 0.21.0-alpha."); + logsToExpect.add("Migrating from version: 0.22.0-alpha to version 0.23.0-alpha."); + logsToExpect.add("Migrations complete. Now on version: " + targetVersionWithoutPatch(targetVersion)); + + DockerComposeContainer dockerComposeContainer = new DockerComposeContainer(secondRun) + .withLogConsumer("server", logConsumerForServer(logsToExpect)) + .withEnv(environmentVariables); + + try { + dockerComposeContainer.start(); + + Thread.sleep(50000); + + ApiClient apiClient = getApiClient(); + healthCheck(apiClient); + + assertTrue(logsToExpect.isEmpty()); + assertDataFromApi(apiClient); + } finally { + dockerComposeContainer.stop(); + } + } + + private void assertDataFromApi(ApiClient apiClient) throws ApiException { + WorkspaceIdRequestBody workspaceIdRequestBody = assertWorkspaceInformation(apiClient); + assertSourceDefinitionInformation(apiClient); + assertDestinationDefinitionInformation(apiClient); + assertConnectionInformation(apiClient, workspaceIdRequestBody); + } + + private void assertSourceDefinitionInformation(ApiClient apiClient) throws ApiException { + SourceDefinitionApi sourceDefinitionApi = new SourceDefinitionApi(apiClient); + List sourceDefinitions = sourceDefinitionApi.listSourceDefinitions() + .getSourceDefinitions(); + assertTrue(sourceDefinitions.size() >= 58); + boolean foundMysqlSourceDefinition = false; + boolean foundPostgresSourceDefinition = false; + for (SourceDefinitionRead sourceDefinitionRead : sourceDefinitions) { + if (sourceDefinitionRead.getSourceDefinitionId().toString() + .equals("435bb9a5-7887-4809-aa58-28c27df0d7ad")) { + assertEquals(sourceDefinitionRead.getName(), "MySQL"); + assertEquals(sourceDefinitionRead.getDockerImageTag(), "0.2.0"); + foundMysqlSourceDefinition = true; + } else if (sourceDefinitionRead.getSourceDefinitionId().toString() + .equals("decd338e-5647-4c0b-adf4-da0e75f5a750")) { + assertTrue(sourceDefinitionRead.getDockerImageTag().compareTo("0.3.4") >= 0); + assertTrue(sourceDefinitionRead.getName().contains("Postgres")); + foundPostgresSourceDefinition = true; + } + } + + assertTrue(foundMysqlSourceDefinition); + assertTrue(foundPostgresSourceDefinition); + } + + private void assertDestinationDefinitionInformation(ApiClient apiClient) throws ApiException { + DestinationDefinitionApi destinationDefinitionApi = new DestinationDefinitionApi(apiClient); + List destinationDefinitions = destinationDefinitionApi + .listDestinationDefinitions().getDestinationDefinitions(); + assertTrue(destinationDefinitions.size() >= 10); + boolean foundPostgresDestinationDefinition = false; + boolean foundLocalCSVDestinationDefinition = false; + boolean foundSnowflakeDestinationDefintion = false; + for (DestinationDefinitionRead destinationDefinitionRead : destinationDefinitions) { + switch (destinationDefinitionRead.getDestinationDefinitionId().toString()) { + case "25c5221d-dce2-4163-ade9-739ef790f503" -> { + assertEquals(destinationDefinitionRead.getName(), "Postgres"); + assertEquals(destinationDefinitionRead.getDockerImageTag(), "0.2.0"); + foundPostgresDestinationDefinition = true; + } + case "8be1cf83-fde1-477f-a4ad-318d23c9f3c6" -> { + assertEquals(destinationDefinitionRead.getDockerImageTag(), "0.2.0"); + assertTrue(destinationDefinitionRead.getName().contains("Local CSV")); + foundLocalCSVDestinationDefinition = true; + } + case "424892c4-daac-4491-b35d-c6688ba547ba" -> { + assertTrue(destinationDefinitionRead.getDockerImageTag().compareTo("0.3.9") >= 0); + assertTrue(destinationDefinitionRead.getName().contains("Snowflake")); + foundSnowflakeDestinationDefintion = true; + } + } + } + + assertTrue(foundPostgresDestinationDefinition); + assertTrue(foundLocalCSVDestinationDefinition); + assertTrue(foundSnowflakeDestinationDefintion); + } + + private void assertConnectionInformation(ApiClient apiClient, + WorkspaceIdRequestBody workspaceIdRequestBody) + throws ApiException { + ConnectionApi connectionApi = new ConnectionApi(apiClient); + List connections = connectionApi + .listConnectionsForWorkspace(workspaceIdRequestBody).getConnections(); + assertEquals(connections.size(), 2); + for (ConnectionRead connection : connections) { + if (connection.getConnectionId().toString() + .equals("a294256f-1abe-4837-925f-91602c7207b4")) { + assertEquals(connection.getPrefix(), ""); + assertEquals(connection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(connection.getDestinationId().toString(), + "4e00862d-5484-4f50-9860-f3bbb4317397"); + assertEquals(connection.getName(), "default"); + assertEquals(connection.getStatus(), ConnectionStatus.ACTIVE); + assertNull(connection.getSchedule()); + } else if (connection.getConnectionId().toString() + .equals("49dae3f0-158b-4737-b6e4-0eed77d4b74e")) { + assertEquals(connection.getPrefix(), ""); + assertEquals(connection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(connection.getDestinationId().toString(), + "5434615d-a3b7-4351-bc6b-a9a695555a30"); + assertEquals(connection.getName(), "default"); + assertEquals(connection.getStatus(), ConnectionStatus.ACTIVE); + assertNull(connection.getSchedule()); + } else { + fail("Unknown sync " + connection.getConnectionId().toString()); + } + } + } + + private WorkspaceIdRequestBody assertWorkspaceInformation(ApiClient apiClient) + throws ApiException { + WorkspaceApi workspaceApi = new WorkspaceApi(apiClient); + WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody() + .workspaceId(UUID.fromString("5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6")); + WorkspaceRead workspace = workspaceApi.getWorkspace(workspaceIdRequestBody); + assertEquals(workspace.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(workspace.getCustomerId().toString(), "17f90b72-5ae4-40b7-bc49-d6c2943aea57"); + assertEquals(workspace.getName(), "default"); + assertEquals(workspace.getSlug(), "default"); + assertEquals(workspace.getInitialSetupComplete(), true); + assertEquals(workspace.getAnonymousDataCollection(), false); + assertEquals(workspace.getNews(), false); + assertEquals(workspace.getSecurityUpdates(), false); + assertEquals(workspace.getDisplaySetupWizard(), false); + return workspaceIdRequestBody; + } + + private void populateDataForFirstRun(ApiClient apiClient) + throws ApiException, URISyntaxException { + ImportApi deploymentApi = new ImportApi(apiClient); + final File file = Path + .of(Resources.getResource("03a4c904-c91d-447f-ab59-27a43b52c2fd.gz").toURI()) + .toFile(); + ImportRead importRead = deploymentApi.importArchive(file); + assertTrue(importRead.getStatus() == StatusEnum.SUCCEEDED); + } + + private void healthCheck(ApiClient apiClient) throws ApiException { + HealthApi healthApi = new HealthApi(apiClient); + HealthCheckRead healthCheck = healthApi.getHealthCheck(); + assertTrue(healthCheck.getDb()); + } + + private ApiClient getApiClient() { + ApiClient apiClient = new ApiClient().setScheme("http") + .setHost("localhost") + .setPort(7001) + .setBasePath("/api"); + return apiClient; + } + + private Map getEnvironmentVariables(String version) { + Map env = new HashMap<>(); + env.put("VERSION", version); + env.put("DATABASE_USER", "docker"); + env.put("DATABASE_PASSWORD", "docker"); + env.put("DATABASE_DB", "airbyte"); + env.put("CONFIG_ROOT", "/data"); + env.put("WORKSPACE_ROOT", "/tmp/workspace"); + env.put("DATA_DOCKER_MOUNT", "airbyte_data_migration_test"); + env.put("DB_DOCKER_MOUNT", "airbyte_db_migration_test"); + env.put("WORKSPACE_DOCKER_MOUNT", "airbyte_workspace_migration_test"); + env.put("LOCAL_ROOT", "/tmp/airbyte_local_migration_test"); + env.put("LOCAL_DOCKER_MOUNT", "/tmp/airbyte_local_migration_test"); + env.put("TRACKING_STRATEGY", "logging"); + env.put("HACK_LOCAL_ROOT_PARENT", "/tmp"); + env.put("WEBAPP_URL", "http://localhost:7000/"); + env.put("API_URL", "http://localhost:7001/api/v1/"); + env.put("TEMPORAL_HOST", "airbyte-temporal:7233"); + env.put("INTERNAL_API_HOST", "airbyte-server:7001"); + return env; + } + +} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz new file mode 100644 index 0000000000000..0e9f6b2c40691 Binary files /dev/null and b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/03a4c904-c91d-447f-ab59-27a43b52c2fd.gz differ diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-first-run.yaml b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-first-run.yaml new file mode 100644 index 0000000000000..4491b5e7c2e06 --- /dev/null +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-first-run.yaml @@ -0,0 +1,63 @@ +#The file is used for testing ans is from version 0.17.0-alpha +version: "3.7" +#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments +x-logging: &default-logging + options: + max-size: "1m" + max-file: "1" + driver: json-file +services: + init: + image: airbyte/init:${VERSION} + logging: *default-logging + command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}" + environment: + - LOCAL_ROOT=${LOCAL_ROOT} + - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT} + volumes: + - ${HACK_LOCAL_ROOT_PARENT}:/local_parent + db: + image: airbyte/db:${VERSION} + logging: *default-logging + restart: unless-stopped + environment: + - POSTGRES_USER=${DATABASE_USER} + - POSTGRES_PASSWORD=${DATABASE_PASSWORD} + volumes: + - db:/var/lib/postgresql/data + seed: + image: airbyte/seed:${VERSION} + # Pre-populate the volume if it is empty. + # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container + volumes: + - data:/app/seed + server: + image: airbyte/server:${VERSION} + logging: *default-logging + restart: unless-stopped + environment: + - WEBAPP_URL=${WEBAPP_URL} + - WAIT_BEFORE_HOSTS=5 + - WAIT_HOSTS_TIMEOUT=45 + - WAIT_HOSTS=db:5432 + - DATABASE_USER=${DATABASE_USER} + - DATABASE_PASSWORD=${DATABASE_PASSWORD} + - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB} + - WORKSPACE_ROOT=${WORKSPACE_ROOT} + - CONFIG_ROOT=${CONFIG_ROOT} + - TRACKING_STRATEGY=${TRACKING_STRATEGY} + - AIRBYTE_VERSION=${VERSION} + - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} + - TEMPORAL_HOST=${TEMPORAL_HOST} + ports: + - 7001:8001 + volumes: + - workspace:${WORKSPACE_ROOT} + - data:${CONFIG_ROOT} +volumes: + workspace: + name: ${WORKSPACE_DOCKER_MOUNT} + data: + name: ${DATA_DOCKER_MOUNT} + db: + name: ${DB_DOCKER_MOUNT} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-second-run.yaml b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-second-run.yaml new file mode 100644 index 0000000000000..e6ff83a55cf90 --- /dev/null +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-second-run.yaml @@ -0,0 +1,79 @@ +version: "3.7" +#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments +x-logging: &default-logging + options: + max-size: "1m" + max-file: "1" + driver: json-file +services: + # hook in case we need to add init behavior + # every root service (no depends_on) should depend on init + init: + image: airbyte/init:dev + logging: *default-logging + command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}" + environment: + - LOCAL_ROOT=${LOCAL_ROOT} + - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT} + volumes: + - ${HACK_LOCAL_ROOT_PARENT}:/local_parent + db: + image: airbyte/db:dev + logging: *default-logging + restart: unless-stopped + environment: + - POSTGRES_USER=${DATABASE_USER} + - POSTGRES_PASSWORD=${DATABASE_PASSWORD} + volumes: + - db:/var/lib/postgresql/data + seed: + image: airbyte/seed:${VERSION} + # Pre-populate the volume if it is empty. + # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container + volumes: + - data:/latest_seeds + server: + image: airbyte/server:dev + logging: *default-logging + restart: unless-stopped + environment: + - WEBAPP_URL=${WEBAPP_URL} + - WAIT_BEFORE_HOSTS=5 + - WAIT_HOSTS_TIMEOUT=45 + - WAIT_HOSTS=db:5432 + - DATABASE_USER=${DATABASE_USER} + - DATABASE_PASSWORD=${DATABASE_PASSWORD} + - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB} + - WORKSPACE_ROOT=${WORKSPACE_ROOT} + - CONFIG_ROOT=${CONFIG_ROOT} + - TRACKING_STRATEGY=${TRACKING_STRATEGY} + - AIRBYTE_VERSION=${VERSION} + - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} + - TEMPORAL_HOST=${TEMPORAL_HOST} + ports: + - 7001:8001 + volumes: + - workspace:${WORKSPACE_ROOT} + - data:${CONFIG_ROOT} + airbyte-temporal: + image: temporalio/auto-setup:1.7.0 + logging: *default-logging + restart: unless-stopped + ports: + - 7233:7233 + environment: + - DB=postgresql + - DB_PORT=5432 + - POSTGRES_USER=${DATABASE_USER} + - POSTGRES_PWD=${DATABASE_PASSWORD} + - POSTGRES_SEEDS=db + - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml + volumes: + - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig +volumes: + workspace: + name: ${WORKSPACE_DOCKER_MOUNT} + data: + name: ${DATA_DOCKER_MOUNT} + db: + name: ${DB_DOCKER_MOUNT} diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index 567d95ef7aac4..646fad68e7734 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -18,8 +18,8 @@ services: seed: image: airbyte/seed:${VERSION} build: - dockerfile: Dockerfile - context: airbyte-config/init + dockerfile: seed.Dockerfile + context: airbyte-server labels: io.airbyte.git-revision: ${GIT_REVISION} scheduler: diff --git a/docker-compose.yaml b/docker-compose.yaml index fdff06895d090..49b4c1297a7f3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: # Pre-populate the volume if it is empty. # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container volumes: - - data:/app/seed + - data:/latest_seeds scheduler: image: airbyte/scheduler:${VERSION} logging: *default-logging @@ -67,7 +67,7 @@ services: - ${LOCAL_ROOT}:${LOCAL_ROOT} - data:${CONFIG_ROOT} server: - image: airbyte/server:${VERSION} + image: airbyte/server:dev logging: *default-logging container_name: airbyte-server restart: unless-stopped diff --git a/docs/contributing-to-airbyte/building-new-connector/README.md b/docs/contributing-to-airbyte/building-new-connector/README.md index cc6620f8aa8c6..6ca4309131d94 100644 --- a/docs/contributing-to-airbyte/building-new-connector/README.md +++ b/docs/contributing-to-airbyte/building-new-connector/README.md @@ -92,11 +92,11 @@ Once you've finished iterating on the changes to a connector as specified in its 1. Bump the version in the `Dockerfile` of the connector \(`LABEL io.airbyte.version=X.X.X`\). 2. Update the connector definition in the Airbyte connector index to use the new version: - * `airbyte-config/init/src/main/resources/seed/source_definitions.yaml` if it is a source - * `airbyte-config/init/src/main/resources/seed/destination_definitions.yaml` if it is a destination. + * `airbyte-server/src/main/resources/seed/source_definitions.yaml` if it is a source + * `airbyte-server/src/main/resources/seed/destination_definitions.yaml` if it is a destination. 3. Update the connector JSON definition. To find the appropriate JSON file to update, find a JSON file `.json` where the UUID portion is the ID specified in the YAML file you modified in step 2. The relevant directories are: - * `airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/.json` for sources - * `airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/.json` for destinations + * `airbyte-server/src/main/resources/config/STANDARD_SOURCE_DEFINITION/.json` for sources + * `airbyte-server/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/.json` for destinations 4. Submit a PR containing the changes you made. 5. One of Airbyte maintainers will review the change and publish the new version of the connector to Docker hub. Triggering tests and publishing connectors can be done by leaving a comment on the PR with the following format \(the PR must be from the Airbyte repo, not a fork\): diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 147399f024d2c..2294a38b11dc6 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -124,7 +124,7 @@ VERSION=dev docker-compose up ### Resetting the Airbyte developer environment -Sometimes you'll want to reset the data in your local environment. One common case for this is if you are updating an connector's entry in the database \(`airbyte-config/init/src/main/resources/config`\), often the easiest thing to do, is wipe the local database and start it from scratch. To reset your data back to a clean install of Airbyte, follow these steps: +Sometimes you'll want to reset the data in your local environment. One common case for this is if you are updating an connector's entry in the database \(`airbyte-server/src/main/resources/config`\), often the easiest thing to do, is wipe the local database and start it from scratch. To reset your data back to a clean install of Airbyte, follow these steps: * Delete the datastore volumes in docker diff --git a/docs/contributing-to-airbyte/tutorials/building-a-python-source.md b/docs/contributing-to-airbyte/tutorials/building-a-python-source.md index 007fba2825fc2..9df6b8e22d597 100644 --- a/docs/contributing-to-airbyte/tutorials/building-a-python-source.md +++ b/docs/contributing-to-airbyte/tutorials/building-a-python-source.md @@ -26,7 +26,7 @@ All the commands below assume that `python` points to a version of python >3. * Step 8: Set up Standard Tests * Step 9: Write unit tests or integration tests * Step 10: Update the `README.md` \(If API credentials are required to run the integration, please document how they can be obtained or link to a how-to guide.\) -* Step 11: Add the connector to the API/UI \(by adding an entry in `airbyte-config/init/src/main/resources/seed/source_definitions.yaml`\) +* Step 11: Add the connector to the API/UI \(by adding an entry in `airbyte-server/src/main/resources/seed/source_definitions.yaml`\) * Step 12: Add docs \(in `docs/integrations/sources/.md`\) {% hint style="info" %} @@ -201,7 +201,7 @@ The template fills in most of the information for the readme for you. Unless the #### Step 11: Add the connector to the API/UI -Open the following file: `airbyte-config/init/src/main/resources/seed/source_definitions.yaml`. You'll find a list of all the connectors that Airbyte displays in the UI. Pattern match to add your own connector. Make sure to generate a new _unique_ UUIDv4 for the `sourceDefinitionId` field. You can get one [here](https://www.uuidgenerator.net/). +Open the following file: `airbyte-server/src/main/resources/seed/source_definitions.yaml`. You'll find a list of all the connectors that Airbyte displays in the UI. Pattern match to add your own connector. Make sure to generate a new _unique_ UUIDv4 for the `sourceDefinitionId` field. You can get one [here](https://www.uuidgenerator.net/). Note that for simple and quick testing use cases, you can also do this step [using the UI](../../integrations/custom-connectors.md#adding-your-connectors-in-the-ui). diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index fcb88d7d20a69..257581c7f647d 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -20,19 +20,27 @@ If you inadvertently upgrade to a version of Airbyte that is not compatible with docker-compose down ``` -2. Turn back on the Airbyte web app, server, and db \(using the "old" working version from which you are upgrading from\). +2. Upgrade the docker instance to new version. + + i. If you are running Airbyte from a cloned version of the Airbyte GitHub repo and want to use the current most recent stable version, just `git pull`. + + ii. If you are running Airbyte from downloaded `docker-compose.yaml` and `.env` files without a GitHub repo, run `wget -N https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml}` to pull the latest versions and overwrite both files. + +3. Bring Airbyte back online. ```bash - docker-compose up -d db server webapp + docker-compose up ``` - _Note: the `-d` flag in the previous command is optional and is meant to run the docker services in the background. This would let you free to keep interacting with the terminal and type the next commands in the tutorial._ +## Upgrading \(K8s\) + +Airbyte Kubernetes **does not** support automatic migration. Please follow the following steps to upgrade your Airbyte Kubernetes deployment. -3. Switching over to your browser, navigate to the Admin page in the UI. Then go to the Configuration Tab. Click Export. This will download a compressed back-up archive \(gzipped tarball\) of all of your Airbyte configuration data and sync history locally. +1. Switching over to your browser, navigate to the Admin page in the UI. Then go to the Configuration Tab. Click Export. This will download a compressed back-up archive \(gzipped tarball\) of all of your Airbyte configuration data and sync history locally. _Note: Any secrets that you have entered into Airbyte will be in this archive, so you should treat it as secret._ -4. Back to the terminal, migrate the local archive to the new version using the Migration App \(packaged in a docker container\). +2. Back to the terminal, migrate the local archive to the new version using the Migration App \(packaged in a docker container\). ```bash docker run --rm -v :/config airbyte/migration: --\ @@ -49,52 +57,6 @@ If you inadvertently upgrade to a version of Airbyte that is not compatible with --output /config/airbyte_archive_migrated.tar.gz ``` -5. Turn off Airbyte fully and **\(see warning\)** delete the existing Airbyte docker volumes. - - _WARNING: Make sure you have already exported your data \(step 3\). This command is going to delete your data in Docker, you may lose your airbyte configurations!_ - - This is where all airbyte configurations are saved. Those configuration files need to be upgraded and restored with the proper version in the following steps. - - ```bash - # Careful, this is deleting data! - docker-compose down -v - ``` - - The `-v` flag is equivalent to running the following two steps in a single command: - - ```bash - # Turn off Airbyte fully - docker-compose down - # Delete the existing Airbyte docker volumes where all airbyte configurations are saved. - docker volume rm $(docker volume ls -q | grep airbyte) - ``` - - 5.b. (Skip if you are using default database `airbyte-db`) If you [deployed an external database](configuring-airbyte-db.md) to be used by Airbyte, you might need to drop all tables from the database and re-initialize them from scratch using the latest updated [schema script](https://github.com/airbytehq/airbyte/blob/master/airbyte-db/src/main/resources/schema.sql) (replace "airbyte" and "docker" with the appropriate values). If no changes were made to the SQL tables since the last Airbyte version you were using, you can safely skip this step. - -6. Upgrade the docker instance to new version. - - i. If you are running Airbyte from a cloned version of the Airbyte GitHub repo and want to use the current most recent stable version, just `git pull`. - - ii. If you are running Airbyte from downloaded `docker-compose.yaml` and `.env` files without a GitHub repo, run `wget -N https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml}` to pull the latest versions and overwrite both files. - -7. Bring Airbyte back online. - - ```bash - docker-compose up - ``` - -8. Complete Preferences section. In the subsequent setup page click "Skip Onboarding". Navigate to the Admin page in the UI. Then go to the Configuration Tab. Click Import. This will prompt you to upload the migrated archive to Airbyte. After this completes, your upgraded Airbyte instance will now be running with all of your original configuration. - -This step will throw an exception if the data you are trying to upload does not match the version of Airbyte that is running. - -## Upgrading \(K8s\) - -This process is similar to the Docker upgrade process with several changes to account for the Kubernetes resources. - -1. Follow **Step 3** in the [Docker upgrade process](#Upgrading-\(Docker\)) to export your current configurations as an archive. - -2. Follow **Step 4**. in the [Docker upgrade process](#Upgrading-\(Docker\)) process to run the migration on your archive. - 3. Turn off Airbyte fully and **\(see warning\)** delete the existing Airbyte Kubernetes volumes. _WARNING: Make sure you have already exported your data \(step 1\). This command is going to delete your data in Kubernetes, you may lose your airbyte configurations!_ @@ -105,7 +67,7 @@ This process is similar to the Docker upgrade process with several changes to ac # Careful, this is deleting data! kubectl delete -k kube/overlays/stable ``` -4. Follow **Step 6** in the [Docker upgrade process](#Upgrading-\(Docker\)) to check out the most recent version of Airbyte. Although it is possible to +4. Follow **Step 2** in the [Docker upgrade process](#Upgrading-\(Docker\)) to check out the most recent version of Airbyte. Although it is possible to migrate by changing the `.env` file in the kube overlay directory, this is not recommended as it does not capture any changes to the Kubernetes manifests. 5. Bring Airbyte back up. diff --git a/docs/troubleshooting/new-connection.md b/docs/troubleshooting/new-connection.md index 69d309be0d769..5a410899b75f0 100644 --- a/docs/troubleshooting/new-connection.md +++ b/docs/troubleshooting/new-connection.md @@ -15,7 +15,7 @@ To load configuration parameters, Airbyte must first `docker pull` the connector One workaround is to manually pull the latest version of every connector you'll use then resetting Airbyte. Note that this will remove any configured connections, sources, or destinations you currently have in Airbyte. To do this: 1. Decide which connectors you'd like to use. For this example let's say you want the Postgres source and the Snowflake destination. -2. Find the Docker image name of those connectors. Look [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/source_definitions.yaml) for sources and [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml) for destinations. For each of the connectors you'd like to use, copy the value of the `dockerRepository` and `dockerImageTag` fields. For example, for the Postgres source this would be `airbyte/source-postgres` and e.g `0.1.6`. +2. Find the Docker image name of those connectors. Look [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-server/src/main/resources/seed/source_definitions.yaml) for sources and [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-server/src/main/resources/seed/destination_definitions.yaml) for destinations. For each of the connectors you'd like to use, copy the value of the `dockerRepository` and `dockerImageTag` fields. For example, for the Postgres source this would be `airbyte/source-postgres` and e.g `0.1.6`. 3. For **each of the connectors** you'd like to use, from your shell run `docker pull :`, replacing `` and `` with the values copied from the step above e.g: `docker pull airbyte/source-postgres:0.1.6`. 4. Once you've finished downloading all the images, from the Airbyte repository root run `docker-compose down -v` followed by `docker-compose up`. 5. The issue should be resolved. diff --git a/docs/troubleshooting/running-sync.md b/docs/troubleshooting/running-sync.md index 1dc4a40fcff16..3917c146ad6a0 100644 --- a/docs/troubleshooting/running-sync.md +++ b/docs/troubleshooting/running-sync.md @@ -5,7 +5,7 @@ Several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) -* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-server/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). @@ -16,7 +16,7 @@ Our current version of incremental is [append](../understanding-airbyte/connecti If this is true, then, there are still several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) -* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-server/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). @@ -27,6 +27,6 @@ Several things to check: * What is the name of the table you are looking at in the destination? Let's make sure you're not looking at a temporary table. * **Is the basic normalization toggle set to true at the connection settings?** If it's false, you won't see columns but most probably a JSON file. So you need to switch it on true, and try again. * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) -* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-server/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). \ No newline at end of file diff --git a/kube/resources/scheduler.yaml b/kube/resources/scheduler.yaml index 79fd1ea380b80..669e57c1e91f2 100644 --- a/kube/resources/scheduler.yaml +++ b/kube/resources/scheduler.yaml @@ -154,7 +154,7 @@ spec: args: [ "-c", - "mkdir -p /configs/config && yes n | cp -r -i /app/seed/config /configs", + "mkdir -p /configs/config && yes n | cp -r -i /latest_seeds/config /configs", ] volumeMounts: - name: airbyte-volume-configs diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh index d3e1bc1de5a60..c7bdb84655188 100755 --- a/tools/bin/check_images_exist.sh +++ b/tools/bin/check_images_exist.sh @@ -12,7 +12,7 @@ echo "Checking core images exist..." docker-compose pull || exit 1 echo "Checking integration images exist..." -CONFIG_FILES=$(find airbyte-config/init | grep json | grep -v STANDARD_WORKSPACE | grep -v build) +CONFIG_FILES=$(find airbyte-server | grep json | grep -v STANDARD_WORKSPACE | grep -v build | grep -v test) [ -z "$CONFIG_FILES" ] && echo "ERROR: Could not find any config files." && exit 1 while IFS= read -r file; do