From 183e0ef31d716e020f1304f995fff3fcf6ecf1bc Mon Sep 17 00:00:00 2001 From: subodh Date: Thu, 8 Jul 2021 15:50:01 +0530 Subject: [PATCH 1/3] set db version after full import is complete --- .../java/io/airbyte/server/ConfigDumpImport.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java index 94bca93f3b0ba..96bf779091fd1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java @@ -117,11 +117,7 @@ public ImportRead importData(File archive) { // 1. Unzip source Archives.extractArchive(archive.toPath(), sourceRoot); - // 2. Set DB version - LOGGER.info("Setting the DB Airbyte version to : " + targetVersion); - postgresPersistence.setVersion(targetVersion); - - // 3. dry run + // 2. dry run try { checkImport(sourceRoot); } catch (Exception e) { @@ -130,11 +126,15 @@ public ImportRead importData(File archive) { throw e; } - // 4. Import Postgres content + // 3. Import Postgres content importDatabaseFromArchive(sourceRoot, targetVersion); - // 5. Import Configs + // 4. Import Configs importConfigsFromArchive(sourceRoot, false); + + // 5. Set DB version + LOGGER.info("Setting the DB Airbyte version to : " + targetVersion); + postgresPersistence.setVersion(targetVersion); result = new ImportRead().status(StatusEnum.SUCCEEDED); } finally { FileUtils.deleteDirectory(sourceRoot.toFile()); @@ -164,7 +164,6 @@ private void checkImport(Path tempFolder) throws IOException, JsonValidationExce "Please upgrade your Airbyte Archive, see more at https://docs.airbyte.io/tutorials/upgrading-airbyte\n", importVersion, targetVersion)); } - checkDBVersion(targetVersion); importConfigsFromArchive(tempFolder, true); } From f6ffdf741416ecec91d7ab110a1b7956e30b0eef Mon Sep 17 00:00:00 2001 From: subodh Date: Thu, 8 Jul 2021 16:12:55 +0530 Subject: [PATCH 2/3] check db version in the last step --- .../src/main/java/io/airbyte/server/ConfigDumpImport.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java index 96bf779091fd1..2eaee3bfd9857 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImport.java @@ -135,6 +135,9 @@ public ImportRead importData(File archive) { // 5. Set DB version LOGGER.info("Setting the DB Airbyte version to : " + targetVersion); postgresPersistence.setVersion(targetVersion); + + // 6. check db version + checkDBVersion(targetVersion); result = new ImportRead().status(StatusEnum.SUCCEEDED); } finally { FileUtils.deleteDirectory(sourceRoot.toFile()); From 59bc1975d44d9b7220682dcd0c64a19989d99cb5 Mon Sep 17 00:00:00 2001 From: subodh Date: Thu, 8 Jul 2021 21:42:30 +0530 Subject: [PATCH 3/3] add comment --- .../java/io/airbyte/scheduler/persistence/JobPersistence.java | 4 ++++ airbyte-server/src/main/java/io/airbyte/server/ServerApp.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java index 068c871e1ab3f..420f8fd566c74 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java @@ -38,6 +38,10 @@ import java.util.UUID; import java.util.stream.Stream; +/** + * TODO Introduce a locking mechanism so that no DB operation is allowed when automatic migration is + * running + */ public interface JobPersistence { Job getJob(long jobId) throws IOException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index c1dd7b69dc481..ef68ad34271aa 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -240,6 +240,10 @@ public static void main(String[] args) throws Exception { } } + /** + * Ideally when automatic migration runs, we should make sure that we acquire a lock on database and + * no other operation is allowed + */ private static void runAutomaticMigration(ConfigRepository configRepository, JobPersistence jobPersistence, String airbyteVersion,