Remove v0.30.0 file-based migration (#5984)

* Add warning in migration readme

* Revert yaml file update

* Remove resource files for v0.30.0 migration

* Revert "Revert yaml file update"

This reverts commit 4cc0905.

* Remove v0.30.0 migration

* Update unit test

* Check for null

Author: tuliren

airbyte-config/models/src/main/resources/types/Notification.yaml

@@ -6,8 +6,6 @@ description: Notification Settings
 type: object
 required:
   - notificationType
-  - sendOnSuccess
-  - sendOnFailure
 additionalProperties: false
 properties:
   # Instead of this type field, we would prefer a json schema "oneOf" but unfortunately,

airbyte-migration/README.md

@@ -2,6 +2,8 @@
 
 This module migrates configs specified in `airbyte-config` to new versions.
 
+WARNING: the file-based migrations are deprecated. Please write a Flyway migration whenever you want to update the database. See [here](../airbyte-db/lib/README.md) for details.
+
 ## Change Airbyte Configs
 - Update the config json schema in [`airbyte-config/models`](../airbyte-config/models).
 - Add the changed json schema to the [main resources](./src/main/resources/migrations).

airbyte-migration/src/main/java/io/airbyte/migrate/Migrations.java

@@ -37,7 +37,6 @@
 import io.airbyte.migrate.migrations.MigrationV0_27_0;
 import io.airbyte.migrate.migrations.MigrationV0_28_0;
 import io.airbyte.migrate.migrations.MigrationV0_29_0;
-import io.airbyte.migrate.migrations.MigrationV0_30_0;
 import io.airbyte.migrate.migrations.NoOpMigration;
 import java.util.List;
 
@@ -59,8 +58,7 @@ public class Migrations {
   private static final Migration MIGRATION_V_0_26_0 = new MigrationV0_26_0(MIGRATION_V_0_25_0);
   private static final Migration MIGRATION_V_0_27_0 = new MigrationV0_27_0(MIGRATION_V_0_26_0);
   public static final Migration MIGRATION_V_0_28_0 = new MigrationV0_28_0(MIGRATION_V_0_27_0);
-  private static final Migration MIGRATION_V_0_29_0 = new MigrationV0_29_0(MIGRATION_V_0_28_0);
-  private static final Migration MIGRATION_V_0_30_0 = new MigrationV0_30_0(MIGRATION_V_0_29_0);
+  public static final Migration MIGRATION_V_0_29_0 = new MigrationV0_29_0(MIGRATION_V_0_28_0);
 
   // all migrations must be added to the list in the order that they should be applied.
   public static final List<Migration> MIGRATIONS = ImmutableList.of(
@@ -80,7 +78,6 @@ public class Migrations {
       MIGRATION_V_0_26_0,
       MIGRATION_V_0_27_0,
       MIGRATION_V_0_28_0,
-      MIGRATION_V_0_29_0,
-      MIGRATION_V_0_30_0);
+      MIGRATION_V_0_29_0);
 
 }

airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_30_0.java

@@ -26,86 +26,18 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import io.airbyte.commons.enums.Enums;
-import java.nio.file.Path;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.stream.Collectors;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 public class MigrationCurrentSchemaTest {
 
-  // get all of the "current" configs (in other words the one airbyte-config). get all of the configs
-  // from the output schema of the last migration. make sure they match.
+  /**
+   * The file-based migration is deprecated. We need to ensure that v0.29.0 is the last one. All new
+   * migrations should be written in Flyway.
+   */
   @Test
-  void testConfigsOfLastMigrationMatchSource() {
-    final Map<ResourceId, JsonNode> lastMigrationSchema = getSchemaOfLastMigration(ResourceType.CONFIG);
-    final Map<ResourceId, JsonNode> currentSchema = MigrationUtils.getNameToSchemasFromResourcePath(
-        Path.of("types"),
-        ResourceType.CONFIG,
-        Enums.valuesAsStrings(ConfigKeys.class));
-
-    assertSameSchemas(currentSchema, lastMigrationSchema);
-  }
-
-  private static Map<ResourceId, JsonNode> getSchemaOfLastMigration(ResourceType resourceType) {
+  public void testLastMigration() {
     final Migration lastMigration = Migrations.MIGRATIONS.get(Migrations.MIGRATIONS.size() - 1);
-    final Map<ResourceId, JsonNode> lastMigrationOutputSchema = lastMigration.getOutputSchema();
-
-    return lastMigrationOutputSchema.entrySet()
-        .stream()
-        .filter(entry -> entry.getKey().getType() == resourceType)
-        .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
-  }
-
-  // get all of the "current" jobs (in other words the one airbyte-db). get all of the configs
-  // from the output schema of the last migration. make sure they match.
-  @Test
-  @Disabled
-  // TODO(#5902): Liren will adapt this to the new migration system.
-  void testJobsOfLastMigrationMatchSource() {
-    final Map<ResourceId, JsonNode> lastMigrationSchema = getSchemaOfLastMigration(ResourceType.JOB);
-    final Map<ResourceId, JsonNode> currentSchema = MigrationUtils.getNameToSchemasFromResourcePath(
-        Path.of("jobs_database"),
-        ResourceType.JOB,
-        Enums.valuesAsStrings(JobKeys.class));
-
-    assertSameSchemas(currentSchema, lastMigrationSchema);
-  }
-
-  private static void assertSameSchemas(Map<ResourceId, JsonNode> currentSchemas, Map<ResourceId, JsonNode> lastMigrationSchema) {
-    assertEquals(currentSchemas.size(), lastMigrationSchema.size());
-
-    final List<Entry<ResourceId, JsonNode>> lastMigrationOutputSchemaCleanedSorted = lastMigrationSchema.entrySet()
-        .stream()
-        .sorted(Comparator.comparing((v) -> v.getKey().getType()))
-        .collect(Collectors.toList());
-
-    // break out element-wise assertion so it is easier to read any failed tests.
-    for (Map.Entry<ResourceId, JsonNode> lastMigrationEntry : lastMigrationOutputSchemaCleanedSorted) {
-      assertEquals(currentSchemas.get(lastMigrationEntry.getKey()), lastMigrationEntry.getValue());
-    }
-  }
-
-  public enum ConfigKeys {
-    STANDARD_WORKSPACE,
-    STANDARD_SOURCE_DEFINITION,
-    STANDARD_DESTINATION_DEFINITION,
-    SOURCE_CONNECTION,
-    DESTINATION_CONNECTION,
-    STANDARD_SYNC,
-    STANDARD_SYNC_SCHEDULE,
-    STANDARD_SYNC_OPERATION,
-  }
-
-  public enum JobKeys {
-    JOBS,
-    ATTEMPTS,
-    AIRBYTE_METADATA
+    assertEquals(Migrations.MIGRATION_V_0_29_0.getVersion(), lastMigration.getVersion());
   }
 
 }