[test](mtmv)Add mtmv up and down with drop table test case

zfr9527 · zfr9527 · commit 8569f6545fba · 2025-03-05T10:40:49.000+08:00
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -2057,12 +2057,45 @@ class Suite implements GroovyInterceptable {
         }
     }
 
+    def get_follower_ip = {
+        def result = sql """show frontends;"""
+        logger.info("result:" + result)
+        for (int i = 0; i < result.size(); i++) {
+            if (result[i][7] == "FOLLOWER" && result[i][8] == "false" && result[i][11] == "true") {
+                return result[i][1]
+            }
+        }
+        return "null"
+    }
+
+    def get_follower_conn = { def explainSelectSql ->
+        def switch_ip = get_follower_ip()
+        if (switch_ip != "null") {
+            def tokens = context.config.jdbcUrl.split('/')
+            def url_tmp = tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?"
+            def new_jdbc_url = url_tmp.replaceAll(/\/\/[0-9.]+:/, "//${switch_ip}:")
+            logger.info("new_jdbc_url: " + new_jdbc_url)
+
+            def user = context.config.jdbcUser
+            def password = context.config.jdbcPassword
+            def explainRes = connect(user, password, new_jdbc_url) {
+                sql explainSelectSql
+            }
+            return explainRes
+        }
+        logger.info("This cluster not exists follower node")
+        return null
+    }
+
     // mv part in rewrite process, rewrte success and chosen by cbo
     // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite
     // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not
     // if true, check if chosen by cbo or doesn't check
+    // The default parameter of curConn is 0, which means there is no limitation on the FE role for the connection;
+    // when it is 1, it indicates that the operation is to be executed by connecting to the master; when it is 2,
+    // it means the operation is to be executed by connecting to a follower.
     void mv_rewrite_success(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(),
-                               is_partition_statistics_ready = true) {
+                               is_partition_statistics_ready = true, curConn = 0) {
         logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name + ", sync_cbo_rewrite = " +sync_cbo_rewrite
                 + ", is_partition_statistics_ready = " + is_partition_statistics_ready)
         if (!is_partition_statistics_ready) {
@@ -2077,10 +2110,23 @@ class Suite implements GroovyInterceptable {
             }
             return
         }
-        explain {
-            sql(" memo plan ${query_sql}")
-            contains("${mv_name} chose")
+        if (curConn == 1) {
+            explain {
+                master_sql(" memo plan ${query_sql}")
+                contains("${mv_name} chose")
+            }
+        } else if (curConn == 2) {
+            explain {
+                get_follower_conn(" memo plan ${query_sql}")
+                contains("${mv_name} chose")
+            }
+        } else {
+            explain {
+                sql(" memo plan ${query_sql}")
+                contains("${mv_name} chose")
+            }
         }
+
     }
 
     // multi mv part in rewrite process, all rewrte success and chosen by cbo
@@ -2250,8 +2296,11 @@ class Suite implements GroovyInterceptable {
 
     // multi mv part in rewrite process, rewrte success without check if chosen by cbo or not
     // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite
+    // The default parameter of curConn is 0, which means there is no limitation on the FE role for the connection;
+    // when it is 1, it indicates that the operation is to be executed by connecting to the master; when it is 2,
+    // it means the operation is to be executed by connecting to a follower.
     void mv_rewrite_success_without_check_chosen(query_sql, mv_name,
-                                                 sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) {
+                                                 sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), curConn = 0) {
         logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name)
         if (!sync_cbo_rewrite) {
             explain {
@@ -2263,10 +2312,26 @@ class Suite implements GroovyInterceptable {
             }
             return
         }
-        explain {
-            sql(" memo plan ${query_sql}")
-            check { result ->
-                result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
+        if (curConn == 1) {
+            explain {
+                master_sql(" memo plan ${query_sql}")
+                check { result ->
+                    result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
+                }
+            }
+        } else if (curConn == 2) {
+            explain {
+                get_follower_conn(" memo plan ${query_sql}")
+                check { result ->
+                    result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
+                }
+            }
+        } else {
+            explain {
+                sql(" memo plan ${query_sql}")
+                check { result ->
+                    result.contains("${mv_name} chose") || result.contains("${mv_name} not chose")
+                }
             }
         }
     }
diff --git a/regression-test/suites/mtmv_up_down_olap_p0/load.groovy b/regression-test/suites/mtmv_up_down_olap_p0/load.groovy
@@ -15,8 +15,26 @@
 // specific language governing permissions and limitations
 // under the License.
 
+/*
+Build the basic information. When conceiving upgrade and downgrade test cases, you should firmly grasp the object
+representation of the function in the FE meta.
+Taking MTMV as an example, the main function points are creation, refresh, and rewriting, and the involved entities
+are the base table and MTMV.
+1.When creating an MTMV, check if the rewriting meets the expectations.
+2.When refreshing an MTMV, check if the rewriting meets the expectations.
+3.When deleting an MTMV, check if the rewriting meets the expectations.
+4.When deleting a base table, check if the rewriting meets the expectations; then trigger a refresh and check if the
+rewriting meets the expectations.
+5.When deleting a partition of a base table, check if the rewriting meets the expectations; then trigger a refresh and
+check if the rewriting meets the expectations.
+6.Design a slightly more complex scenario. For example: Build an MTMV with two base tables. When deleting one of the
+base tables, check if the refresh of the MTMV meets the expectations and if the rewriting meets the expectations;
+create an MTMV with the undeleted base table and check if it can be created and refreshed normally, and if the
+corresponding rewriting meets the expectations.
+ */
 suite("test_upgrade_downgrade_prepare_olap_mtmv","p0,mtmv,restart_fe") {
     String suiteName = "mtmv_up_down_olap"
+    String dbName = context.config.getDbNameByFile(context.file)
     String mvName = "${suiteName}_mtmv"
     String tableName = "${suiteName}_table"
     String tableName2 = "${suiteName}_table2"
@@ -68,4 +86,123 @@ suite("test_upgrade_downgrade_prepare_olap_mtmv","p0,mtmv,restart_fe") {
             SELECT a.* FROM ${tableName} a inner join ${tableName2} b on a.user_id=b.user_id;
     """
     waitingMTMVTaskFinishedByMvName(mvName)
+
+
+    String dropTableName1 = """${suiteName}_DropTableName1"""
+    String dropTableName2 = """${suiteName}_DropTableName2"""
+    String dropTableName4 = """${suiteName}_DropTableName4"""
+    String dropMtmvName1 = """${suiteName}_dropMtmvName1"""
+    String dropMtmvName2 = """${suiteName}_dropMtmvName2"""
+    String dropMtmvName3 = """${suiteName}_dropMtmvName3"""
+
+    sql """drop materialized view if exists ${dropMtmvName1};"""
+    sql """drop materialized view if exists ${dropMtmvName2};"""
+    sql """drop materialized view if exists ${dropMtmvName3};"""
+    sql """drop table if exists `${dropTableName1}`"""
+    sql """drop table if exists `${dropTableName2}`"""
+    sql """drop table if exists `${dropTableName4}`"""
+
+
+    sql """
+        CREATE TABLE `${dropTableName1}` (
+          `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+          `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`user_id`, `date`, `num`)
+        COMMENT 'OLAP'
+        PARTITION BY RANGE(`date`)
+        (PARTITION p201701_1000 VALUES [('0000-01-01'), ('2017-02-01')),
+        PARTITION p201702_2000 VALUES [('2017-02-01'), ('2017-03-01')),
+        PARTITION p201703_3000 VALUES [('2017-03-01'), ('2017-04-01')),
+        PARTITION p201704_4000 VALUES [('2017-04-01'), ('2017-05-01')),
+        PARTITION p201705_5000 VALUES [('2017-05-01'), ('2017-06-01')),
+        PARTITION p201706_6000 VALUES [('2017-06-01'), ('2017-07-01')),
+        PARTITION p201707_7000 VALUES [('2017-07-01'), ('2017-08-01')),
+        PARTITION p201708_8000 VALUES [('2017-08-01'), ('2017-09-01')),
+        PARTITION p201709_9000 VALUES [('2017-09-01'), ('2017-10-01')),
+        PARTITION p201710_1000 VALUES [('2017-10-01'), ('2017-11-01')),
+        PARTITION p201711_1100 VALUES [('2017-11-01'), ('2017-12-01')),
+        PARTITION p201712_1200 VALUES [('2017-12-01'), ('2018-01-01')))
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+        PROPERTIES ('replication_num' = '1') ;
+        """
+    sql """
+        insert into ${dropTableName1} values(1,"2017-01-15",1),(2,"2017-02-15",2),(3,"2017-03-15",3),(4,"2017-04-15",4),(5,"2017-05-15",5),(6,"2017-06-15",6),(7,"2017-07-15",7),(8,"2017-08-15",8),(9,"2017-09-15",9),(10,"2017-10-15",10),(11,"2017-11-15",11),(12,"2017-12-15",12);
+        """
+
+    sql """
+        CREATE TABLE `${dropTableName2}` (
+          `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+          `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`user_id`, `date`, `num`)
+        COMMENT 'OLAP'
+        PARTITION BY RANGE(`date`)
+        (PARTITION p201701_1000 VALUES [('0000-01-01'), ('2017-02-01')),
+        PARTITION p201702_2000 VALUES [('2017-02-01'), ('2017-03-01')),
+        PARTITION p201703_3000 VALUES [('2017-03-01'), ('2017-04-01')),
+        PARTITION p201704_4000 VALUES [('2017-04-01'), ('2017-05-01')),
+        PARTITION p201705_5000 VALUES [('2017-05-01'), ('2017-06-01')),
+        PARTITION p201706_6000 VALUES [('2017-06-01'), ('2017-07-01')),
+        PARTITION p201707_7000 VALUES [('2017-07-01'), ('2017-08-01')),
+        PARTITION p201708_8000 VALUES [('2017-08-01'), ('2017-09-01')),
+        PARTITION p201709_9000 VALUES [('2017-09-01'), ('2017-10-01')),
+        PARTITION p201710_1000 VALUES [('2017-10-01'), ('2017-11-01')),
+        PARTITION p201711_1100 VALUES [('2017-11-01'), ('2017-12-01')),
+        PARTITION p201712_1200 VALUES [('2017-12-01'), ('2018-01-01')))
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+        PROPERTIES ('replication_num' = '1') ;
+        """
+    sql """
+        insert into ${dropTableName2} values(1,"2017-01-15",1),(2,"2017-02-15",2),(3,"2017-03-15",3),(4,"2017-04-15",4),(5,"2017-05-15",5),(6,"2017-06-15",6),(7,"2017-07-15",7),(8,"2017-08-15",8),(9,"2017-09-15",9),(10,"2017-10-15",10),(11,"2017-11-15",11),(12,"2017-12-15",12);
+        """
+
+    sql """
+        CREATE TABLE `${dropTableName4}` (
+          `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+          `age` SMALLINT NOT NULL COMMENT '\"年龄\"'
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`user_id`, `age`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+        PROPERTIES ('replication_num' = '1') ;
+        """
+    sql """
+        insert into ${dropTableName4} values(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10),(11,11),(12,12);
+        """
+
+
+    sql """
+        CREATE MATERIALIZED VIEW ${dropMtmvName1}
+            REFRESH AUTO ON MANUAL
+            partition by(`date`)
+            DISTRIBUTED BY RANDOM BUCKETS 2
+            PROPERTIES ('replication_num' = '1')
+            AS
+            SELECT a.* FROM ${dropTableName1} a inner join ${dropTableName4} b on a.user_id=b.user_id;
+    """
+    waitingMTMVTaskFinishedByMvName(dropMtmvName1)
+
+    sql """
+        CREATE MATERIALIZED VIEW ${dropMtmvName2}
+            REFRESH AUTO ON MANUAL
+            partition by(`date`)
+            DISTRIBUTED BY RANDOM BUCKETS 2
+            PROPERTIES ('replication_num' = '1')
+            AS
+            SELECT a.* FROM ${dropTableName2} a inner join ${dropTableName4} b on a.user_id=b.user_id;
+    """
+    waitingMTMVTaskFinishedByMvName(dropMtmvName2)
+
+    def state_mtmv1 = sql """select State,RefreshState,SyncWithBaseTables from mv_infos('database'='${dbName}') where Name = '${dropMtmvName1}';"""
+    assertTrue(state_mtmv1[0][0] == "NORMAL")
+    assertTrue(state_mtmv1[0][1] == "SUCCESS")
+    assertTrue(state_mtmv1[0][2] == true)
+    def state_mtmv2 = sql """select State,RefreshState,SyncWithBaseTables from mv_infos('database'='${dbName}') where Name = '${dropMtmvName2}';"""
+    assertTrue(state_mtmv2[0][0] == "NORMAL")
+    assertTrue(state_mtmv2[0][1] == "SUCCESS")
+    assertTrue(state_mtmv2[0][2] == true)
+
 }
diff --git a/regression-test/suites/mtmv_up_down_olap_p0/test_upgrade_downgrade_olap_mtmv.groovy b/regression-test/suites/mtmv_up_down_olap_p0/test_upgrade_downgrade_olap_mtmv.groovy
@@ -20,13 +20,95 @@ suite("test_upgrade_downgrade_olap_mtmv","p0,mtmv,restart_fe") {
     String dbName = context.config.getDbNameByFile(context.file)
     String mvName = "${suiteName}_mtmv"
     String tableName = "${suiteName}_table"
+
     // test data is normal
     order_qt_refresh_init "SELECT * FROM ${mvName}"
     // test is sync
     order_qt_mtmv_sync "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
-     sql """
+    sql """
             REFRESH MATERIALIZED VIEW ${mvName} complete
         """
     // test can refresh success
     waitingMTMVTaskFinishedByMvName(mvName)
+
+    String dropTableName1 = """${suiteName}_DropTableName1"""
+    String dropTableName2 = """${suiteName}_DropTableName2"""
+    String dropTableName4 = """${suiteName}_DropTableName4"""
+    String dropMtmvName1 = """${suiteName}_dropMtmvName1"""
+    String dropMtmvName2 = """${suiteName}_dropMtmvName2"""
+    String dropMtmvName3 = """${suiteName}_dropMtmvName3"""
+
+    // drop table
+    sql """drop table if exists ${dropTableName1}"""
+    def state_mtmv1 = sql """select State,RefreshState,SyncWithBaseTables from mv_infos('database'='${dbName}') where Name = '${dropMtmvName1}';"""
+    assertTrue(state_mtmv1[0][0] == "SCHEMA_CHANGE")
+    assertTrue(state_mtmv1[0][1] == "SUCCESS" || state_mtmv1[0][1] == "INIT")
+    assertTrue(state_mtmv1[0][2] == false)
+
+    // drop partition
+    def parts_res = sql """show partitions from ${dropTableName2}"""
+    sql """ALTER TABLE ${dropTableName2} DROP PARTITION ${parts_res[0][1]};"""
+    def state_mtmv2 = sql """select State,RefreshState,SyncWithBaseTables from mv_infos('database'='${dbName}') where Name = '${dropMtmvName2}';"""
+    assertTrue(state_mtmv2[0][0] == "NORMAL")
+    assertTrue(state_mtmv2[0][1] == "SUCCESS")
+    def mtmv_part_res = sql """show partitions from ${dropMtmvName2}"""
+    logger.info("mtmv_part_res[0][18]: " + mtmv_part_res[0][18])
+    logger.info("mtmv_part_res[0][19]: " + mtmv_part_res[0][19])
+    logger.info("mtmv_part_res:" + mtmv_part_res)
+    def part_1 = mtmv_part_res.size()
+    def diff_part = 0
+    for (int i = 0; i < mtmv_part_res.size(); i++) {
+        if (mtmv_part_res[i][18] == "false" && mtmv_part_res[i][19] as String == "[${dropTableName2}]") {
+            diff_part = diff_part + 1
+        }
+    }
+
+    def sql2 = "SELECT a.* FROM ${dropTableName2} a inner join ${dropTableName4} b on a.user_id=b.user_id;"
+    mv_rewrite_success(sql2, dropMtmvName2)
+
+    // An error occurred when refreshing the partition individually, and the partition was not deleted after the refresh.
+    try {
+        sql """refresh MATERIALIZED VIEW ${dropMtmvName2} partition(${mtmv_part_res[0][1]})"""
+    } catch (Exception e) {
+        logger.info(e.getMessage())
+    }
+
+    // When refreshing the entire MTMV, the partition will be deleted.
+    sql """refresh MATERIALIZED VIEW ${dropMtmvName2} auto"""
+    waitingMTMVTaskFinishedByMvName(dropMtmvName2)
+    mtmv_part_res = sql """show partitions from ${dropMtmvName2}"""
+    logger.info("mtmv_part_res:" + mtmv_part_res)
+    def part_2 = mtmv_part_res.size()
+    assertTrue(part_1 == part_2 + diff_part)
+
+    state_mtmv2 = sql """select State,RefreshState,SyncWithBaseTables from mv_infos('database'='${dbName}') where Name = '${dropMtmvName2}';"""
+    logger.info("state_mtmv2:" + state_mtmv2)
+    assertTrue(state_mtmv2[0][0] == "NORMAL")
+    assertTrue(state_mtmv2[0][1] == "SUCCESS")
+    assertTrue(state_mtmv2[0][2] == true)
+    mv_rewrite_success(sql2, dropMtmvName2)
+    def follower_ip = get_follower_ip()
+    if (follower_ip != "null") {
+        mv_rewrite_success(sql2, dropMtmvName2,
+                enable_sync_mv_cost_based_rewrite(), is_partition_statistics_ready(dbName, [dropTableName2, dropTableName4]), 2)
+    }
+
+
+    // After deleting the table, you can create a new MTMV
+    def cur_dropMtmvName3 = dropMtmvName3 + UUID.randomUUID().toString().replaceAll("-", "")
+    sql """
+        CREATE MATERIALIZED VIEW ${cur_dropMtmvName3}
+            REFRESH AUTO ON MANUAL
+            DISTRIBUTED BY RANDOM BUCKETS 2
+            PROPERTIES ('replication_num' = '1')
+            AS
+            SELECT user_id, age FROM ${dropTableName4};
+        """
+    waitingMTMVTaskFinishedByMvName(cur_dropMtmvName3)
+    mv_rewrite_success_without_check_chosen("""SELECT user_id FROM ${dropTableName4}""", cur_dropMtmvName3)
+    if (follower_ip != "null") {
+        mv_rewrite_success_without_check_chosen("""SELECT user_id FROM ${dropTableName4}""", cur_dropMtmvName3,
+                enable_sync_mv_cost_based_rewrite(), 2)
+    }
+
 }
