[Enhancement] Add medium type of data dir in be_tablets (#37070)

nshangyiming · mergify[bot] · commit 32c40bd03018 · 2024-01-16T11:46:56.000Z
Add storage medium type in be_tablets table so that we can debug some balancing issue. Signed-off-by: Dejun Xia <xiadejun@starrocks.com> (cherry picked from commit cceb194) # Conflicts: # be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp # be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.h # fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java # fe/fe-core/src/test/resources/sql/scheduler/schema_scan.sql
diff --git a/be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp b/be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
@@ -15,7 +15,12 @@
 #include "exec/vectorized/schema_scanner/schema_be_tablets_scanner.h"
 
 #include "agent/master_info.h"
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
 #include "exec/vectorized/schema_scanner/schema_helper.h"
+=======
+#include "exec/schema_scanner/schema_helper.h"
+#include "gen_cpp/Types_types.h" // for TStorageMedium::type
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
 #include "gutil/strings/substitute.h"
 #include "runtime/string_value.h"
 #include "storage/storage_engine.h"
@@ -27,6 +32,7 @@ using starrocks::vectorized::fill_column_with_slot;
 namespace starrocks::vectorized {
 
 SchemaScanner::ColumnDesc SchemaBeTabletsScanner::_s_columns[] = {
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
         {"BE_ID", TYPE_BIGINT, sizeof(int64_t), false},         {"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), false},
         {"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), false},  {"TABLET_ID", TYPE_BIGINT, sizeof(int64_t), false},
         {"NUM_VERSION", TYPE_BIGINT, sizeof(int64_t), false},   {"MAX_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
@@ -36,6 +42,28 @@ SchemaScanner::ColumnDesc SchemaBeTabletsScanner::_s_columns[] = {
         {"STATE", TYPE_VARCHAR, sizeof(StringValue), false},    {"TYPE", TYPE_VARCHAR, sizeof(StringValue), false},
         {"DATA_DIR", TYPE_VARCHAR, sizeof(StringValue), false}, {"SHARD_ID", TYPE_BIGINT, sizeof(int64_t), false},
         {"SCHEMA_HASH", TYPE_BIGINT, sizeof(int64_t), false},
+=======
+        {"BE_ID", TYPE_BIGINT, sizeof(int64_t), false},
+        {"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), false},
+        {"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), false},
+        {"TABLET_ID", TYPE_BIGINT, sizeof(int64_t), false},
+        {"NUM_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
+        {"MAX_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
+        {"MIN_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
+        {"NUM_ROWSET", TYPE_BIGINT, sizeof(int64_t), false},
+        {"NUM_ROW", TYPE_BIGINT, sizeof(int64_t), false},
+        {"DATA_SIZE", TYPE_BIGINT, sizeof(int64_t), false},
+        {"INDEX_MEM", TYPE_BIGINT, sizeof(int64_t), false},
+        {"CREATE_TIME", TYPE_BIGINT, sizeof(int64_t), false},
+        {"STATE", TYPE_VARCHAR, sizeof(StringValue), false},
+        {"TYPE", TYPE_VARCHAR, sizeof(StringValue), false},
+        {"DATA_DIR", TYPE_VARCHAR, sizeof(StringValue), false},
+        {"SHARD_ID", TYPE_BIGINT, sizeof(int64_t), false},
+        {"SCHEMA_HASH", TYPE_BIGINT, sizeof(int64_t), false},
+        {"INDEX_DISK", TYPE_BIGINT, sizeof(int64_t), false},
+        {"MEDIUM_TYPE", TYPE_VARCHAR, sizeof(StringValue), false},
+
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
 };
 
 SchemaBeTabletsScanner::SchemaBeTabletsScanner()
@@ -87,13 +115,32 @@ static const char* keys_type_to_string(KeysType type) {
     }
 }
 
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
 Status SchemaBeTabletsScanner::fill_chunk(vectorized::ChunkPtr* chunk) {
+=======
+static const char* medium_type_to_string(TStorageMedium::type type) {
+    switch (type) {
+    case TStorageMedium::SSD:
+        return "SSD";
+    case TStorageMedium::HDD:
+        return "HDD";
+    default:
+        return "UNKNOWN";
+    }
+}
+
+Status SchemaBeTabletsScanner::fill_chunk(ChunkPtr* chunk) {
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
     const auto& slot_id_to_index_map = (*chunk)->get_slot_id_to_index_map();
     auto end = _cur_idx + 1;
     for (; _cur_idx < end; _cur_idx++) {
         auto& info = _infos[_cur_idx];
         for (const auto& [slot_id, index] : slot_id_to_index_map) {
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
             if (slot_id < 1 || slot_id > 17) {
+=======
+            if (slot_id < 1 || slot_id > 19) {
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
                 return Status::InternalError(strings::Substitute("invalid slot id:$0", slot_id));
             }
             vectorized::ColumnPtr column = (*chunk)->get_column_by_slot_id(slot_id);
@@ -139,7 +186,7 @@ Status SchemaBeTabletsScanner::fill_chunk(vectorized::ChunkPtr* chunk) {
                 break;
             }
             case 9: {
-                // num rowt
+                // num rows
                 fill_column_with_slot<TYPE_BIGINT>(column.get(), (void*)&info.num_row);
                 break;
             }
@@ -171,8 +218,14 @@ Status SchemaBeTabletsScanner::fill_chunk(vectorized::ChunkPtr* chunk) {
                 break;
             }
             case 15: {
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
                 Slice type = Slice(info.data_dir);
                 fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&type);
+=======
+                // DATA_DIR
+                Slice data_dir = Slice(info.data_dir);
+                fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&data_dir);
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
                 break;
             }
             case 16: {
@@ -183,6 +236,20 @@ Status SchemaBeTabletsScanner::fill_chunk(vectorized::ChunkPtr* chunk) {
                 fill_column_with_slot<TYPE_BIGINT>(column.get(), (void*)&info.schema_hash);
                 break;
             }
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.cpp
+=======
+            case 18: {
+                // INDEX_DISK
+                fill_column_with_slot<TYPE_BIGINT>(column.get(), (void*)&info.index_disk_usage);
+                break;
+            }
+            case 19: {
+                // medium type
+                Slice medium_type = Slice(medium_type_to_string(info.medium_type));
+                fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&medium_type);
+                break;
+            }
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
             default:
                 break;
             }
diff --git a/be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.h b/be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.h
@@ -18,6 +18,7 @@
 
 #include "exec/vectorized/schema_scanner.h"
 #include "gen_cpp/FrontendService_types.h"
+#include "gen_cpp/Types_types.h" // for TStorageMedium::type
 
 namespace starrocks {
 
@@ -38,6 +39,11 @@ struct TabletBasicInfo {
     std::string data_dir;
     int64_t shard_id{0};
     int64_t schema_hash{0};
+<<<<<<< HEAD:be/src/exec/vectorized/schema_scanner/schema_be_tablets_scanner.h
+=======
+    int64_t index_disk_usage{0};
+    TStorageMedium::type medium_type;
+>>>>>>> cceb1943b0 ([Enhancement] Add medium type of data dir in be_tablets (#37070)):be/src/exec/schema_scanner/schema_be_tablets_scanner.h
 };
 
 namespace vectorized {
diff --git a/be/src/storage/tablet.cpp b/be/src/storage/tablet.cpp
@@ -1283,6 +1283,7 @@ void Tablet::get_basic_info(TabletBasicInfo& info) {
     info.data_dir = data_dir()->path();
     info.shard_id = shard_id();
     info.schema_hash = schema_hash();
+    info.medium_type = data_dir()->storage_medium();
     if (_updates != nullptr) {
         _updates->get_basic_info_extra(info);
     } else {
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java
@@ -0,0 +1,53 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.starrocks.catalog.system.information;
+
+import com.starrocks.catalog.PrimitiveType;
+import com.starrocks.catalog.ScalarType;
+import com.starrocks.catalog.Table;
+import com.starrocks.catalog.system.SystemId;
+import com.starrocks.catalog.system.SystemTable;
+import com.starrocks.thrift.TSchemaTableType;
+
+import static com.starrocks.catalog.system.SystemTable.NAME_CHAR_LEN;
+import static com.starrocks.catalog.system.SystemTable.builder;
+
+public class BeTabletsSystemTable {
+    public static SystemTable create() {
+        return new SystemTable(SystemId.BE_TABLETS_ID,
+                "be_tablets",
+                Table.TableType.SCHEMA,
+                builder()
+                        .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("TABLE_ID", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("PARTITION_ID", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("TABLET_ID", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("NUM_VERSION", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("MAX_VERSION", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("MIN_VERSION", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("NUM_ROWSET", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("NUM_ROW", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("DATA_SIZE", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("INDEX_MEM", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("CREATE_TIME", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("STATE", ScalarType.createVarchar(NAME_CHAR_LEN))
+                        .column("TYPE", ScalarType.createVarchar(NAME_CHAR_LEN))
+                        .column("DATA_DIR", ScalarType.createVarchar(NAME_CHAR_LEN))
+                        .column("SHARD_ID", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("SCHEMA_HASH", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("INDEX_DISK", ScalarType.createType(PrimitiveType.BIGINT))
+                        .column("MEDIUM_TYPE", ScalarType.createVarchar(NAME_CHAR_LEN))
+                        .build(), TSchemaTableType.SCH_BE_TABLETS);
+    }
+}
diff --git a/fe/fe-core/src/test/resources/sql/scheduler/schema_scan.sql b/fe/fe-core/src/test/resources/sql/scheduler/schema_scan.sql
@@ -0,0 +1,70 @@
+[sql]
+select * from information_schema.session_variables
+[scheduler]
+PLAN FRAGMENT 0(F00)
+  DOP: 16
+  INSTANCES
+    INSTANCE(0-F00#0)
+      BE: 10001
+
+[fragment]
+PLAN FRAGMENT 0
+ OUTPUT EXPRS:1: VARIABLE_NAME | 2: VARIABLE_VALUE
+  PARTITION: UNPARTITIONED
+
+  RESULT SINK
+
+  0:SCAN SCHEMA
+[end]
+
+[sql]
+select * from information_schema.be_tablets
+[scheduler]
+PLAN FRAGMENT 0(F01)
+  DOP: 16
+  INSTANCES
+    INSTANCE(0-F01#0)
+      BE: 10002
+
+PLAN FRAGMENT 1(F00)
+  DOP: 16
+  INSTANCES
+    INSTANCE(1-F00#0)
+      DESTINATIONS: 0-F01#0
+      BE: 10001
+      SCAN RANGES
+        0:SCAN SCHEMA
+          1. <PLACEHOLDER>
+    INSTANCE(2-F00#1)
+      DESTINATIONS: 0-F01#0
+      BE: 10002
+      SCAN RANGES
+        0:SCAN SCHEMA
+          1. <PLACEHOLDER>
+    INSTANCE(3-F00#2)
+      DESTINATIONS: 0-F01#0
+      BE: 10003
+      SCAN RANGES
+        0:SCAN SCHEMA
+          1. <PLACEHOLDER>
+
+[fragment]
+PLAN FRAGMENT 0
+ OUTPUT EXPRS:1: BE_ID | 2: TABLE_ID | 3: PARTITION_ID | 4: TABLET_ID | 5: NUM_VERSION | 6: MAX_VERSION | 7: MIN_VERSION | 8: NUM_ROWSET | 9: NUM_ROW | 10: DATA_SIZE | 11: INDEX_MEM | 12: CREATE_TIME | 13: STATE | 14: TYPE | 15: DATA_DIR | 16: SHARD_ID | 17: SCHEMA_HASH | 18: INDEX_DISK | 19: MEDIUM_TYPE
+  PARTITION: UNPARTITIONED
+
+  RESULT SINK
+
+  1:EXCHANGE
+
+PLAN FRAGMENT 1
+ OUTPUT EXPRS:
+  PARTITION: RANDOM
+
+  STREAM DATA SINK
+    EXCHANGE ID: 01
+    UNPARTITIONED
+
+  0:SCAN SCHEMA
+[end]
+
