Skip to content

[Enhancement] Add medium type of data dir in be_tablets #37070

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 16, 2024
Merged

[Enhancement] Add medium type of data dir in be_tablets #37070

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a5b977
[Enhancement] Add medium type of data dir in be_tablets
nshangyiming Dec 14, 2023
52f58a5
fix ut
nshangyiming Dec 15, 2023
4fa9525
fix format
nshangyiming Dec 15, 2023
339119e
fix ut
nshangyiming Dec 15, 2023
38797ea
Update schema_be_tablets_scanner.cpp
nshangyiming Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 42 additions & 14 deletions be/src/exec/schema_scanner/schema_be_tablets_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,36 @@

#include "agent/master_info.h"
#include "exec/schema_scanner/schema_helper.h"
#include "gen_cpp/Types_types.h" // for TStorageMedium::type
#include "gutil/strings/substitute.h"
#include "runtime/string_value.h"
#include "storage/storage_engine.h"
#include "storage/tablet.h"
#include "storage/tablet_manager.h"
#include "types/logical_type.h"

namespace starrocks {

SchemaScanner::ColumnDesc SchemaBeTabletsScanner::_s_columns[] = {
{"BE_ID", TYPE_BIGINT, sizeof(int64_t), false}, {"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), false}, {"TABLET_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"NUM_VERSION", TYPE_BIGINT, sizeof(int64_t), false}, {"MAX_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
{"MIN_VERSION", TYPE_BIGINT, sizeof(int64_t), false}, {"NUM_ROWSET", TYPE_BIGINT, sizeof(int64_t), false},
{"NUM_ROW", TYPE_BIGINT, sizeof(int64_t), false}, {"DATA_SIZE", TYPE_BIGINT, sizeof(int64_t), false},
{"INDEX_MEM", TYPE_BIGINT, sizeof(int64_t), false}, {"CREATE_TIME", TYPE_BIGINT, sizeof(int64_t), false},
{"STATE", TYPE_VARCHAR, sizeof(StringValue), false}, {"TYPE", TYPE_VARCHAR, sizeof(StringValue), false},
{"DATA_DIR", TYPE_VARCHAR, sizeof(StringValue), false}, {"SHARD_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"SCHEMA_HASH", TYPE_BIGINT, sizeof(int64_t), false}, {"INDEX_DISK", TYPE_BIGINT, sizeof(int64_t), false},
{"BE_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"TABLET_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"NUM_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
{"MAX_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
{"MIN_VERSION", TYPE_BIGINT, sizeof(int64_t), false},
{"NUM_ROWSET", TYPE_BIGINT, sizeof(int64_t), false},
{"NUM_ROW", TYPE_BIGINT, sizeof(int64_t), false},
{"DATA_SIZE", TYPE_BIGINT, sizeof(int64_t), false},
{"INDEX_MEM", TYPE_BIGINT, sizeof(int64_t), false},
{"CREATE_TIME", TYPE_BIGINT, sizeof(int64_t), false},
{"STATE", TYPE_VARCHAR, sizeof(StringValue), false},
{"TYPE", TYPE_VARCHAR, sizeof(StringValue), false},
{"DATA_DIR", TYPE_VARCHAR, sizeof(StringValue), false},
{"SHARD_ID", TYPE_BIGINT, sizeof(int64_t), false},
{"SCHEMA_HASH", TYPE_BIGINT, sizeof(int64_t), false},
{"INDEX_DISK", TYPE_BIGINT, sizeof(int64_t), false},
{"MEDIUM_TYPE", TYPE_VARCHAR, sizeof(StringValue), false},

};

SchemaBeTabletsScanner::SchemaBeTabletsScanner()
Expand Down Expand Up @@ -86,13 +97,24 @@ static const char* keys_type_to_string(KeysType type) {
}
}

static const char* medium_type_to_string(TStorageMedium::type type) {
switch (type) {
case TStorageMedium::SSD:
return "SSD";
case TStorageMedium::HDD:
return "HDD";
default:
return "UNKNOWN";
}
}

Status SchemaBeTabletsScanner::fill_chunk(ChunkPtr* chunk) {
const auto& slot_id_to_index_map = (*chunk)->get_slot_id_to_index_map();
auto end = _cur_idx + 1;
for (; _cur_idx < end; _cur_idx++) {
auto& info = _infos[_cur_idx];
for (const auto& [slot_id, index] : slot_id_to_index_map) {
if (slot_id < 1 || slot_id > 18) {
if (slot_id < 1 || slot_id > 19) {
return Status::InternalError(strings::Substitute("invalid slot id:$0", slot_id));
}
ColumnPtr column = (*chunk)->get_column_by_slot_id(slot_id);
Expand Down Expand Up @@ -138,7 +160,7 @@ Status SchemaBeTabletsScanner::fill_chunk(ChunkPtr* chunk) {
break;
}
case 9: {
// num rowt
// num rows
fill_column_with_slot<TYPE_BIGINT>(column.get(), (void*)&info.num_row);
break;
}
Expand Down Expand Up @@ -171,8 +193,8 @@ Status SchemaBeTabletsScanner::fill_chunk(ChunkPtr* chunk) {
}
case 15: {
// DATA_DIR
Slice type = Slice(info.data_dir);
fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&type);
Slice data_dir = Slice(info.data_dir);
fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&data_dir);
break;
}
case 16: {
Expand All @@ -190,6 +212,12 @@ Status SchemaBeTabletsScanner::fill_chunk(ChunkPtr* chunk) {
fill_column_with_slot<TYPE_BIGINT>(column.get(), (void*)&info.index_disk_usage);
break;
}
case 19: {
// medium type
Slice medium_type = Slice(medium_type_to_string(info.medium_type));
fill_column_with_slot<TYPE_VARCHAR>(column.get(), (void*)&medium_type);
break;
}
default:
break;
}
Expand Down
2 changes: 2 additions & 0 deletions be/src/exec/schema_scanner/schema_be_tablets_scanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include "exec/schema_scanner.h"
#include "gen_cpp/FrontendService_types.h"
#include "gen_cpp/Types_types.h" // for TStorageMedium::type

namespace starrocks {

Expand All @@ -39,6 +40,7 @@ struct TabletBasicInfo {
int64_t shard_id{0};
int64_t schema_hash{0};
int64_t index_disk_usage{0};
TStorageMedium::type medium_type;
};

class SchemaBeTabletsScanner : public SchemaScanner {
Expand Down
1 change: 1 addition & 0 deletions be/src/storage/tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1637,6 +1637,7 @@ void Tablet::get_basic_info(TabletBasicInfo& info) {
info.data_dir = data_dir()->path();
info.shard_id = shard_id();
info.schema_hash = schema_hash();
info.medium_type = data_dir()->storage_medium();
if (_updates != nullptr) {
_updates->get_basic_info_extra(info);
} else {
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public static SystemTable create() {
.column("SHARD_ID", ScalarType.createType(PrimitiveType.BIGINT))
.column("SCHEMA_HASH", ScalarType.createType(PrimitiveType.BIGINT))
.column("INDEX_DISK", ScalarType.createType(PrimitiveType.BIGINT))
.column("MEDIUM_TYPE", ScalarType.createVarchar(NAME_CHAR_LEN))
.build(), TSchemaTableType.SCH_BE_TABLETS);
}
}
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/resources/sql/scheduler/schema_scan.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ PLAN FRAGMENT 1(F00)

[fragment]
PLAN FRAGMENT 0
OUTPUT EXPRS:1: BE_ID | 2: TABLE_ID | 3: PARTITION_ID | 4: TABLET_ID | 5: NUM_VERSION | 6: MAX_VERSION | 7: MIN_VERSION | 8: NUM_ROWSET | 9: NUM_ROW | 10: DATA_SIZE | 11: INDEX_MEM | 12: CREATE_TIME | 13: STATE | 14: TYPE | 15: DATA_DIR | 16: SHARD_ID | 17: SCHEMA_HASH | 18: INDEX_DISK
OUTPUT EXPRS:1: BE_ID | 2: TABLE_ID | 3: PARTITION_ID | 4: TABLET_ID | 5: NUM_VERSION | 6: MAX_VERSION | 7: MIN_VERSION | 8: NUM_ROWSET | 9: NUM_ROW | 10: DATA_SIZE | 11: INDEX_MEM | 12: CREATE_TIME | 13: STATE | 14: TYPE | 15: DATA_DIR | 16: SHARD_ID | 17: SCHEMA_HASH | 18: INDEX_DISK | 19: MEDIUM_TYPE
PARTITION: UNPARTITIONED

RESULT SINK
Expand Down