Skip to content

[enhance](cloud)add log for compaction read time #51355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+48 −0
Open

[enhance](cloud)add log for compaction read time #51355

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ba744de
[enhance](cloud)add log for compaction read time
koarz May 29, 2025
6a85f3a
1
koarz May 29, 2025
ae97809
fix
koarz May 30, 2025
631fe2f
fix
koarz Jun 3, 2025
4cb5c79
using us
koarz Jun 3, 2025
6456b92
Merge branch 'master' into cloud_io_metric
koarz Jun 4, 2025
93eb822
Merge branch 'master' into cloud_io_metric
koarz Jun 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions be/src/cloud/cloud_base_compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,8 @@ Status CloudBaseCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
.tag("local_read_time_us", _stats.cloud_local_read_time)
.tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);

Expand Down
2 changes: 2 additions & 0 deletions be/src/cloud/cloud_cumulative_compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ Status CloudCumulativeCompaction::execute_compact() {
.tag("cumulative_point", cloud_tablet()->cumulative_layer_point())
.tag("num_rowsets", cloud_tablet()->fetch_add_approximate_num_rowsets(0))
.tag("cumu_num_rowsets", cloud_tablet()->fetch_add_approximate_cumu_num_rowsets(0))
.tag("local_read_time_us", _stats.cloud_local_read_time)
.tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);

Expand Down
2 changes: 2 additions & 0 deletions be/src/cloud/cloud_full_compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,8 @@ Status CloudFullCompaction::execute_compact() {
.tag("output_rowset_data_size", _output_rowset->data_disk_size())
.tag("output_rowset_index_size", _output_rowset->index_disk_size())
.tag("output_rowset_total_size", _output_rowset->total_disk_size())
.tag("local_read_time_us", _stats.cloud_local_read_time)
.tag("remote_read_time_us", _stats.cloud_remote_read_time)
.tag("local_read_bytes", _local_read_bytes_total)
.tag("remote_read_bytes", _remote_read_bytes_total);

Expand Down
16 changes: 16 additions & 0 deletions be/src/cloud/cloud_tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "cloud/cloud_tablet.h"

#include <gen_cpp/Types_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <rapidjson/document.h>
#include <rapidjson/encodings.h>
Expand Down Expand Up @@ -733,6 +734,21 @@ void CloudTablet::get_compaction_status(std::string* json_result) {
cast_set<uint>(_last_full_compaction_status.length()),
root.GetAllocator());
root.AddMember("last full status", full_compaction_status_value, root.GetAllocator());
rapidjson::Value exec_compaction_time;
std::string num_str {std::to_string(exec_compaction_time_us.load())};
exec_compaction_time.SetString(num_str.c_str(), cast_set<uint>(num_str.length()),
root.GetAllocator());
root.AddMember("exec compaction time us", exec_compaction_time, root.GetAllocator());
rapidjson::Value local_read_time;
num_str = std::to_string(local_read_time_us.load());
local_read_time.SetString(num_str.c_str(), cast_set<uint>(num_str.length()),
root.GetAllocator());
root.AddMember("compaction local read time us", local_read_time, root.GetAllocator());
rapidjson::Value remote_read_time;
num_str = std::to_string(remote_read_time_us.load());
remote_read_time.SetString(num_str.c_str(), cast_set<uint>(num_str.length()),
root.GetAllocator());
root.AddMember("compaction remote read time us", remote_read_time, root.GetAllocator());

// print all rowsets' version as an array
rapidjson::Document versions_arr;
Expand Down
4 changes: 4 additions & 0 deletions be/src/cloud/cloud_tablet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,10 @@ class CloudTablet final : public BaseTablet {
int64_t last_cumu_no_suitable_version_ms = 0;
int64_t last_access_time_ms = 0;

std::atomic<int64_t> local_read_time_us = 0;
std::atomic<int64_t> remote_read_time_us = 0;
std::atomic<int64_t> exec_compaction_time_us = 0;

// Return merged extended schema
TabletSchemaSPtr merged_tablet_schema() const override;

Expand Down
7 changes: 7 additions & 0 deletions be/src/olap/compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@

#include "cloud/cloud_meta_mgr.h"
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
#include "common/config.h"
#include "common/status.h"
#include "cpp/sync_point.h"
Expand Down Expand Up @@ -1498,6 +1499,12 @@ Status CloudCompactionMixin::execute_compact_impl(int64_t permits) {
// 4. modify rowsets in memory
RETURN_IF_ERROR(modify_rowsets());

// update compaction status data
auto tablet = std::static_pointer_cast<CloudTablet>(_tablet);
tablet->local_read_time_us.fetch_add(_stats.cloud_local_read_time);
tablet->remote_read_time_us.fetch_add(_stats.cloud_remote_read_time);
tablet->exec_compaction_time_us.fetch_add(watch.get_elapse_time_us());

return Status::OK();
}

Expand Down
13 changes: 13 additions & 0 deletions be/src/olap/merger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include <utility>
#include <vector>

#include "cloud/config.h"
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
Expand Down Expand Up @@ -147,6 +148,12 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type,
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total = reader.stats().file_cache_stats.bytes_write_into_cache;
if (config::is_cloud_mode()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What would happen if there is no cloud mode judgment?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing will happen, but I think these calculations are not necessary.

stats_output->cloud_local_read_time =
reader.stats().file_cache_stats.local_io_timer / 1000;
stats_output->cloud_remote_read_time =
reader.stats().file_cache_stats.remote_io_timer / 1000;
}
}

RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->flush(),
Expand Down Expand Up @@ -327,6 +334,12 @@ Status Merger::vertical_compact_one_group(
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
stats_output->cached_bytes_total = reader.stats().file_cache_stats.bytes_write_into_cache;
if (config::is_cloud_mode()) {
stats_output->cloud_local_read_time =
reader.stats().file_cache_stats.local_io_timer / 1000;
stats_output->cloud_remote_read_time =
reader.stats().file_cache_stats.remote_io_timer / 1000;
}
}
RETURN_IF_ERROR(dst_rowset_writer->flush_columns(is_key));

Expand Down
2 changes: 2 additions & 0 deletions be/src/olap/merger.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ class VerticalBlockReader;
class Merger {
public:
struct Statistics {
int64_t cloud_local_read_time = 0;
int64_t cloud_remote_read_time = 0;
// number of rows written to the destination rowset after merge
int64_t output_rows = 0;
int64_t merged_rows = 0;
Expand Down
Loading