From 19cf2bca1ef27981c475dc3f7eb1fc9ce6cae1a0 Mon Sep 17 00:00:00 2001 From: Haijun Yu Date: Sun, 8 Dec 2024 10:09:00 +0800 Subject: [PATCH] [feat][store] Add backup feature details. --- CMakeLists.txt | 3 + dingo-store-proto | 2 +- src/br/backup.cc | 1210 +++++++++++++++++ src/br/backup.h | 129 ++ src/br/backup_data.cc | 306 +++++ src/br/backup_data.h | 77 ++ src/br/backup_data_base.cc | 327 +++++ src/br/backup_data_base.h | 111 ++ src/br/backup_meta.cc | 283 ++++ src/br/backup_meta.h | 84 ++ src/br/backup_meta_base.cc | 319 +++++ src/br/backup_meta_base.h | 101 ++ src/br/backup_sdk_data.cc | 264 ++++ src/br/backup_sdk_data.h | 59 + src/br/backup_sdk_meta.cc | 122 ++ src/br/backup_sdk_meta.h | 59 + src/br/backup_sql_data.cc | 275 ++++ src/br/backup_sql_data.h | 64 + src/br/backup_sql_meta.cc | 73 + src/br/backup_sql_meta.h | 54 + src/br/br.cc | 23 - src/br/br.h | 29 - src/br/helper.cc | 41 + src/br/helper.h | 579 +------- src/br/interaction_manager.cc | 197 +-- src/br/interaction_manager.h | 145 +- src/br/interation.cc | 34 + src/br/interation.h | 27 +- src/br/main.cc | 662 ++++++++- src/br/parameter.cc | 55 + src/br/parameter.h | 70 + src/br/router.cc | 18 +- src/br/router.h | 4 +- src/br/sst_file_writer.cc | 48 + src/br/sst_file_writer.h | 49 + src/br/utils.cc | 264 ++++ src/br/utils.h | 44 + src/common/constant.h | 44 + src/common/helper.cc | 163 +++ src/common/helper.h | 9 + src/coordinator/coordinator_control.cc | 88 ++ src/coordinator/coordinator_control.h | 56 +- src/coordinator/coordinator_control_coor.cc | 168 ++- src/coordinator/coordinator_control_fsm.cc | 25 +- src/coordinator/coordinator_control_meta.cc | 6 + src/engine/rocks_raw_engine.cc | 21 + src/engine/rocks_raw_engine.h | 1 + src/engine/storage.cc | 48 + src/engine/storage.h | 16 + src/engine/txn_engine_helper.cc | 747 +++++++++- src/engine/txn_engine_helper.h | 94 ++ src/server/coordinator_service.cc | 177 ++- src/server/coordinator_service.h | 12 + src/server/document_service.cc | 106 +- src/server/document_service.h | 4 + src/server/index_service.cc | 141 ++ src/server/index_service.h | 7 + src/server/meta_service.cc | 241 ++++ src/server/meta_service.h | 9 + src/server/store_service.cc | 242 ++++ src/server/store_service.h | 10 + test/unit_test/CMakeLists.txt | 4 +- test/unit_test/br/CMakeLists.txt | 7 + test/unit_test/br/test_backup.cc | 107 ++ test/unit_test/br/test_backup_data.cc | 126 ++ test/unit_test/br/test_backup_meta.cc | 155 +++ test/unit_test/br/test_backup_misc.cc | 488 +++++++ test/unit_test/br/test_backup_sdk_data.cc | 150 ++ test/unit_test/br/test_backup_sdk_meta.cc | 106 ++ test/unit_test/br/test_backup_sql_data.cc | 154 +++ test/unit_test/br/test_backup_sql_meta.cc | 147 ++ test/unit_test/br/test_helper.cc | 104 ++ test/unit_test/br/test_interaction_manager.cc | 289 ++++ test/unit_test/br/test_interation.cc | 287 ++++ test/unit_test/br/test_sst_file_writer.cc | 104 ++ test/unit_test/br/test_utils.cc | 69 + 76 files changed, 10007 insertions(+), 936 deletions(-) create mode 100644 src/br/backup.cc create mode 100644 src/br/backup.h create mode 100644 src/br/backup_data.cc create mode 100644 src/br/backup_data.h create mode 100644 src/br/backup_data_base.cc create mode 100644 src/br/backup_data_base.h create mode 100644 src/br/backup_meta.cc create mode 100644 src/br/backup_meta.h create mode 100644 src/br/backup_meta_base.cc create mode 100644 src/br/backup_meta_base.h create mode 100644 src/br/backup_sdk_data.cc create mode 100644 src/br/backup_sdk_data.h create mode 100644 src/br/backup_sdk_meta.cc create mode 100644 src/br/backup_sdk_meta.h create mode 100644 src/br/backup_sql_data.cc create mode 100644 src/br/backup_sql_data.h create mode 100644 src/br/backup_sql_meta.cc create mode 100644 src/br/backup_sql_meta.h delete mode 100644 src/br/br.cc delete mode 100644 src/br/br.h create mode 100644 src/br/parameter.cc create mode 100644 src/br/parameter.h create mode 100644 src/br/sst_file_writer.cc create mode 100644 src/br/sst_file_writer.h create mode 100644 src/br/utils.cc create mode 100644 src/br/utils.h create mode 100644 test/unit_test/br/CMakeLists.txt create mode 100644 test/unit_test/br/test_backup.cc create mode 100644 test/unit_test/br/test_backup_data.cc create mode 100644 test/unit_test/br/test_backup_meta.cc create mode 100644 test/unit_test/br/test_backup_misc.cc create mode 100644 test/unit_test/br/test_backup_sdk_data.cc create mode 100644 test/unit_test/br/test_backup_sdk_meta.cc create mode 100644 test/unit_test/br/test_backup_sql_data.cc create mode 100644 test/unit_test/br/test_backup_sql_meta.cc create mode 100644 test/unit_test/br/test_helper.cc create mode 100644 test/unit_test/br/test_interaction_manager.cc create mode 100644 test/unit_test/br/test_interation.cc create mode 100644 test/unit_test/br/test_sst_file_writer.cc create mode 100644 test/unit_test/br/test_utils.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e58f49516..5e682ed2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -719,6 +719,9 @@ add_executable( src/mvcc/codec.cc src/vector/codec.cc src/document/codec.cc + src/common/synchronization.cc + src/common/uuid.cc + src/common/version.cc ${SERIAL_SRCS} ${VERSION_SRCS} $) diff --git a/dingo-store-proto b/dingo-store-proto index 642a60df9..7d0a6fd4f 160000 --- a/dingo-store-proto +++ b/dingo-store-proto @@ -1 +1 @@ -Subproject commit 642a60df90440ce9e61d713e0f674605698af4ba +Subproject commit 7d0a6fd4f811fd80707f3a775ebf997fb7c2ccfb diff --git a/src/br/backup.cc b/src/br/backup.cc new file mode 100644 index 000000000..466612f7f --- /dev/null +++ b/src/br/backup.cc @@ -0,0 +1,1210 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup.h" + +#include +#include +#include +#include + +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/sst_file_writer.h" +#include "br/utils.h" +#include "common/constant.h" +#include "common/helper.h" +#include "common/logging.h" +#include "common/synchronization.h" +#include "common/uuid.h" +#include "common/version.h" +#include "fmt/core.h" +#include "proto/coordinator.pb.h" +#include "proto/error.pb.h" + +namespace br { + +#ifndef ENABLE_BACKUP_PTHREAD +#define ENABLE_BACKUP_PTHREAD +#endif + +// #undef ENABLE_BACKUP_PTHREAD + +Backup::Backup(const BackupParams& params) + : is_gc_stop_(false), + is_gc_enable_after_finish_(false), + is_need_exit_(false), + is_already_register_backup_to_coordinator_(false), + is_exit_register_backup_to_coordinator_thread_(true), + region_auto_split_enable_after_finish_(false), + region_auto_merge_enable_after_finish_(false), + balance_leader_enable_after_finish_(false), + balance_region_enable_after_finish_(false), + start_time_ms_(dingodb::Helper::TimestampMs()), + end_time_ms_(0) { + coor_url_ = params.coor_url; + br_type_ = params.br_type; + br_backup_type_ = params.br_backup_type; + backupts_ = params.backupts; + backuptso_internal_ = params.backuptso_internal; + storage_ = params.storage; + storage_internal_ = params.storage_internal; + + bthread_mutex_init(&mutex_, nullptr); +} + +Backup::~Backup() { bthread_mutex_destroy(&mutex_); }; + +std::shared_ptr Backup::GetSelf() { return shared_from_this(); } + +butil::Status Backup::Init() { + butil::Status status; + status = ParamsCheck(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // version match check + dingodb::pb::common::VersionInfo version_info_remote; + status = + GetVersionFromCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction(), version_info_remote); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + dingodb::pb::common::VersionInfo version_info_local = dingodb::GetVersionInfo(); + status = CompareVersion(version_info_local, version_info_remote); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::cout << "version compare ok" << std::endl; + DINGO_LOG(INFO) << "version compare ok"; + + status = GetGcSafePoint(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::cout << "safe point ts check ok" << std::endl; + DINGO_LOG(INFO) << "safe point ts check ok"; + + backup_task_id_ = dingodb::UUIDGenerator::GenerateUUID(); + + auto lambda_exit_function = [this, &status]() { + if (!status.ok()) { + DoFinish(); + last_error_ = status; + }; + }; + + std::cout << "backup_task_id : " << backup_task_id_ << std::endl; + DINGO_LOG(INFO) << "backup_task_id : " << backup_task_id_ << std::endl; + + dingodb::ON_SCOPE_EXIT(lambda_exit_function); + + // try to register backup task + bool is_first = true; + status = RegisterBackupToCoordinator(is_first, br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + is_already_register_backup_to_coordinator_ = true; + + std::cout << "register backup To coordinator ok" << std::endl; + DINGO_LOG(INFO) << "register backup To coordinator ok"; + + // set gc stop + if (!is_gc_stop_) { + status = SetGcStop(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + std::cout << "gc set stopped ok" << std::endl; + DINGO_LOG(INFO) << "gc set stopped ok"; + } else { + std::cout << "gc already stopped. ignore" << std::endl; + DINGO_LOG(INFO) << "gc already stopped. ignore"; + } + + status = DisableBalanceToCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (balance_leader_enable_after_finish_) { + std::cout << "balance leader set stopped ok" << std::endl; + DINGO_LOG(INFO) << "balance leader set stopped ok"; + } else { + std::cout << "balance leader already stopped. ignore" << std::endl; + DINGO_LOG(INFO) << "balance leader already stopped. ignore"; + } + + if (balance_region_enable_after_finish_) { + std::cout << "balance region set stopped ok" << std::endl; + DINGO_LOG(INFO) << "balance region set stopped ok"; + } else { + std::cout << "balance region already stopped. ignore" << std::endl; + DINGO_LOG(INFO) << "balance region already stopped. ignore"; + } + + status = DisableSplitAndMergeToStoreAndIndex(br::InteractionManager::GetInstance().GetStoreInteraction(), + br::InteractionManager::GetInstance().GetIndexInteraction()); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (region_auto_split_enable_after_finish_) { + std::cout << "region auto split set stopped ok" << std::endl; + DINGO_LOG(INFO) << "region auto split set stopped ok"; + } else { + std::cout << "region auto split already stopped. ignore" << std::endl; + DINGO_LOG(INFO) << "region auto split already stopped. ignore"; + } + + if (region_auto_merge_enable_after_finish_) { + std::cout << "region auto merge set stopped ok" << std::endl; + DINGO_LOG(INFO) << "region auto merge set stopped ok"; + } else { + std::cout << "region auto merge already stopped. ignore" << std::endl; + DINGO_LOG(INFO) << "region auto merge already stopped. ignore"; + } + + status = Utils::DirExists(storage_internal_); + if (status.ok()) { + // clean backup dir + status = Utils::ClearDir(storage_internal_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + DINGO_LOG(INFO) << "backup dir : " << storage_internal_ << " already exist. clear dir."; + } else if (dingodb::pb::error::EFILE_NOT_EXIST == status.error_code()) { + // create backup dir + status = Utils::CreateDirRecursion(storage_internal_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + DINGO_LOG(INFO) << "backup dir : " << storage_internal_ << " not exist. create recursion dir."; + } else { // error + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // create backup lock file + std::ofstream writer; + std::string lock_path = storage_internal_ + "/" + kBackupFileLock; + status = Utils::CreateFile(writer, lock_path); + if (!status.ok()) { + if (writer.is_open()) { + writer.close(); + } + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + writer << "DO NOT DELETE" << std::endl; + writer << "This file exists to remind other backup jobs won't use this path" << std::endl; + + if (writer.is_open()) { + writer.close(); + } + + DINGO_LOG(INFO) << "write backup lock file : " << lock_path << " success"; + + DINGO_LOG(INFO) << "Backup::Init " << " success"; + + return butil::Status::OK(); +} + +butil::Status Backup::Run() { + butil::Status status; + + auto lambda_exit_function = [this, &status]() { + if (!status.ok()) { + DoFinish(); + last_error_ = status; + }; + }; + + dingodb::ON_SCOPE_EXIT(lambda_exit_function); + + std::vector coordinator_addrs = + br::InteractionManager::GetInstance().GetCoordinatorInteraction()->GetAddrs(); + + // create register backup task to coordinator + { + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // register backup task to coordinator + status = DoAsyncRegisterBackupToCoordinator(coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + std::cout << "Backup task " << backup_task_id_ << " is registered to coordinator Periodicity." << std::endl; + DINGO_LOG(INFO) << "Backup task " << backup_task_id_ << " is registered to coordinator Periodicity."; + + status = DoRun(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + return butil::Status::OK(); +} + +butil::Status Backup::DoRun() { + butil::Status status; + std::vector coordinator_addrs = + br::InteractionManager::GetInstance().GetCoordinatorInteraction()->GetAddrs(); + + std::vector store_addrs = br::InteractionManager::GetInstance().GetStoreInteraction()->GetAddrs(); + + std::vector index_addrs = br::InteractionManager::GetInstance().GetIndexInteraction()->GetAddrs(); + + std::vector document_addrs = br::InteractionManager::GetInstance().GetDocumentInteraction()->GetAddrs(); + + // create backup meta + { + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr store_interaction; + status = ServerInteraction::CreateInteraction(store_addrs, store_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr index_interaction; + status = ServerInteraction::CreateInteraction(index_addrs, index_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr document_interaction; + status = ServerInteraction::CreateInteraction(document_addrs, document_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + backup_meta_ = + std::make_shared(coordinator_interaction, store_interaction, index_interaction, + document_interaction, backupts_, backuptso_internal_, storage_, storage_internal_); + } + + status = backup_meta_->Init(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::cout << "Back Meta Init ok" << std::endl; + DINGO_LOG(INFO) << "Back Meta Init ok"; + + // create backup data + { + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr store_interaction; + status = ServerInteraction::CreateInteraction(store_addrs, store_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr index_interaction; + status = ServerInteraction::CreateInteraction(index_addrs, index_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr document_interaction; + status = ServerInteraction::CreateInteraction(document_addrs, document_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + backup_data_ = + std::make_shared(coordinator_interaction, store_interaction, index_interaction, + document_interaction, backupts_, backuptso_internal_, storage_, storage_internal_); + } + + std::vector meta_region_list = backup_meta_->GetSqlMetaRegionList(); + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << "sql meta region list size : " << meta_region_list.size(); + for (const auto& region_id : meta_region_list) { + DINGO_LOG(INFO) << "sql meta region id : " << region_id; + } + } + status = backup_data_->Init(meta_region_list); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::cout << "Back Data Init ok" << std::endl; + DINGO_LOG(INFO) << "Back Data Init ok"; + + status = backup_data_->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_data_->Finish(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_meta_->Run(backup_data_->GetRegionMap()); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_meta_->Finish(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // + std::shared_ptr meta_meta = backup_meta_->GetBackupMeta(); + std::shared_ptr meta_data = backup_data_->GetBackupMeta(); + + const auto& [status2, id_epoch_type_and_value] = backup_meta_->GetIdEpochTypeAndValue(); + if (!status2.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + const auto& [status3, table_increment_group] = backup_meta_->GetAllTableIncrement(); + if (!status3.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // write backup meta + { + std::string file_name = dingodb::Constant::kBackupMetaName; + std::string file_path = storage_internal_ + "/" + file_name; + + std::map kvs; + + { kvs.emplace(meta_meta->file_name(), meta_meta->SerializeAsString()); } + + { kvs.emplace(meta_data->file_name(), meta_data->SerializeAsString()); } + + { kvs.emplace(dingodb::Constant::kIdEpochTypeAndValueKey, id_epoch_type_and_value->SerializeAsString()); } + { kvs.emplace(dingodb::Constant::kTableIncrementKey, table_increment_group->SerializeAsString()); } + + dingodb::pb::common::VersionInfo version_info = dingodb::GetVersionInfo(); + + { kvs.emplace(dingodb::Constant::kBackupVersionKey, version_info.SerializeAsString()); } + + dingodb::pb::common::BackupParam backup_param; + backup_param.set_coor_addr(coor_url_); + backup_param.set_store_addr(dingodb::Helper::VectorToString(store_addrs)); + backup_param.set_store_addr(dingodb::Helper::VectorToString(index_addrs)); + backup_param.set_store_addr(dingodb::Helper::VectorToString(document_addrs)); + backup_param.set_br_type(br_type_); + backup_param.set_br_backup_type(br_backup_type_); + backup_param.set_backupts(backupts_); + backup_param.set_backuptso_internal(backuptso_internal_); + backup_param.set_storage(storage_); + backup_param.set_storage_internal(storage_internal_); + { kvs.emplace(dingodb::Constant::kBackupBackupParamKey, backup_param.SerializeAsString()); } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // create backupmeta encryption file + std::ofstream writer; + std::string backupmeta_encryption_path = storage_internal_ + "/" + dingodb::Constant::kBackupMetaEncryptionName; + status = Utils::CreateFile(writer, backupmeta_encryption_path); + if (!status.ok()) { + if (writer.is_open()) { + writer.close(); + } + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + writer << hash_code; + + if (writer.is_open()) { + writer.close(); + } + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << dingodb::Constant::kBackupMetaName + << "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"; + DINGO_LOG(INFO) << dingodb::Constant::kBackupVersionKey << " :"; + DINGO_LOG(INFO) << version_info.DebugString() << std::endl; + + DINGO_LOG(INFO) << dingodb::Constant::kBackupBackupParamKey << " :"; + DINGO_LOG(INFO) << backup_param.DebugString() << std::endl; + + DINGO_LOG(INFO) << dingodb::Constant::kIdEpochTypeAndValueKey << " :"; + DINGO_LOG(INFO) << id_epoch_type_and_value->DebugString() << std::endl; + + DINGO_LOG(INFO) << dingodb::Constant::kTableIncrementKey << " :"; + DINGO_LOG(INFO) << table_increment_group->DebugString() << std::endl; + + DINGO_LOG(INFO) << meta_meta->file_name() << " :"; + DINGO_LOG(INFO) << meta_meta->DebugString() << std::endl; + + DINGO_LOG(INFO) << meta_data->file_name() << " :"; + DINGO_LOG(INFO) << meta_data->DebugString() << std::endl; + DINGO_LOG(INFO) << dingodb::Constant::kBackupMetaName + << "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"; + } + + // create backupmeta debug file + std::string backupmeta_debug_path = storage_internal_ + "/" + dingodb::Constant::kBackupMetaDebugName; + status = Utils::CreateFile(writer, backupmeta_debug_path); + if (!status.ok()) { + if (writer.is_open()) { + writer.close(); + } + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + writer << "This is Debug File" << std::endl; + + writer << dingodb::Constant::kBackupVersionKey << " : " << std::endl; + writer << version_info.DebugString() << std::endl; + + writer << dingodb::Constant::kBackupBackupParamKey << " : " << std::endl; + writer << backup_param.DebugString() << std::endl; + + writer << dingodb::Constant::kIdEpochTypeAndValueKey << " : " << std::endl; + writer << id_epoch_type_and_value->DebugString() << std::endl; + + writer << dingodb::Constant::kTableIncrementKey << " : " << std::endl; + writer << table_increment_group->DebugString() << std::endl; + + writer << meta_meta->file_name() << " : " << std::endl; + writer << meta_meta->DebugString() << std::endl; + + writer << meta_data->file_name() << " : " << std::endl; + writer << meta_data->DebugString() << std::endl; + + if (writer.is_open()) { + writer.close(); + } + } + + return butil::Status::OK(); +} + +butil::Status Backup::Finish() { + butil::Status status; + status = DoFinish(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + end_time_ms_ = dingodb::Helper::TimestampMs(); + + std::string s = + fmt::format("[Full Backup success summary][backup-total-ranges={}] [backup-sql-meta-ranges={}][total-take={}s]", + backup_data_->GetRegionMap()->regions_size(), backup_meta_->GetSqlMetaRegionList().size(), + (end_time_ms_ - start_time_ms_) / 1000.0); + std::cout << s << std::endl; + DINGO_LOG(INFO) << s; + return butil::Status::OK(); +} + +butil::Status Backup::ParamsCheck() { + butil::Status status; + status = ParamsCheckForStorage(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + return butil::Status::OK(); +} + +butil::Status Backup::ParamsCheckForStorage() { + butil::Status status; + status = Utils::DirExists(storage_internal_); + if (status.ok()) { + std::string lock_path = storage_internal_ + "/" + kBackupFileLock; + status = Utils::FileExistsAndRegular(lock_path); + if (status.ok()) { + std::string s = fmt::format("Backup may be running, please wait or delete lock file : {}", lock_path); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EFILE_EXIST, s); + } else if (status.error_code() != dingodb::pb::error::EFILE_NOT_EXIST) { + std::string s = fmt::format("Check lock file : {} failed: {}", lock_path, status.error_cstr()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + } else if (status.error_code() != dingodb::pb::error::EFILE_NOT_EXIST) { + std::string s = fmt::format("Check storage : {} storage_internal_ : {} failed: {}", storage_, storage_internal_, + status.error_cstr()); + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + return butil::Status::OK(); +} + +butil::Status Backup::GetGcSafePoint() { + dingodb::pb::coordinator::GetGCSafePointRequest request; + dingodb::pb::coordinator::GetGCSafePointResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_get_all_tenant(true); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "GetGCSafePoint", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to get GC safe point, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to get GC safe point, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + int64_t max_tenant_safe_points; + int64_t min_tenant_resolve_lock_safe_points; + + max_tenant_safe_points = response.safe_point(); + min_tenant_resolve_lock_safe_points = response.resolve_lock_safe_point(); + + for (const auto& [id, safe_point] : response.tenant_safe_points()) { + if (safe_point > max_tenant_safe_points) { + max_tenant_safe_points = safe_point; + } + } + + for (const auto& [id, safe_point] : response.tenant_resolve_lock_safe_points()) { + if (safe_point < min_tenant_resolve_lock_safe_points) { + min_tenant_resolve_lock_safe_points = safe_point; + } + } + + // compare safe points + if (backuptso_internal_ > max_tenant_safe_points && backuptso_internal_ <= min_tenant_resolve_lock_safe_points) { + DINGO_LOG(INFO) << "Backup safe point is " << backuptso_internal_; + } else { + std::string s = fmt::format( + "Backup safe point is {}, but max tenant safe point is {}, min tenant resolve lock safe point is {}", + backuptso_internal_, max_tenant_safe_points, min_tenant_resolve_lock_safe_points); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EILLEGAL_PARAMTETERS, s); + } + + std::string s = + fmt::format("# max tenant safe points : {} min tenant resolve lock safe points : {} backuptso(internal) : {}", + max_tenant_safe_points, min_tenant_resolve_lock_safe_points, backuptso_internal_); + std::cout << s << std::endl; + DINGO_LOG(INFO) << s; + + if (response.gc_stop()) { + is_gc_stop_ = true; + is_gc_enable_after_finish_ = false; + DINGO_LOG(INFO) << "GC is already stopped. Backup will not enable if backup is finished."; + } + + return butil::Status::OK(); +} + +butil::Status Backup::SetGcStop() { + if (is_gc_stop_) { + return butil::Status::OK(); + } + + DINGO_LOG(INFO) << "Set GC stop ..."; + + dingodb::pb::coordinator::UpdateGCSafePointRequest request; + dingodb::pb::coordinator::UpdateGCSafePointResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_gc_flag( + ::dingodb::pb::coordinator::UpdateGCSafePointRequest_GcFlagType::UpdateGCSafePointRequest_GcFlagType_GC_STOP); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "UpdateGCSafePoint", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set GC stop, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set GC stop, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + is_gc_stop_ = true; + is_gc_enable_after_finish_ = true; + + DINGO_LOG(INFO) << "GC is stopped. Backup will enable GC. if backup is finished."; + + return butil::Status::OK(); +} + +butil::Status Backup::SetGcStart() { + if (!is_gc_enable_after_finish_) { + return butil::Status::OK(); + } + DINGO_LOG(INFO) << "Set GC start ..."; + + dingodb::pb::coordinator::UpdateGCSafePointRequest request; + dingodb::pb::coordinator::UpdateGCSafePointResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_gc_flag( + ::dingodb::pb::coordinator::UpdateGCSafePointRequest_GcFlagType::UpdateGCSafePointRequest_GcFlagType_GC_START); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "UpdateGCSafePoint", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set GC stop, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set GC stop, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + is_gc_stop_ = false; + is_gc_enable_after_finish_ = false; + + DINGO_LOG(INFO) << "Set GC start success."; + + return butil::Status::OK(); +} + +butil::Status Backup::RegisterBackupToCoordinator(bool is_first, ServerInteractionPtr coordinator_interaction) { + dingodb::pb::coordinator::RegisterBackupRequest request; + dingodb::pb::coordinator::RegisterBackupResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + request.set_backup_name(backup_task_id_); + request.set_backup_path(storage_internal_); + int64_t current_now_s = dingodb::Helper::Timestamp(); + if (is_first) { + request.set_backup_start_timestamp(current_now_s); + } + request.set_backup_current_timestamp(current_now_s); + request.set_backup_timeout_s(FLAGS_backup_task_timeout_s); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "RegisterBackup", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set RegisterBackup, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set RegisterBackup, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(response.error().errcode(), s); + } + + return butil::Status::OK(); +} + +butil::Status Backup::UnregisterBackupToCoordinator(ServerInteractionPtr coordinator_interaction) { + dingodb::pb::coordinator::UnRegisterBackupRequest request; + dingodb::pb::coordinator::UnRegisterBackupResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + request.set_backup_name(backup_task_id_); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "UnRegisterBackup", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set UnRegisterBackup, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set UnRegisterBackup, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(response.error().errcode(), s); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + return butil::Status::OK(); +} + +butil::Status Backup::DoAsyncRegisterBackupToCoordinator(ServerInteractionPtr coordinator_interaction) { + std::shared_ptr self = GetSelf(); + auto lambda_call = [self, coordinator_interaction]() { + self->DoRegisterBackupToCoordinatorInternal(coordinator_interaction); + }; + +#if defined(ENABLE_BACKUP_PTHREAD) + std::thread th(lambda_call); + th.detach(); +#else + + std::function* call = new std::function; + *call = lambda_call; + bthread_t th; + + int ret = bthread_start_background( + &th, nullptr, + [](void* arg) -> void* { + auto* call = static_cast*>(arg); + (*call)(); + delete call; + return nullptr; + }, + call); + if (ret != 0) { + DINGO_LOG(ERROR) << fmt::format("bthread_start_background fail"); + return butil::Status(dingodb::pb::error::EINTERNAL, "bthread_start_background fail"); + } +#endif // #if defined(ENABLE_BACKUP_PTHREAD) + + return butil::Status::OK(); +} + +butil::Status Backup::DoRegisterBackupToCoordinatorInternal(ServerInteractionPtr coordinator_interaction) { + butil::Status status; + bool is_first = false; + is_exit_register_backup_to_coordinator_thread_ = false; + while (!is_need_exit_) { + bool is_error_occur = true; + uint32_t retry_times = FLAGS_backup_task_max_retry; + do { + status = RegisterBackupToCoordinator(is_first, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } else { // success + is_error_occur = false; + break; + } + sleep(FLAGS_backup_watch_interval_s); + } while (!is_need_exit_ && retry_times-- > 0); + + if (is_error_occur) { + if (!is_need_exit_) { + is_need_exit_ = true; + } + { + BAIDU_SCOPED_LOCK(mutex_); + last_error_ = status; + } + break; + } + + sleep(FLAGS_backup_watch_interval_s); + } + + is_exit_register_backup_to_coordinator_thread_ = true; + DINGO_LOG(INFO) << "exit register backup thread."; + + return butil::Status::OK(); +} + +butil::Status Backup::DisableBalanceToCoordinator(ServerInteractionPtr coordinator_interaction) { + dingodb::pb::coordinator::ControlConfigRequest request; + dingodb::pb::coordinator::ControlConfigResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + dingodb::pb::common::ControlConfigVariable config_balance_leader; + config_balance_leader.set_name("FLAGS_enable_balance_leader"); + config_balance_leader.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_balance_leader)); + + dingodb::pb::common::ControlConfigVariable config_balance_region; + config_balance_region.set_name("FLAGS_enable_balance_region"); + config_balance_region.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_balance_region)); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + for (const auto& config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_enable_balance_leader") { + balance_leader_enable_after_finish_ = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_enable_balance_region") { + balance_region_enable_after_finish_ = true; + } + } + + return butil::Status::OK(); +} + +butil::Status Backup::EnableBalanceToCoordinator(ServerInteractionPtr coordinator_interaction) const { + dingodb::pb::coordinator::ControlConfigRequest request; + dingodb::pb::coordinator::ControlConfigResponse response; + + if (balance_leader_enable_after_finish_) { + dingodb::pb::common::ControlConfigVariable config_balance_leader; + config_balance_leader.set_name("FLAGS_enable_balance_leader"); + config_balance_leader.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_balance_leader)); + } + + if (balance_region_enable_after_finish_) { + dingodb::pb::common::ControlConfigVariable config_balance_region; + config_balance_region.set_name("FLAGS_enable_balance_region"); + config_balance_region.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_balance_region)); + } + + if (!request.control_config_variable().empty()) { + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + } + + return butil::Status::OK(); +} + +butil::Status Backup::DisableSplitAndMergeToStoreAndIndex(ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction) { + dingodb::pb::store::ControlConfigRequest request; + dingodb::pb::store::ControlConfigResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + dingodb::pb::common::ControlConfigVariable config_auto_split; + config_auto_split.set_name("FLAGS_region_enable_auto_split"); + config_auto_split.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_split)); + + dingodb::pb::common::ControlConfigVariable config_auto_merge; + config_auto_merge.set_name("FLAGS_region_enable_auto_merge"); + config_auto_merge.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_merge)); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = store_interaction->AllSendRequest("StoreService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + for (const auto& config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_split") { + region_auto_split_enable_after_finish_ = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_merge") { + region_auto_merge_enable_after_finish_ = true; + } + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + status = index_interaction->AllSendRequest("IndexService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + for (const auto& config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_split") { + region_auto_split_enable_after_finish_ = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_merge") { + region_auto_merge_enable_after_finish_ = true; + } + } + + return butil::Status::OK(); +} +butil::Status Backup::EnableSplitAndMergeToStoreAndIndex(ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction) const { + dingodb::pb::store::ControlConfigRequest request; + dingodb::pb::store::ControlConfigResponse response; + + if (region_auto_split_enable_after_finish_) { + dingodb::pb::common::ControlConfigVariable config_auto_split; + config_auto_split.set_name("FLAGS_region_enable_auto_split"); + config_auto_split.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_auto_split)); + } + + if (region_auto_merge_enable_after_finish_) { + dingodb::pb::common::ControlConfigVariable config_auto_merge; + config_auto_merge.set_name("FLAGS_region_enable_auto_merge"); + config_auto_merge.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_auto_merge)); + } + + if (!request.control_config_variable().empty()) { + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = store_interaction->AllSendRequest("StoreService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + status = index_interaction->AllSendRequest("IndexService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + } + return butil::Status::OK(); +} + +butil::Status Backup::GetVersionFromCoordinator(ServerInteractionPtr coordinator_interaction, + dingodb::pb::common::VersionInfo& version_info) { + dingodb::pb::coordinator::HelloRequest request; + dingodb::pb::coordinator::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = coordinator_interaction->SendRequest("CoordinatorService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + version_info = response.version_info(); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + return butil::Status::OK(); +} + +butil::Status Backup::CompareVersion(const dingodb::pb::common::VersionInfo& version_info_local, + const dingodb::pb::common::VersionInfo& version_info_remote) { + DINGO_LOG(INFO) << "local version info : " << version_info_local.DebugString() << std::endl; + DINGO_LOG(INFO) << "remote version info : " << version_info_remote.DebugString() << std::endl; + + if (version_info_local.git_commit_hash() != version_info_remote.git_commit_hash()) { + std::string s = fmt::format("git_commit_hash is different. local : {} remote : {}", + version_info_local.git_commit_hash(), version_info_remote.git_commit_hash()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EBACKUP_VERSION_NOT_MATCH, s); + } + + return butil::Status::OK(); +} + +butil::Status Backup::DoFinish() { + if (is_already_register_backup_to_coordinator_) { + if (!is_need_exit_) { + is_need_exit_ = true; + } + std::cerr << "Waiting for register backup thread exit. (Do not use kill -9 or Ctrl-C to exit.) <"; + DINGO_LOG(INFO) << "Waiting for register backup thread exit. (Do not use kill -9 or Ctrl-C to exit.) <"; + std::string s; + while (true) { + if (is_exit_register_backup_to_coordinator_thread_) { + break; + } + // sleep(FLAGS_backup_watch_interval_s); + sleep(1); + std::cerr << "-"; + s += "-"; + } + std::cout << ">" << std::endl; + DINGO_LOG(INFO) << s << ">"; + UnregisterBackupToCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + } + + if (is_gc_enable_after_finish_) { + SetGcStart(); + } + + butil::Status status = EnableBalanceToCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + if (status.ok()) { + if (balance_leader_enable_after_finish_) { + std::cout << "balance leader set start ok" << std::endl; + DINGO_LOG(INFO) << "balance leader set start ok"; + } + + if (balance_region_enable_after_finish_) { + std::cout << "balance region set start ok" << std::endl; + DINGO_LOG(INFO) << "balance region set start ok"; + } + } + + status = EnableSplitAndMergeToStoreAndIndex(br::InteractionManager::GetInstance().GetStoreInteraction(), + br::InteractionManager::GetInstance().GetIndexInteraction()); + if (status.ok()) { + if (region_auto_split_enable_after_finish_) { + std::cout << "region auto split set start ok" << std::endl; + DINGO_LOG(INFO) << "region auto split set start ok"; + } + + if (region_auto_merge_enable_after_finish_) { + std::cout << "region auto merge set start ok" << std::endl; + DINGO_LOG(INFO) << "region auto merge set start ok"; + } + } + + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup.h b/src/br/backup.h new file mode 100644 index 000000000..864b184c4 --- /dev/null +++ b/src/br/backup.h @@ -0,0 +1,129 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_H_ +#define DINGODB_BR_BACKUP_H_ + +#include +#include +#include +#include + +#include "br/backup_data.h" +#include "br/backup_meta.h" +#include "br/interation.h" +#include "br/parameter.h" +#include "butil/status.h" +#include "fmt/core.h" + +namespace br { + +class Backup : public std::enable_shared_from_this { + public: + Backup(const BackupParams& params); + ~Backup(); + + Backup(const Backup&) = delete; + const Backup& operator=(const Backup&) = delete; + Backup(Backup&&) = delete; + Backup& operator=(Backup&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status Init(); + + butil::Status Run(); + + butil::Status Finish(); + + protected: + private: + butil::Status DoRun(); + butil::Status ParamsCheck(); + butil::Status ParamsCheckForStorage(); + butil::Status GetGcSafePoint(); + butil::Status SetGcStop(); + butil::Status SetGcStart(); + butil::Status RegisterBackupToCoordinator(bool is_first, ServerInteractionPtr coordinator_interaction); + butil::Status UnregisterBackupToCoordinator(ServerInteractionPtr coordinator_interaction); + butil::Status DoAsyncRegisterBackupToCoordinator(ServerInteractionPtr coordinator_interaction); + butil::Status DoRegisterBackupToCoordinatorInternal(ServerInteractionPtr coordinator_interaction); + + butil::Status DisableBalanceToCoordinator(ServerInteractionPtr coordinator_interaction); + butil::Status EnableBalanceToCoordinator(ServerInteractionPtr coordinator_interaction) const; + + butil::Status DisableSplitAndMergeToStoreAndIndex(ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction); + butil::Status EnableSplitAndMergeToStoreAndIndex(ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction) const; + + static butil::Status GetVersionFromCoordinator(ServerInteractionPtr coordinator_interaction, + dingodb::pb::common::VersionInfo& version_info); + + static butil::Status CompareVersion(const dingodb::pb::common::VersionInfo& version_info_local, + const dingodb::pb::common::VersionInfo& version_info_remote); + + butil::Status DoFinish(); + + std::string coor_url_; + std::string br_type_; + std::string br_backup_type_; + std::string backupts_; + int64_t backuptso_internal_; + std::string storage_; + std::string storage_internal_; + + // gc is stop or not + bool is_gc_stop_; + + // gc is enable after finish or not + bool is_gc_enable_after_finish_; + + // notify other threads to exit + std::atomic is_need_exit_; + + // backup task id. use uuid to generate. + std::string backup_task_id_; + + // is already register backup to coordinator or not + bool is_already_register_backup_to_coordinator_; + + // is exit register backup to coordinator thread. default true. + bool is_exit_register_backup_to_coordinator_thread_; + + bool region_auto_split_enable_after_finish_; + bool region_auto_merge_enable_after_finish_; + + bool balance_leader_enable_after_finish_; + bool balance_region_enable_after_finish_; + + // last error + butil::Status last_error_; + + std::shared_ptr backup_meta_; + + std::shared_ptr backup_data_; + + // statistics + int64_t start_time_ms_; + + // statistics + int64_t end_time_ms_; + + bthread_mutex_t mutex_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_H_ \ No newline at end of file diff --git a/src/br/backup_data.cc b/src/br/backup_data.cc new file mode 100644 index 000000000..e30fabe45 --- /dev/null +++ b/src/br/backup_data.cc @@ -0,0 +1,306 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_data.h" + +#include +#include +#include + +#include "br/helper.h" +#include "br/interation.h" +#include "br/parameter.h" +#include "br/sst_file_writer.h" +#include "common/constant.h" +#include "common/helper.h" +#include "common/logging.h" +#include "fmt/core.h" +#include "proto/coordinator.pb.h" + +namespace br { + +BackupData::BackupData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : coordinator_interaction_(coordinator_interaction), + store_interaction_(store_interaction), + index_interaction_(index_interaction), + document_interaction_(document_interaction), + backupts_(backupts), + backuptso_internal_(backuptso_internal), + storage_(storage), + storage_internal_(storage_internal) {} + +BackupData::~BackupData() = default; + +std::shared_ptr BackupData::GetSelf() { return shared_from_this(); } + +butil::Status BackupData::Init(const std::vector& meta_region_list) { + butil::Status status = GetAllRegionMapFromCoordinator(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + DINGO_LOG(INFO) << "GetAllRegionMapFromCoordinator success"; + + // init sql data + { + if (!backup_sql_data_) { + std::vector coordinator_addrs = coordinator_interaction_->GetAddrs(); + std::vector store_addrs = store_interaction_->GetAddrs(); + std::vector index_addrs = index_interaction_->GetAddrs(); + std::vector document_addrs = document_interaction_->GetAddrs(); + + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr store_interaction; + status = ServerInteraction::CreateInteraction(store_addrs, store_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr index_interaction; + status = ServerInteraction::CreateInteraction(index_addrs, index_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr document_interaction; + status = ServerInteraction::CreateInteraction(document_addrs, document_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + backup_sql_data_ = std::make_shared(coordinator_interaction, store_interaction, index_interaction, + document_interaction, backupts_, backuptso_internal_, storage_, + storage_internal_); + } + + backup_sql_data_->SetRegionMap(region_map_); + + status = backup_sql_data_->RemoveSqlMeta(meta_region_list); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sql_data_->Filter(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + // init sdk data + { + if (!backup_sdk_data_) { + std::vector coordinator_addrs = coordinator_interaction_->GetAddrs(); + std::vector store_addrs = store_interaction_->GetAddrs(); + std::vector index_addrs = index_interaction_->GetAddrs(); + std::vector document_addrs = document_interaction_->GetAddrs(); + + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr store_interaction; + status = ServerInteraction::CreateInteraction(store_addrs, store_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr index_interaction; + status = ServerInteraction::CreateInteraction(index_addrs, index_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr document_interaction; + status = ServerInteraction::CreateInteraction(document_addrs, document_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + backup_sdk_data_ = std::make_shared(coordinator_interaction, store_interaction, index_interaction, + document_interaction, backupts_, backuptso_internal_, storage_, + storage_internal_); + } + + backup_sdk_data_->SetRegionMap(region_map_); + + status = backup_sdk_data_->Filter(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + return butil::Status::OK(); +} + +butil::Status BackupData::Run() { + butil::Status status; + + // backup sql data + { + status = backup_sql_data_->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sql_data_->Backup(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + // backup sdk data + { + status = backup_sdk_data_->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sdk_data_->Backup(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + return butil::Status::OK(); +} + +butil::Status BackupData::Finish() { + butil::Status status; + + std::string file_name = dingodb::Constant::kBackupMetaDataFileName; + std::string file_path = storage_internal_ + "/" + file_name; + + std::map kvs; + + std::shared_ptr> sql_backup_meta = backup_sql_data_->GetBackupMeta(); + std::shared_ptr> sdk_backup_meta = backup_sdk_data_->GetBackupMeta(); + for (const auto& meta : *sql_backup_meta) { + kvs.emplace(meta.file_name(), meta.SerializeAsString()); + } + + for (const auto& meta : *sdk_backup_meta) { + kvs.emplace(meta.file_name(), meta.SerializeAsString()); + } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (!backup_meta_) { + backup_meta_ = std::make_shared(); + } + + backup_meta_->set_remark("backup sdk data and sql data. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. "); + backup_meta_->set_exec_node(dingodb::Constant::kBackupRegionName); + backup_meta_->set_dir_name(""); + backup_meta_->set_file_size(sst->GetSize()); + backup_meta_->set_encryption(hash_code); + backup_meta_->set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta_->DebugString() << std::endl; + } + + return butil::Status::OK(); +} + +std::shared_ptr BackupData::GetBackupMeta() { return backup_meta_; } + +butil::Status BackupData::GetAllRegionMapFromCoordinator() { + dingodb::pb::coordinator::GetRegionMapRequest request; + dingodb::pb::coordinator::GetRegionMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_tenant_id(-1); // get all tenants region map + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + auto status = coordinator_interaction_->SendRequest("CoordinatorService", "GetRegionMap", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + return butil::Status(response.error().errcode(), response.error().errmsg()); + } + + region_map_ = std::make_shared(response.regionmap()); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << "==================================================="; + DINGO_LOG(INFO) << "GetRegionMap region size = " << response.regionmap().regions_size(); + int i = 0; + std::string s; + for (const auto& region : response.regionmap().regions()) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "region_id=[" << s << "]"; + s.clear(); + i = 0; + } + } + if (!s.empty()) { + DINGO_LOG(INFO) << "region_id=[" << s << "]"; + } + DINGO_LOG(INFO) << "==================================================="; + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_data.h b/src/br/backup_data.h new file mode 100644 index 000000000..b998d690d --- /dev/null +++ b/src/br/backup_data.h @@ -0,0 +1,77 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_DATA_H_ +#define DINGODB_BR_BACKUP_DATA_H_ + +#include +#include +#include + +#include "br/backup_sdk_data.h" +#include "br/backup_sql_data.h" +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +class BackupData : public std::enable_shared_from_this { + public: + BackupData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + ~BackupData(); + + BackupData(const BackupData&) = delete; + const BackupData& operator=(const BackupData&) = delete; + BackupData(BackupData&&) = delete; + BackupData& operator=(BackupData&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status Init(const std::vector& meta_region_list); + + butil::Status Run(); + + butil::Status Finish(); + + std::shared_ptr GetRegionMap() const { return region_map_; } + + std::shared_ptr GetBackupMeta(); + + protected: + private: + butil::Status GetAllRegionMapFromCoordinator(); + ServerInteractionPtr coordinator_interaction_; + ServerInteractionPtr store_interaction_; + ServerInteractionPtr index_interaction_; + ServerInteractionPtr document_interaction_; + std::string backupts_; + int64_t backuptso_internal_; + std::string storage_; + std::string storage_internal_; + std::shared_ptr region_map_; + + std::shared_ptr backup_sql_data_; + std::shared_ptr backup_sdk_data_; + + std::shared_ptr backup_meta_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_SDK_DATA_H_ \ No newline at end of file diff --git a/src/br/backup_data_base.cc b/src/br/backup_data_base.cc new file mode 100644 index 000000000..5fa990fd5 --- /dev/null +++ b/src/br/backup_data_base.cc @@ -0,0 +1,327 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_data_base.h" + +#include +#include +#include + +#include "br/helper.h" +#include "br/sst_file_writer.h" +#include "common/constant.h" +#include "common/helper.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +BackupDataBase::BackupDataBase(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal, const std::string& name) + : coordinator_interaction_(coordinator_interaction), + store_interaction_(store_interaction), + index_interaction_(index_interaction), + document_interaction_(document_interaction), + backupts_(backupts), + backuptso_internal_(backuptso_internal), + storage_(storage), + storage_internal_(storage_internal), + is_need_exit_(false), + already_handle_regions_(0), + already_handle_store_regions_(0), + already_handle_index_regions_(0), + already_handle_document_regions_(0), + name_(name) {} + +BackupDataBase::~BackupDataBase() = default; + +void BackupDataBase::SetRegionMap(std::shared_ptr region_map) { + region_map_ = region_map; +} + +butil::Status BackupDataBase::Filter() { return butil::Status::OK(); } + +butil::Status BackupDataBase::Run() { return butil::Status::OK(); } + +butil::Status BackupDataBase::Backup() { + butil::Status status; + + if (!backup_data_base_) { + backup_data_base_ = std::make_shared>(); + } + + status = BackupRegion(); + if (!status.ok()) { + is_need_exit_ = true; + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + status = BackupCfSstMeta(); + if (!status.ok()) { + is_need_exit_ = true; + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +std::shared_ptr> BackupDataBase::GetBackupMeta() { + return backup_data_base_; +} + +butil::Status BackupDataBase::BackupRegion() { + butil::Status status; + std::string file_name; + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kStoreRegionSdkDataSstName; + } else { + file_name = dingodb::Constant::kStoreRegionSqlDataSstName; + } + status = DoBackupRegion(wait_for_handle_store_regions_, file_name); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kIndexRegionSdkDataSstName; + } else { + file_name = dingodb::Constant::kIndexRegionSqlDataSstName; + } + status = DoBackupRegion(wait_for_handle_index_regions_, file_name); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kDocumentRegionSdkDataSstName; + } else { + file_name = dingodb::Constant::kDocumentRegionSqlDataSstName; + } + status = DoBackupRegion(wait_for_handle_document_regions_, file_name); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +butil::Status BackupDataBase::BackupCfSstMeta() { + butil::Status status; + std::string file_name; + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kStoreCfSstMetaSdkDataSstName; + } else { + file_name = dingodb::Constant::kStoreCfSstMetaSqlDataSstName; + } + status = DoBackupCfSstMeta(save_store_region_map_, file_name, dingodb::Constant::kStoreRegionName); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kIndexCfSstMetaSdkDataSstName; + } else { + file_name = dingodb::Constant::kIndexCfSstMetaSqlDataSstName; + } + status = DoBackupCfSstMeta(save_index_region_map_, file_name, dingodb::Constant::kIndexRegionName); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (name_ == dingodb::Constant::kSdkData) { + file_name = dingodb::Constant::kDocumentCfSstMetaSdkDataSstName; + } else { + file_name = dingodb::Constant::kDocumentCfSstMetaSqlDataSstName; + } + status = DoBackupCfSstMeta(save_document_region_map_, file_name, dingodb::Constant::kDocumentRegionName); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +butil::Status BackupDataBase::DoBackupRegion( + std::shared_ptr> wait_for_handle_regions, const std::string& file_name) { + butil::Status status; + + std::string file_path = storage_internal_ + "/" + file_name; + + if (!wait_for_handle_regions->empty()) { + std::map kvs; + for (const auto& region : *wait_for_handle_regions) { + kvs.emplace(std::to_string(region.id()), region.SerializeAsString()); + } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + save_region_files_.push_back(file_path); + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + dingodb::pb::common::BackupMeta backup_meta; + + if (name_ == dingodb::Constant::kSdkData) { + backup_meta.set_remark("python sdk create region meta info. save dingodb::pb::common::Region. "); + } else { + backup_meta.set_remark("executor create region meta info. save dingodb::pb::common::Region. "); + } + backup_meta.set_exec_node(dingodb::Constant::kCoordinatorRegionName); + backup_meta.set_dir_name(""); + backup_meta.set_file_size(sst->GetSize()); + backup_meta.set_encryption(hash_code); + backup_meta.set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta.DebugString() << std::endl; + } + + backup_data_base_->push_back(std::move(backup_meta)); + } + return butil::Status::OK(); +} + +butil::Status BackupDataBase::DoBackupCfSstMeta( + std::shared_ptr> save_region_map, + const std::string& file_name, const std::string& region_from) { + butil::Status status; + std::string file_path = storage_internal_ + "/" + file_name; + + if (!save_region_map->empty()) { + std::map kvs; + for (const auto& [region_id, group] : *save_region_map) { + kvs.emplace(std::to_string(region_id), group.SerializeAsString()); + } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + save_cf_sst_meta_files_.push_back(file_path); + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + dingodb::pb::common::BackupMeta backup_meta; + + if (name_ == dingodb::Constant::kSdkData) { + backup_meta.set_remark( + "python sdk create region sst meta info. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. "); + + } else { + backup_meta.set_remark( + "executor create region sst meta info. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. "); + } + backup_meta.set_exec_node(region_from); + backup_meta.set_dir_name(""); + backup_meta.set_file_size(sst->GetSize()); + backup_meta.set_encryption(hash_code); + backup_meta.set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta.DebugString() << std::endl; + } + + backup_data_base_->push_back(std::move(backup_meta)); + } + return butil::Status::OK(); +} + +butil::Status BackupDataBase::DoBackupRegionInternal( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map) { + for (const auto& region : *wait_for_handle_regions) { + if (is_need_exit_) { + break; + } + dingodb::pb::store::BackupDataRequest request; + dingodb::pb::store::BackupDataResponse response; + + request.mutable_request_info()->set_request_id(Helper::GetRandInt()); + request.mutable_context()->set_region_id(region.id()); + request.mutable_context()->mutable_region_epoch()->CopyFrom(region.definition().epoch()); + request.set_start_key(region.definition().range().start_key()); + request.set_end_key(region.definition().range().end_key()); + request.set_need_leader(true); + request.set_region_type(region.region_type()); + request.set_backup_ts(backupts_); + request.set_backup_tso(backuptso_internal_); + request.set_storage_path(storage_); + request.mutable_storage_backend()->mutable_local()->set_path(storage_internal_); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << name_ << " " << request.DebugString(); + + butil::Status status = interaction->SendRequest(service_name, "BackupData", request, response); + if (!status.ok()) { + is_need_exit_ = true; + std::string s = fmt::format("Fail to backup region, region_id={}, status={}", region.id(), status.error_cstr()); + DINGO_LOG(ERROR) << s; + last_error_ = status; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + is_need_exit_ = true; + std::string s = + fmt::format("Fail to backup region, region_id={}, error={}", region.id(), response.error().errmsg()); + DINGO_LOG(ERROR) << s; + status = butil::Status(response.error().errcode(), s); + last_error_ = status; + return status; + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << name_ << " " << response.DebugString(); + + save_region_map->insert({region.id(), response.sst_metas()}); + + already_handle_regions++; + } + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_data_base.h b/src/br/backup_data_base.h new file mode 100644 index 000000000..8de7390fe --- /dev/null +++ b/src/br/backup_data_base.h @@ -0,0 +1,111 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_DATA_BASE_H_ +#define DINGODB_BR_BACKUP_DATA_BASE_H_ + +#include +#include +#include + +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +class BackupSdkData; +class BackupSqlData; +class BackupDataBase { + public: + BackupDataBase(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal, const std::string& name); + virtual ~BackupDataBase(); + + BackupDataBase(const BackupDataBase&) = delete; + const BackupDataBase& operator=(const BackupDataBase&) = delete; + BackupDataBase(BackupDataBase&&) = delete; + BackupDataBase& operator=(BackupDataBase&&) = delete; + + void SetRegionMap(std::shared_ptr region_map); + + virtual butil::Status Filter(); + + virtual butil::Status Run(); + + virtual butil::Status Backup(); + + std::shared_ptr> GetBackupMeta(); + + friend BackupSdkData; + friend BackupSqlData; + + protected: + butil::Status BackupRegion(); + butil::Status BackupCfSstMeta(); + butil::Status DoBackupRegion(std::shared_ptr> wait_for_handle_regions, + const std::string& file_name); + butil::Status DoBackupCfSstMeta( + std::shared_ptr> save_region_map, + const std::string& file_name, const std::string& region_from); + butil::Status DoBackupRegionInternal( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map); + + private: + ServerInteractionPtr coordinator_interaction_; + ServerInteractionPtr store_interaction_; + ServerInteractionPtr index_interaction_; + ServerInteractionPtr document_interaction_; + std::shared_ptr region_map_; + std::shared_ptr> wait_for_handle_store_regions_; + std::shared_ptr> wait_for_handle_index_regions_; + std::shared_ptr> wait_for_handle_document_regions_; + + std::string backupts_; + int64_t backuptso_internal_; + std::string storage_; + std::string storage_internal_; + + // notify other threads to exit + std::atomic is_need_exit_; + + std::atomic already_handle_regions_; + std::atomic already_handle_store_regions_; + std::atomic already_handle_index_regions_; + std::atomic already_handle_document_regions_; + + std::shared_ptr> save_store_region_map_; + std::shared_ptr> save_index_region_map_; + std::shared_ptr> save_document_region_map_; + + std::string name_; + + std::vector save_region_files_; + std::vector save_cf_sst_meta_files_; + + std::shared_ptr> backup_data_base_; + + // last error + butil::Status last_error_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_DATA_BASE_H_ \ No newline at end of file diff --git a/src/br/backup_meta.cc b/src/br/backup_meta.cc new file mode 100644 index 000000000..b7d00f23d --- /dev/null +++ b/src/br/backup_meta.cc @@ -0,0 +1,283 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_meta.h" + +#include +#include + +#include "br/helper.h" +#include "br/sst_file_writer.h" +#include "common/constant.h" +#include "common/helper.h" +#include "common/logging.h" +#include "fmt/core.h" + +namespace br { + +BackupMeta::BackupMeta(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : coordinator_interaction_(coordinator_interaction), + store_interaction_(store_interaction), + index_interaction_(index_interaction), + document_interaction_(document_interaction), + backupts_(backupts), + backuptso_internal_(backuptso_internal), + storage_(storage), + storage_internal_(storage_internal) {} + +BackupMeta::~BackupMeta() = default; + +std::shared_ptr BackupMeta::GetSelf() { return shared_from_this(); } + +butil::Status BackupMeta::Init() { + butil::Status status; + if (!backup_sql_meta_) { + std::vector coordinator_addrs = coordinator_interaction_->GetAddrs(); + std::vector store_addrs = store_interaction_->GetAddrs(); + + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::shared_ptr store_interaction; + status = ServerInteraction::CreateInteraction(store_addrs, store_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + backup_sql_meta_ = std::make_shared(coordinator_interaction, store_interaction, backupts_, + backuptso_internal_, storage_, storage_internal_); + } + + status = backup_sql_meta_->GetSqlMetaRegionFromCoordinator(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + DINGO_LOG(INFO) << "GetSqlMetaRegionFromCoordinator success"; + + if (!backup_sdk_meta_) { + std::vector coordinator_addrs = coordinator_interaction_->GetAddrs(); + + std::shared_ptr coordinator_interaction; + status = ServerInteraction::CreateInteraction(coordinator_addrs, coordinator_interaction); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + backup_sdk_meta_ = std::make_shared(coordinator_interaction, storage_internal_); + } + + status = backup_sdk_meta_->GetSdkMetaFromCoordinator(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + DINGO_LOG(INFO) << "GetSdkMetaFromCoordinator success"; + + return butil::Status::OK(); +} + +butil::Status BackupMeta::Run(std::shared_ptr region_map) { + butil::Status status; + + // backup sql meta + { + backup_sql_meta_->SetRegionMap(region_map); + + std::vector region_list; + backup_sql_meta_->GetSqlMetaRegionList(region_list); + status = backup_sql_meta_->ReserveSqlMeta(region_list); + + status = backup_sql_meta_->Filter(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sql_meta_->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sql_meta_->Backup(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + // backup sdk meta + { + status = backup_sdk_meta_->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = backup_sdk_meta_->Backup(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + return butil::Status::OK(); +} + +butil::Status BackupMeta::Finish() { + butil::Status status; + + std::string file_name = dingodb::Constant::kBackupMetaSchemaName; + std::string file_path = storage_internal_ + "/" + file_name; + + std::map kvs; + + std::shared_ptr> sql_meta = backup_sql_meta_->GetBackupMeta(); + std::shared_ptr sdk_meta = backup_sdk_meta_->GetBackupMeta(); + for (const auto& meta : *sql_meta) { + kvs.emplace(meta.file_name(), meta.SerializeAsString()); + } + + { kvs.emplace(sdk_meta->file_name(), sdk_meta->SerializeAsString()); } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (!backup_meta_) { + backup_meta_ = std::make_shared(); + } + + backup_meta_->set_remark("backup sdk meta and sql meta. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. "); + backup_meta_->set_exec_node(dingodb::Constant::kBackupRegionName); + backup_meta_->set_dir_name(""); + backup_meta_->set_file_size(sst->GetSize()); + backup_meta_->set_encryption(hash_code); + backup_meta_->set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta_->DebugString() << std::endl; + } + + return butil::Status::OK(); +} + +std::vector BackupMeta::GetSqlMetaRegionList() { + std::vector region_list; + backup_sql_meta_->GetSqlMetaRegionList(region_list); + return region_list; +} + +std::shared_ptr BackupMeta::GetBackupMeta() { return backup_meta_; } + +std::pair> BackupMeta::GetIdEpochTypeAndValue() { + butil::Status status; + status = GetPresentIdsFromCoordinator(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return {status, nullptr}; + } + return {butil::Status::OK(), id_epoch_type_and_value_}; +} + +std::pair> BackupMeta::GetAllTableIncrement() { + butil::Status status; + status = GetAllTableIncrementFromMeta(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return {status, nullptr}; + } + return {butil::Status::OK(), table_increment_group_}; +} + +butil::Status BackupMeta::GetPresentIdsFromCoordinator() { + dingodb::pb::meta::SaveIdEpochTypeRequest request; + dingodb::pb::meta::SaveIdEpochTypeResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + auto status = coordinator_interaction_->SendRequest("MetaService", "SaveIdEpochType", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + return butil::Status(response.error().errcode(), response.error().errmsg()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + id_epoch_type_and_value_ = + std::make_shared(response.id_epoch_type_and_value()); + + return butil::Status::OK(); +} + +butil::Status BackupMeta::GetAllTableIncrementFromMeta() { + dingodb::pb::meta::GetAutoIncrementsRequest request; + dingodb::pb::meta::GetAutoIncrementsResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + auto status = coordinator_interaction_->SendRequest("MetaService", "GetAutoIncrements", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + return butil::Status(response.error().errcode(), response.error().errmsg()); + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + table_increment_group_ = std::make_shared(); + for (const auto& table_increment : response.table_increments()) { + table_increment_group_->add_table_increments()->CopyFrom(table_increment); + } + + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_meta.h b/src/br/backup_meta.h new file mode 100644 index 000000000..62ad240b3 --- /dev/null +++ b/src/br/backup_meta.h @@ -0,0 +1,84 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_META_H_ +#define DINGODB_BR_BACKUP_META_H_ + +#include +#include +#include +#include + +#include "br/backup_sdk_meta.h" +#include "br/backup_sql_meta.h" +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/meta.pb.h" + +namespace br { + +class BackupMeta : public std::enable_shared_from_this { + public: + BackupMeta(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + ~BackupMeta(); + + BackupMeta(const BackupMeta&) = delete; + const BackupMeta& operator=(const BackupMeta&) = delete; + BackupMeta(BackupMeta&&) = delete; + BackupMeta& operator=(BackupMeta&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status Init(); + + butil::Status Run(std::shared_ptr region_map); + + butil::Status Finish(); + + std::vector GetSqlMetaRegionList(); + + std::shared_ptr GetBackupMeta(); + + std::pair> GetIdEpochTypeAndValue(); + + std::pair> GetAllTableIncrement(); + + protected: + private: + butil::Status GetPresentIdsFromCoordinator(); + butil::Status GetAllTableIncrementFromMeta(); + ServerInteractionPtr coordinator_interaction_; + ServerInteractionPtr store_interaction_; + ServerInteractionPtr index_interaction_; + ServerInteractionPtr document_interaction_; + std::string backupts_; + int64_t backuptso_internal_; + std::string storage_; + std::string storage_internal_; + std::shared_ptr id_epoch_type_and_value_; + std::shared_ptr table_increment_group_; + + std::shared_ptr backup_sql_meta_; + std::shared_ptr backup_sdk_meta_; + + std::shared_ptr backup_meta_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_META_H_ \ No newline at end of file diff --git a/src/br/backup_meta_base.cc b/src/br/backup_meta_base.cc new file mode 100644 index 000000000..335e097d3 --- /dev/null +++ b/src/br/backup_meta_base.cc @@ -0,0 +1,319 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_meta_base.h" + +#include +#include +#include +#include + +#include "br/helper.h" +#include "br/parameter.h" +#include "br/sst_file_writer.h" +#include "common/constant.h" +#include "common/helper.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +BackupMetaBase::BackupMetaBase(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : coordinator_interaction_(coordinator_interaction), + store_interaction_(store_interaction), + backupts_(backupts), + backuptso_internal_(backuptso_internal), + storage_(storage), + storage_internal_(storage_internal), + is_need_exit_(false), + already_handle_store_regions_(0) {} + +BackupMetaBase::~BackupMetaBase() = default; + +void BackupMetaBase::SetRegionMap(std::shared_ptr region_map) { + region_map_ = region_map; +} + +butil::Status BackupMetaBase::Filter() { + if (!wait_for_handle_store_regions_) { + wait_for_handle_store_regions_ = std::make_shared>(); + } + + for (const auto& region : region_map_->regions()) { + // only handle executor meta region + if (dingodb::Helper::IsExecutorTxn(region.definition().range().start_key())) { + auto iter = std::find(meta_region_list_.begin(), meta_region_list_.end(), region.id()); + if (iter != meta_region_list_.end()) { + if (dingodb::pb::common::RegionType::STORE_REGION == region.region_type()) { + wait_for_handle_store_regions_->push_back(region); + } + } + } + } + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::ReserveSqlMeta(std::vector& meta_region_list) { + meta_region_list_ = meta_region_list; + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::Run() { + butil::Status status; + if (!save_store_region_map_) { + save_store_region_map_ = + std::make_shared>(); + } + + int64_t total_store_regions_count = wait_for_handle_store_regions_->size(); + + std::cerr << "Full Backup Sql Meta " << "<"; + DINGO_LOG(INFO) << "Full Backup Sql Meta " << "<"; + while (!is_need_exit_) { + std::cerr << "-"; + DINGO_LOG(INFO) << "-"; + if (already_handle_store_regions_ >= total_store_regions_count) { + break; + } + + // store + status = DoBackupRegionInternal(store_interaction_, "StoreService", wait_for_handle_store_regions_, + already_handle_store_regions_, save_store_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + } + + std::cerr << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() << "]"; + DINGO_LOG(INFO) << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() << "]"; + + std::cout << std::endl; + + DINGO_LOG(INFO) << "backup sql meta " << "total_regions : " << already_handle_store_regions_ + << ", store_regions : " << already_handle_store_regions_; + + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::Backup() { + butil::Status status; + + if (!backup_meta_base_) { + backup_meta_base_ = std::make_shared>(); + } + + status = BackupRegion(); + if (!status.ok()) { + is_need_exit_ = true; + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + status = BackupCfSstMeta(); + if (!status.ok()) { + is_need_exit_ = true; + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +std::shared_ptr> BackupMetaBase::GetBackupMeta() { + return backup_meta_base_; +} + +butil::Status BackupMetaBase::BackupRegion() { + butil::Status status; + + std::string file_name = dingodb::Constant::kStoreRegionSqlMetaSstName; + + status = DoBackupRegion(wait_for_handle_store_regions_, file_name); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::BackupCfSstMeta() { + butil::Status status; + std::string file_name = dingodb::Constant::kStoreCfSstMetaSqlMetaSstName; + status = DoBackupCfSstMeta(save_store_region_map_, file_name); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::DoBackupRegion( + std::shared_ptr> wait_for_handle_regions, const std::string& file_name) { + butil::Status status; + + std::string file_path = storage_internal_ + "/" + file_name; + + if (!wait_for_handle_regions->empty()) { + std::map kvs; + for (const auto& region : *wait_for_handle_regions) { + kvs.emplace(std::to_string(region.id()), region.SerializeAsString()); + } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + save_region_files_.push_back(file_path); + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + dingodb::pb::common::BackupMeta backup_meta; + + backup_meta.set_remark( + "executor sql create tenants, schema, table, index etc. meta info. save dingodb::pb::common::Region. "); + backup_meta.set_exec_node(dingodb::Constant::kCoordinatorRegionName); + backup_meta.set_dir_name(""); + backup_meta.set_file_size(sst->GetSize()); + backup_meta.set_encryption(hash_code); + backup_meta.set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta.DebugString() << std::endl; + } + + backup_meta_base_->push_back(std::move(backup_meta)); + } + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::DoBackupCfSstMeta( + std::shared_ptr> save_region_map, + const std::string& file_name) { + butil::Status status; + + std::string file_path = storage_internal_ + "/" + file_name; + + if (!save_region_map->empty()) { + std::map kvs; + for (const auto& [region_id, group] : *save_region_map) { + kvs.emplace(std::to_string(region_id), group.SerializeAsString()); + } + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + save_cf_sst_meta_files_.push_back(file_path); + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + dingodb::pb::common::BackupMeta backup_meta; + + backup_meta.set_remark( + "executor sql create tenants, schema, table, index etc. meta info. save " + "dingodb::pb::common::BackupDataFileValueSstMetaGroup."); + backup_meta.set_exec_node(dingodb::Constant::kStoreRegionName); + backup_meta.set_dir_name(""); + backup_meta.set_file_size(sst->GetSize()); + backup_meta.set_encryption(hash_code); + backup_meta.set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_meta.DebugString() << std::endl; + } + + backup_meta_base_->push_back(std::move(backup_meta)); + } + return butil::Status::OK(); +} + +butil::Status BackupMetaBase::DoBackupRegionInternal( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map) { + for (const auto& region : *wait_for_handle_regions) { + if (is_need_exit_) { + break; + } + dingodb::pb::store::BackupMetaRequest request; + dingodb::pb::store::BackupMetaResponse response; + + request.mutable_request_info()->set_request_id(Helper::GetRandInt()); + request.mutable_context()->set_region_id(region.id()); + request.mutable_context()->mutable_region_epoch()->CopyFrom(region.definition().epoch()); + request.set_start_key(region.definition().range().start_key()); + request.set_end_key(region.definition().range().end_key()); + request.set_need_leader(true); + request.set_region_type(region.region_type()); + request.set_backup_ts(backupts_); + request.set_backup_tso(backuptso_internal_); + request.set_storage_path(storage_); + request.mutable_storage_backend()->mutable_local()->set_path(storage_internal_); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + butil::Status status = interaction->SendRequest(service_name, "BackupMeta", request, response); + if (!status.ok()) { + is_need_exit_ = true; + std::string s = fmt::format("Fail to backup region, region_id={}, status={}", region.id(), status.error_cstr()); + DINGO_LOG(ERROR) << s; + last_error_ = status; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + is_need_exit_ = true; + std::string s = + fmt::format("Fail to backup region, region_id={}, error={}", region.id(), response.error().errmsg()); + DINGO_LOG(ERROR) << s; + status = butil::Status(response.error().errcode(), s); + last_error_ = status; + return status; + } + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + save_region_map->insert({region.id(), response.sst_metas()}); + + already_handle_regions++; + } + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_meta_base.h b/src/br/backup_meta_base.h new file mode 100644 index 000000000..5298f0ea5 --- /dev/null +++ b/src/br/backup_meta_base.h @@ -0,0 +1,101 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_META_BASE_H_ +#define DINGODB_BR_BACKUP_META_BASE_H_ + +#include +#include +#include + +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +class BackupSqlMeta; +class BackupMetaBase { + public: + BackupMetaBase(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + virtual ~BackupMetaBase(); + + BackupMetaBase(const BackupMetaBase&) = delete; + const BackupMetaBase& operator=(const BackupMetaBase&) = delete; + BackupMetaBase(BackupMetaBase&&) = delete; + BackupMetaBase& operator=(BackupMetaBase&&) = delete; + + void SetRegionMap(std::shared_ptr region_map); + + virtual butil::Status Filter(); + + butil::Status ReserveSqlMeta(std::vector& meta_region_list); + + virtual butil::Status Run(); + + virtual butil::Status Backup(); + + std::shared_ptr> GetBackupMeta(); + + friend BackupSqlMeta; + + protected: + butil::Status BackupRegion(); + butil::Status BackupCfSstMeta(); + butil::Status DoBackupRegion(std::shared_ptr> wait_for_handle_regions, + const std::string& file_name); + butil::Status DoBackupCfSstMeta( + std::shared_ptr> save_region_map, + const std::string& file_name); + butil::Status DoBackupRegionInternal( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map); + + private: + ServerInteractionPtr coordinator_interaction_; + ServerInteractionPtr store_interaction_; + std::shared_ptr region_map_; + std::shared_ptr> wait_for_handle_store_regions_; + + std::string backupts_; + int64_t backuptso_internal_; + std::string storage_; + std::string storage_internal_; + + // notify other threads to exit + std::atomic is_need_exit_; + + std::atomic already_handle_store_regions_; + + std::shared_ptr> save_store_region_map_; + + std::vector meta_region_list_; + + std::vector save_region_files_; + std::vector save_cf_sst_meta_files_; + + std::shared_ptr> backup_meta_base_; + + // last error + butil::Status last_error_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_META_BASE_H_ \ No newline at end of file diff --git a/src/br/backup_sdk_data.cc b/src/br/backup_sdk_data.cc new file mode 100644 index 000000000..94bda90f0 --- /dev/null +++ b/src/br/backup_sdk_data.cc @@ -0,0 +1,264 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_sdk_data.h" + +#include +#include +#include +#include +#include + +#include "common/constant.h" +#include "common/helper.h" +#include "fmt/core.h" + +namespace br { + +#ifndef ENABLE_BACKUP_SDK_DATA_PTHREAD +#define ENABLE_BACKUP_SDK_DATA_PTHREAD +#endif + +// #undef ENABLE_BACKUP_SDK_DATA_PTHREAD + +BackupSdkData::BackupSdkData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : BackupDataBase(coordinator_interaction, store_interaction, index_interaction, document_interaction, backupts, + backuptso_internal, storage, storage_internal, dingodb::Constant::kSdkData) {} + +BackupSdkData::~BackupSdkData() = default; + +std::shared_ptr BackupSdkData::GetSelf() { return shared_from_this(); } + +butil::Status BackupSdkData::Filter() { + if (!wait_for_handle_store_regions_) { + wait_for_handle_store_regions_ = std::make_shared>(); + } + + if (!wait_for_handle_index_regions_) { + wait_for_handle_index_regions_ = std::make_shared>(); + } + + if (!wait_for_handle_document_regions_) { + wait_for_handle_document_regions_ = std::make_shared>(); + } + + for (const auto& region : region_map_->regions()) { + // only handle sdk non txn region + if (dingodb::Helper::IsClientRaw(region.definition().range().start_key())) { + if (dingodb::pb::common::RegionType::STORE_REGION == region.region_type()) { + wait_for_handle_store_regions_->push_back(region); + } else if (dingodb::pb::common::RegionType::INDEX_REGION == region.region_type()) { + wait_for_handle_index_regions_->push_back(region); + } else if (dingodb::pb::common::RegionType::DOCUMENT_REGION == region.region_type()) { + wait_for_handle_document_regions_->push_back(region); + } + } + } + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_store_regions size = " << wait_for_handle_store_regions_->size(); + int i = 0; + std::string s; + for (const auto& region : *wait_for_handle_store_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_store_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + + if (!s.empty()) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_store_regions region id=[" << s << "]"; + } + + s.clear(); + i = 0; + + DINGO_LOG(INFO) << "sdk data : wait_for_handle_index_regions size = " << wait_for_handle_index_regions_->size(); + for (const auto& region : *wait_for_handle_index_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_index_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + + if (!s.empty()) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_index_regions region id=[" << s << "]"; + } + + s.clear(); + i = 0; + + DINGO_LOG(INFO) << "sdk data : wait_for_handle_document_regions size = " + << wait_for_handle_document_regions_->size(); + for (const auto& region : *wait_for_handle_document_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_document_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + if (!s.empty()) { + DINGO_LOG(INFO) << "sdk data : wait_for_handle_document_regions region id=[" << s << "]"; + } + } + + return butil::Status::OK(); +} + +butil::Status BackupSdkData::Run() { + butil::Status status; + if (!save_store_region_map_) { + save_store_region_map_ = + std::make_shared>(); + } + + if (!save_index_region_map_) { + save_index_region_map_ = + std::make_shared>(); + } + + if (!save_document_region_map_) { + save_document_region_map_ = + std::make_shared>(); + } + + int64_t total_regions_count = wait_for_handle_store_regions_->size() + wait_for_handle_index_regions_->size() + + wait_for_handle_document_regions_->size(); + + // store + status = DoAsyncBackupRegion(store_interaction_, "StoreService", wait_for_handle_store_regions_, + already_handle_store_regions_, save_store_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // index + status = DoAsyncBackupRegion(index_interaction_, "IndexService", wait_for_handle_index_regions_, + already_handle_index_regions_, save_index_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // document + status = DoAsyncBackupRegion(document_interaction_, "DocumentService", wait_for_handle_document_regions_, + already_handle_document_regions_, save_document_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::atomic last_already_handle_regions = 0; + std::cerr << "Full Backup Sdk Data " << "<"; + DINGO_LOG(INFO) << "Full Backup Sdk Data " << "<"; + std::string s; + while (!is_need_exit_) { + already_handle_regions_ = + already_handle_store_regions_ + already_handle_index_regions_ + already_handle_document_regions_; + + int64_t diff = already_handle_regions_ - last_already_handle_regions; + for (int i = 0; i < diff; i++) { + std::cerr << "-"; + s += "-"; + } + + if (already_handle_regions_ >= total_regions_count) { + break; + } + + last_already_handle_regions.store(already_handle_regions_); + + sleep(1); + } + + if (is_need_exit_) { + return last_error_; + } + + std::cerr << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() + << ",I:" << wait_for_handle_index_regions_->size() << ",D:" << wait_for_handle_document_regions_->size() + << "]"; + DINGO_LOG(INFO) << s; + DINGO_LOG(INFO) << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() + << ",I:" << wait_for_handle_index_regions_->size() + << ",D:" << wait_for_handle_document_regions_->size() << "]"; + + std::cout << std::endl; + + DINGO_LOG(INFO) << "backup sdk data " << "total_regions : " << already_handle_regions_ + << ", store_regions : " << already_handle_store_regions_ + << ", index_regions : " << already_handle_index_regions_ + << ", document_regions : " << already_handle_document_regions_; + + return butil::Status::OK(); +} + +butil::Status BackupSdkData::DoAsyncBackupRegion( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map) { + std::shared_ptr self = GetSelf(); + auto lambda_call = [self, interaction, &service_name, wait_for_handle_regions, &already_handle_regions, + save_region_map]() { + self->DoBackupRegionInternal(interaction, service_name, wait_for_handle_regions, already_handle_regions, + save_region_map); + }; + +#if defined(ENABLE_BACKUP_SDK_DATA_PTHREAD) + std::thread th(lambda_call); + th.detach(); +#else + + std::function* call = new std::function; + *call = lambda_call; + bthread_t th; + + int ret = bthread_start_background( + &th, nullptr, + [](void* arg) -> void* { + auto* call = static_cast*>(arg); + (*call)(); + delete call; + return nullptr; + }, + call); + if (ret != 0) { + DINGO_LOG(ERROR) << fmt::format("bthread_start_background fail"); + return butil::Status(dingodb::pb::error::EINTERNAL, "bthread_start_background fail"); + } +#endif // #if defined(ENABLE_BACKUP_SDK_DATA_PTHREAD) + + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_sdk_data.h b/src/br/backup_sdk_data.h new file mode 100644 index 000000000..b8a4a564a --- /dev/null +++ b/src/br/backup_sdk_data.h @@ -0,0 +1,59 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_SDK_DATA_H_ +#define DINGODB_BR_BACKUP_SDK_DATA_H_ + +#include +#include +#include + +#include "br/backup_data_base.h" +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" + +namespace br { + +class BackupSdkData : public BackupDataBase, public std::enable_shared_from_this { + public: + BackupSdkData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + ~BackupSdkData() override; + + BackupSdkData(const BackupSdkData&) = delete; + const BackupSdkData& operator=(const BackupSdkData&) = delete; + BackupSdkData(BackupSdkData&&) = delete; + BackupSdkData& operator=(BackupSdkData&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status Filter() override; + + butil::Status Run() override; + + protected: + private: + butil::Status DoAsyncBackupRegion( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map); +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_SDK_DATA_H_ \ No newline at end of file diff --git a/src/br/backup_sdk_meta.cc b/src/br/backup_sdk_meta.cc new file mode 100644 index 000000000..7ae86dc50 --- /dev/null +++ b/src/br/backup_sdk_meta.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_sdk_meta.h" + +#include +#include + +#include "br/helper.h" +#include "br/sst_file_writer.h" +#include "common/constant.h" +#include "common/helper.h" +#include "fmt/core.h" + +namespace br { + +BackupSdkMeta::BackupSdkMeta(ServerInteractionPtr coordinator_interaction, const std::string &storage_internal) + : coordinator_interaction_(coordinator_interaction), storage_internal_(storage_internal) {} + +BackupSdkMeta::~BackupSdkMeta() = default; + +std::shared_ptr BackupSdkMeta::GetSelf() { return shared_from_this(); } + +butil::Status BackupSdkMeta::GetSdkMetaFromCoordinator() { + dingodb::pb::meta::ExportMetaRequest request; + dingodb::pb::meta::ExportMetaResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + auto status = coordinator_interaction_->SendRequest("MetaService", "ExportMeta", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + return butil::Status(response.error().errcode(), response.error().errmsg()); + } + + meta_all_ = std::make_shared(response.meta_all()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << meta_all_->DebugString(); + + return butil::Status::OK(); +} + +butil::Status BackupSdkMeta::Run() { + std::cerr << "Full Backup Sdk Meta " << "<"; + DINGO_LOG(INFO) << "Full Backup Sdk Meta " << "<"; + + std::cerr << "-"; + DINGO_LOG(INFO) << "-"; + + std::cerr << ">" << " 100.00%"; + DINGO_LOG(INFO) << ">" << " 100.00%"; + + std::cout << std::endl; + return butil::Status::OK(); +} + +butil::Status BackupSdkMeta::Backup() { + butil::Status status; + + std::map kvs; + + kvs.emplace(dingodb::Constant::kCoordinatorSdkMetaKeyName, meta_all_->SerializeAsString()); + + rocksdb::Options options; + std::shared_ptr sst = std::make_shared(options); + + std::string file_name = dingodb::Constant::kCoordinatorSdkMetaSstName; + std::string file_path = storage_internal_ + "/" + file_name; + + status = sst->SaveFile(kvs, file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (!backup_sdk_meta_) { + backup_sdk_meta_ = std::make_shared(); + } + + backup_sdk_meta_->set_remark( + "python sdk create tenants, schema, table, index etc. meta info. save dingodb::pb::meta::MetaALL."); + backup_sdk_meta_->set_exec_node(dingodb::Constant::kCoordinatorRegionName); + backup_sdk_meta_->set_dir_name(""); + backup_sdk_meta_->set_file_size(sst->GetSize()); + backup_sdk_meta_->set_encryption(hash_code); + backup_sdk_meta_->set_file_name(file_name); + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << backup_sdk_meta_->DebugString() << std::endl; + } + + return butil::Status::OK(); +} + +std::shared_ptr BackupSdkMeta::GetBackupMeta() { return backup_sdk_meta_; } + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_sdk_meta.h b/src/br/backup_sdk_meta.h new file mode 100644 index 000000000..56286f638 --- /dev/null +++ b/src/br/backup_sdk_meta.h @@ -0,0 +1,59 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_SDK_META_H_ +#define DINGODB_BR_BACKUP_SDK_META_H_ + +#include +#include +#include + +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/meta.pb.h" + +namespace br { + +class BackupSdkMeta : public std::enable_shared_from_this { + public: + BackupSdkMeta(ServerInteractionPtr coordinator_interaction, const std::string &storage_internal); + ~BackupSdkMeta(); + + BackupSdkMeta(const BackupSdkMeta&) = delete; + const BackupSdkMeta& operator=(const BackupSdkMeta&) = delete; + BackupSdkMeta(BackupSdkMeta&&) = delete; + BackupSdkMeta& operator=(BackupSdkMeta&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status GetSdkMetaFromCoordinator(); + + butil::Status Run(); + + butil::Status Backup(); + + std::shared_ptr GetBackupMeta(); + + protected: + private: + ServerInteractionPtr coordinator_interaction_; + std::string storage_internal_; + std::shared_ptr meta_all_; + std::shared_ptr backup_sdk_meta_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_SDK_META_H_ \ No newline at end of file diff --git a/src/br/backup_sql_data.cc b/src/br/backup_sql_data.cc new file mode 100644 index 000000000..77744cc15 --- /dev/null +++ b/src/br/backup_sql_data.cc @@ -0,0 +1,275 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_sql_data.h" + +#include +#include +#include +#include +#include + +#include "common/constant.h" +#include "common/helper.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +#ifndef ENABLE_BACKUP_SQL_DATA_PTHREAD +#define ENABLE_BACKUP_SQL_DATA_PTHREAD +#endif + +// #undef ENABLE_BACKUP_SQL_DATA_PTHREAD + +BackupSqlData::BackupSqlData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : BackupDataBase(coordinator_interaction, store_interaction, index_interaction, document_interaction, backupts, + backuptso_internal, storage, storage_internal, dingodb::Constant::kSqlData) {} + +BackupSqlData::~BackupSqlData() = default; + +std::shared_ptr BackupSqlData::GetSelf() { return shared_from_this(); } + +butil::Status BackupSqlData::Filter() { + if (!wait_for_handle_store_regions_) { + wait_for_handle_store_regions_ = std::make_shared>(); + } + + if (!wait_for_handle_index_regions_) { + wait_for_handle_index_regions_ = std::make_shared>(); + } + + if (!wait_for_handle_document_regions_) { + wait_for_handle_document_regions_ = std::make_shared>(); + } + + for (const auto& region : region_map_->regions()) { + // only handle executor txn region + // remove meta(remove_region_list_) region from region_map_ + if (dingodb::Helper::IsExecutorTxn(region.definition().range().start_key())) { + auto iter = std::find(remove_region_list_.begin(), remove_region_list_.end(), region.id()); + if (iter == remove_region_list_.end()) { + if (dingodb::pb::common::RegionType::STORE_REGION == region.region_type()) { + wait_for_handle_store_regions_->push_back(region); + } else if (dingodb::pb::common::RegionType::INDEX_REGION == region.region_type()) { + wait_for_handle_index_regions_->push_back(region); + } else if (dingodb::pb::common::RegionType::DOCUMENT_REGION == region.region_type()) { + wait_for_handle_document_regions_->push_back(region); + } + } + } + } + + if (FLAGS_br_log_switch_backup_detail) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_store_regions size = " << wait_for_handle_store_regions_->size(); + int i = 0; + std::string s; + for (const auto& region : *wait_for_handle_store_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_store_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + + if (!s.empty()) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_store_regions region id=[" << s << "]"; + } + + s.clear(); + i = 0; + + DINGO_LOG(INFO) << "sql data : wait_for_handle_index_regions size = " << wait_for_handle_index_regions_->size(); + for (const auto& region : *wait_for_handle_index_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_index_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + + if (!s.empty()) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_store_regions region id=[" << s << "]"; + } + + s.clear(); + i = 0; + + DINGO_LOG(INFO) << "sql data : wait_for_handle_document_regions size = " + << wait_for_handle_document_regions_->size(); + for (const auto& region : *wait_for_handle_document_regions_) { + if (0 != i++) { + s += ", "; + } + s += std::to_string(region.id()); + if (i == 10) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_document_regions region id=[" << s << "]"; + s.clear(); + i = 0; + } + } + + if (!s.empty()) { + DINGO_LOG(INFO) << "sql data : wait_for_handle_document_regions region id=[" << s << "]"; + } + } + + return butil::Status::OK(); +} + +butil::Status BackupSqlData::RemoveSqlMeta(const std::vector& meta_region_list) { + remove_region_list_ = meta_region_list; + return butil::Status::OK(); +} + +butil::Status BackupSqlData::Run() { + butil::Status status; + if (!save_store_region_map_) { + save_store_region_map_ = + std::make_shared>(); + } + + if (!save_index_region_map_) { + save_index_region_map_ = + std::make_shared>(); + } + + if (!save_document_region_map_) { + save_document_region_map_ = + std::make_shared>(); + } + + int64_t total_regions_count = wait_for_handle_store_regions_->size() + wait_for_handle_index_regions_->size() + + wait_for_handle_document_regions_->size(); + + // store + status = DoAsyncBackupRegion(store_interaction_, "StoreService", wait_for_handle_store_regions_, + already_handle_store_regions_, save_store_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // index + status = DoAsyncBackupRegion(index_interaction_, "IndexService", wait_for_handle_index_regions_, + already_handle_index_regions_, save_index_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + // document + status = DoAsyncBackupRegion(document_interaction_, "DocumentService", wait_for_handle_document_regions_, + already_handle_document_regions_, save_document_region_map_); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::atomic last_already_handle_regions = 0; + std::cerr << "Full Backup Sql Data " << "<"; + DINGO_LOG(INFO) << "Full Backup Sql Data " << "<"; + std::string s; + while (!is_need_exit_) { + already_handle_regions_ = + already_handle_store_regions_ + already_handle_index_regions_ + already_handle_document_regions_; + + int64_t diff = already_handle_regions_ - last_already_handle_regions; + for (int i = 0; i < diff; i++) { + std::cerr << "-"; + s += "-"; + } + + if (already_handle_regions_ >= total_regions_count) { + break; + } + + last_already_handle_regions.store(already_handle_regions_); + + sleep(1); + } + + if (is_need_exit_) { + return last_error_; + } + + std::cerr << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() + << ",I:" << wait_for_handle_index_regions_->size() << ",D:" << wait_for_handle_document_regions_->size() + << "]"; + DINGO_LOG(INFO) << s; + DINGO_LOG(INFO) << ">" << " 100.00%" << " [" << "S:" << wait_for_handle_store_regions_->size() + << ",I:" << wait_for_handle_index_regions_->size() + << ",D:" << wait_for_handle_document_regions_->size() << "]"; + + std::cout << std::endl; + + DINGO_LOG(INFO) << "backup sql data " << "total_regions : " << already_handle_regions_ + << ", store_regions : " << already_handle_store_regions_ + << ", index_regions : " << already_handle_index_regions_ + << ", document_regions : " << already_handle_document_regions_; + + return butil::Status::OK(); +} + +butil::Status BackupSqlData::DoAsyncBackupRegion( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map) { + std::shared_ptr self = GetSelf(); + auto lambda_call = [self, interaction, service_name, wait_for_handle_regions, &already_handle_regions, + save_region_map]() { + self->DoBackupRegionInternal(interaction, service_name, wait_for_handle_regions, already_handle_regions, + save_region_map); + }; + +#if defined(ENABLE_BACKUP_SQL_DATA_PTHREAD) + std::thread th(lambda_call); + th.detach(); +#else + + std::function* call = new std::function; + *call = lambda_call; + bthread_t th; + + int ret = bthread_start_background( + &th, nullptr, + [](void* arg) -> void* { + auto* call = static_cast*>(arg); + (*call)(); + delete call; + return nullptr; + }, + call); + if (ret != 0) { + DINGO_LOG(ERROR) << fmt::format("bthread_start_background fail"); + return butil::Status(dingodb::pb::error::EINTERNAL, "bthread_start_background fail"); + } +#endif // #if defined(ENABLE_BACKUP_SQL_DATA_PTHREAD) + + return butil::Status::OK(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_sql_data.h b/src/br/backup_sql_data.h new file mode 100644 index 000000000..cdbcfa9a7 --- /dev/null +++ b/src/br/backup_sql_data.h @@ -0,0 +1,64 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_SQL_DATA_H_ +#define DINGODB_BR_BACKUP_SQL_DATA_H_ + +#include +#include +#include + +#include "br/backup_data_base.h" +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" +#include "proto/common.pb.h" + +namespace br { + +class BackupSqlData : public BackupDataBase, public std::enable_shared_from_this { + public: + BackupSqlData(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + ServerInteractionPtr index_interaction, ServerInteractionPtr document_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + ~BackupSqlData() override; + + BackupSqlData(const BackupSqlData&) = delete; + const BackupSqlData& operator=(const BackupSqlData&) = delete; + BackupSqlData(BackupSqlData&&) = delete; + BackupSqlData& operator=(BackupSqlData&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status Filter() override; + + butil::Status RemoveSqlMeta(const std::vector& meta_region_list); + + butil::Status Run() override; + + protected: + private: + butil::Status DoAsyncBackupRegion( + ServerInteractionPtr interaction, const std::string& service_name, + std::shared_ptr> wait_for_handle_regions, + std::atomic& already_handle_regions, + std::shared_ptr> save_region_map); + + std::vector remove_region_list_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_SQL_DATA_H_ \ No newline at end of file diff --git a/src/br/backup_sql_meta.cc b/src/br/backup_sql_meta.cc new file mode 100644 index 000000000..661a26d1e --- /dev/null +++ b/src/br/backup_sql_meta.cc @@ -0,0 +1,73 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/backup_sql_meta.h" + +#include +#include +#include + +#include "br/parameter.h" +#include "common/helper.h" +#include "fmt/core.h" + +namespace br { + +BackupSqlMeta::BackupSqlMeta(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal) + : BackupMetaBase(coordinator_interaction, store_interaction, backupts, backuptso_internal, storage, + storage_internal) {} + +BackupSqlMeta::~BackupSqlMeta() = default; + +std::shared_ptr BackupSqlMeta::GetSelf() { return shared_from_this(); } + +butil::Status BackupSqlMeta::GetSqlMetaRegionFromCoordinator() { // get meta region + dingodb::pb::coordinator::ScanRegionsRequest request; + dingodb::pb::coordinator::ScanRegionsResponse response; + + request.set_key(dingodb::Helper::HexToString("740000000000000000")); + request.set_range_end(dingodb::Helper::HexToString("740000000000000001")); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << request.DebugString(); + + auto status = coordinator_interaction_->SendRequest("CoordinatorService", "ScanRegions", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + return butil::Status(response.error().errcode(), response.error().errmsg()); + } + + scan_region_infos_ = std::make_shared>(); + scan_region_infos_->insert(scan_region_infos_->begin(), response.regions().begin(), response.regions().end()); + + DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << response.DebugString(); + + return butil::Status::OK(); +} + +void BackupSqlMeta::GetSqlMetaRegionList(std::vector& region_list) { + region_list.clear(); + + for (const auto& region_info : *scan_region_infos_) { + region_list.push_back(region_info.region_id()); + } +} + +} // namespace br \ No newline at end of file diff --git a/src/br/backup_sql_meta.h b/src/br/backup_sql_meta.h new file mode 100644 index 000000000..b48d9b287 --- /dev/null +++ b/src/br/backup_sql_meta.h @@ -0,0 +1,54 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_BACKUP_SQL_META_H_ +#define DINGODB_BR_BACKUP_SQL_META_H_ + +#include +#include +#include + +#include "br/backup_meta_base.h" +#include "br/interation.h" +#include "butil/status.h" +#include "fmt/core.h" + +namespace br { + +class BackupSqlMeta : public BackupMetaBase, public std::enable_shared_from_this { + public: + BackupSqlMeta(ServerInteractionPtr coordinator_interaction, ServerInteractionPtr store_interaction, + const std::string& backupts, int64_t backuptso_internal, const std::string& storage, + const std::string& storage_internal); + ~BackupSqlMeta() override; + + BackupSqlMeta(const BackupSqlMeta&) = delete; + const BackupSqlMeta& operator=(const BackupSqlMeta&) = delete; + BackupSqlMeta(BackupSqlMeta&&) = delete; + BackupSqlMeta& operator=(BackupSqlMeta&&) = delete; + + std::shared_ptr GetSelf(); + + butil::Status GetSqlMetaRegionFromCoordinator(); + + void GetSqlMetaRegionList(std::vector& region_list); + + protected: + private: + std::shared_ptr> scan_region_infos_; +}; + +} // namespace br + +#endif // DINGODB_BR_BACKUP_SQL_META_H_ \ No newline at end of file diff --git a/src/br/br.cc b/src/br/br.cc deleted file mode 100644 index d96a93fc4..000000000 --- a/src/br/br.cc +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "br/br.h" - -#include -#include - -namespace br { - -void Dummy() { return; } -} // namespace br \ No newline at end of file diff --git a/src/br/br.h b/src/br/br.h deleted file mode 100644 index 5d398f917..000000000 --- a/src/br/br.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef DINGODB_BR_BR_INTERATION_H_ -#define DINGODB_BR_BR_INTERATION_H_ - -#include -#include -#include -#include - -namespace br { - -void Dummy(); - -} // namespace br - -#endif // DINGODB_BR_BR_INTERATION_H_ \ No newline at end of file diff --git a/src/br/helper.cc b/src/br/helper.cc index d62bcf496..1071b98df 100644 --- a/src/br/helper.cc +++ b/src/br/helper.cc @@ -14,8 +14,34 @@ #include "br/helper.h" +#include +#include + +#include "butil/strings/string_split.h" + namespace br { +std::string Helper::Ltrim(const std::string& s, const std::string& delete_str) { + size_t start = s.find_first_not_of(delete_str); + return (start == std::string::npos) ? "" : s.substr(start); +} + +std::string Helper::Rtrim(const std::string& s, const std::string& delete_str) { + size_t end = s.find_last_not_of(delete_str); + return (end == std::string::npos) ? "" : s.substr(0, end + 1); +} + +std::string Helper::Trim(const std::string& s, const std::string& delete_str) { + return Rtrim(Ltrim(s, delete_str), delete_str); +} + +int Helper::GetRandInt() { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution distrib(1, 1000000000); + return distrib(gen); +} + std::vector Helper::StringToEndpoints(const std::string& str) { std::vector addrs; butil::SplitString(str, ',', &addrs); @@ -47,4 +73,19 @@ std::vector Helper::VectorToEndpoints(std::vector return endpoints; } +std::vector Helper::GetAddrsFromFile(const std::string& path) { + std::vector addrs; + + std::ifstream input(path); + for (std::string line; getline(input, line);) { + if (line.find('#') != std::string::npos) { + continue; + } + + addrs.push_back(Trim(line, " ")); + } + + return addrs; +} + } // namespace br diff --git a/src/br/helper.h b/src/br/helper.h index e0c2cafd2..222baf034 100644 --- a/src/br/helper.h +++ b/src/br/helper.h @@ -15,599 +15,30 @@ #ifndef DINGODB_BR_HELPER_H_ #define DINGODB_BR_HELPER_H_ -#include - -#include #include -#include -#include -#include #include #include -#include "bthread/bthread.h" #include "butil/endpoint.h" -#include "butil/strings/string_split.h" -#include "common/constant.h" -#include "common/helper.h" -#include "common/logging.h" -#include "coordinator/coordinator_interaction.h" -#include "document/codec.h" #include "fmt/core.h" -#include "proto/debug.pb.h" -#include "serial/buf.h" -#include "vector/codec.h" namespace br { -#if 0 - -const char kAlphabet[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}; - -const char kAlphabetV2[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y'}; - -const int kGroupSize = 8; -const int kPadGroupSize = 9; -const uint8_t kMarker = 255; - -class Bthread { - public: - template - Bthread(const bthread_attr_t* attr, Fn&& fn, Args&&... args) { // NOLINT - auto p_wrap_fn = new auto([=] { fn(args...); }); - auto call_back = [](void* ar) -> void* { - auto f = reinterpret_cast(ar); - (*f)(); - delete f; - return nullptr; - }; - - bthread_start_background(&th_, attr, call_back, (void*)p_wrap_fn); - joinable_ = true; - } - - void Join() { - if (joinable_) { - bthread_join(th_, nullptr); - joinable_ = false; - } - } - - bool Joinable() const noexcept { return joinable_; } - - bthread_t GetId() const { return th_; } - - private: - bthread_t th_; - bool joinable_ = false; -}; - -class CoordinatorInteraction { - private: - CoordinatorInteraction() = default; - ~CoordinatorInteraction() = default; - - dingodb::CoordinatorInteractionPtr coordinator_interaction_; - dingodb::CoordinatorInteractionPtr coordinator_interaction_meta_; - dingodb::CoordinatorInteractionPtr coordinator_interaction_version_; - - public: - static CoordinatorInteraction& GetInstance() { - static CoordinatorInteraction instance; - return instance; - } - CoordinatorInteraction(const CoordinatorInteraction&) = delete; - CoordinatorInteraction& operator=(const CoordinatorInteraction) = delete; - void SetCoorinatorInteraction(dingodb::CoordinatorInteractionPtr interaction) { - coordinator_interaction_ = interaction; - } - void SetCoorinatorInteractionMeta(dingodb::CoordinatorInteractionPtr interaction) { - coordinator_interaction_meta_ = interaction; - } - void SetCoorinatorInteractionVersion(dingodb::CoordinatorInteractionPtr interaction) { - coordinator_interaction_version_ = interaction; - } - dingodb::CoordinatorInteractionPtr GetCoorinatorInteraction() { return coordinator_interaction_; } - dingodb::CoordinatorInteractionPtr GetCoorinatorInteractionMeta() { return coordinator_interaction_meta_; } - dingodb::CoordinatorInteractionPtr GetCoorinatorInteractionVersion() { return coordinator_interaction_version_; } -}; - -#endif - class Helper { public: -#if 0 - static std::string ToUpperCase(const std::string& input) { - std::string output = input; - std::transform(output.begin(), output.end(), output.begin(), [](unsigned char c) { return std::toupper(c); }); - return output; - } - static std::string Ltrim(const std::string& s, const std::string& delete_str) { - size_t start = s.find_first_not_of(delete_str); - return (start == std::string::npos) ? "" : s.substr(start); - } - - static std::string Rtrim(const std::string& s, const std::string& delete_str) { - size_t end = s.find_last_not_of(delete_str); - return (end == std::string::npos) ? "" : s.substr(0, end + 1); - } - - static std::string Trim(const std::string& s, const std::string& delete_str) { - return Rtrim(Ltrim(s, delete_str), delete_str); - } - - static int GetRandInt() { - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution distrib(1, 1000000000); - return distrib(gen); - } + static std::string Ltrim(const std::string& s, const std::string& delete_str); - // rand string - static std::string GenRandomString(int len) { - std::string result; - int alphabet_len = sizeof(kAlphabet); + static std::string Rtrim(const std::string& s, const std::string& delete_str); - std::mt19937 rng; - rng.seed(std::random_device()()); - std::uniform_int_distribution distrib(1, 1000000000); - for (int i = 0; i < len; ++i) { - result.append(1, kAlphabet[distrib(rng) % alphabet_len]); - } + static std::string Trim(const std::string& s, const std::string& delete_str); - return result; - } - - static std::string GenRandomStringV2(int len) { - std::string result; - int alphabet_len = sizeof(kAlphabetV2); - - std::mt19937 rng; - rng.seed(std::random_device()()); - std::uniform_int_distribution distrib(1, 1000000000); - for (int i = 0; i < len; ++i) { - result.append(1, kAlphabetV2[distrib(rng) % alphabet_len]); - } - - return result; - } - - static std::vector GenKeys(int nums) { - std::vector vec; - vec.reserve(nums); - for (int i = 0; i < nums; ++i) { - vec.push_back(GenRandomString(4)); - } - - return vec; - } - - static std::map GenDataset(const std::string& prefix, int n) { - std::map dataset; - - for (int i = 0; i < n; ++i) { - std::string key = prefix + GenRandomStringV2(32); - dataset[key] = GenRandomString(256); - } - - return dataset; - } - -#endif + static int GetRandInt(); static std::vector StringToEndpoints(const std::string& str); static std::vector VectorToEndpoints(std::vector addrs); -#if 0 - - static bool RandomChoice() { return GetRandInt() % 2 == 0; } - - static std::vector GetAddrsFromFile(const std::string& path) { - std::vector addrs; - - std::ifstream input(path); - for (std::string line; getline(input, line);) { - if (line.find('#') != std::string::npos) { - continue; - } - - addrs.push_back(Trim(line, " ")); - } - - return addrs; - } - - static std::string EncodeRegionRange(int64_t partition_id) { - dingodb::Buf buf(9); - buf.Write(dingodb::Constant::kClientRaw); - buf.WriteLong(partition_id); - - return buf.GetString(); - } - - static std::string CalculateVectorMiddleKey(const std::string& start_key, const std::string& end_key) { - int64_t partition_id = dingodb::VectorCodec::UnPackagePartitionId(start_key); - int64_t min_vector_id = dingodb::VectorCodec::UnPackageVectorId(start_key); - int64_t max_vector_id = dingodb::VectorCodec::UnPackageVectorId(end_key); - max_vector_id = max_vector_id > 0 ? max_vector_id : INT64_MAX; - int64_t mid_vector_id = min_vector_id + (max_vector_id - min_vector_id) / 2; - - DINGO_LOG(INFO) << "mid_vector_id: " << mid_vector_id; - std::string result = dingodb::VectorCodec::PackageVectorKey(start_key[0], partition_id, mid_vector_id); - return result; - } - - static std::string CalculateDocumentMiddleKey(const std::string& start_key, const std::string& end_key) { - int64_t partition_id = dingodb::DocumentCodec::UnPackagePartitionId(start_key); - int64_t min_document_id = dingodb::DocumentCodec::UnPackageDocumentId(start_key); - int64_t max_document_id = dingodb::DocumentCodec::UnPackageDocumentId(end_key); - max_document_id = max_document_id > 0 ? max_document_id : INT64_MAX; - int64_t mid_document_id = min_document_id + (max_document_id - min_document_id) / 2; - - DINGO_LOG(INFO) << "mid_document_id: " << mid_document_id; - std::string result = dingodb::DocumentCodec::PackageDocumentKey(start_key[0], partition_id, mid_document_id); - return result; - } - - // format and print - static std::string FormatVectorData(const dingodb::pb::common::Vector& vector) { - if (vector.float_values_size() > 0) { - return fmt::format("{}/[{} {}...]", vector.float_values_size(), vector.float_values().at(0), - vector.float_values().at(1)); - } else if (vector.binary_values_size() > 0) { - return fmt::format("{}/[{} {}...]", vector.binary_values_size(), vector.binary_values().at(0), - vector.binary_values().at(1)); - } else { - return "no data"; - } - } - - static std::vector FormatVectorScalar(const dingodb::pb::common::VectorScalardata& scalar) { - auto format_scalar_func = [](const dingodb::pb::common::ScalarValue& value) -> std::string { - std::string str = "["; - switch (value.field_type()) { - case dingodb::pb::common::ScalarFieldType::BOOL: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.bool_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::INT8: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.int_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::INT16: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.int_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::INT32: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.int_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::INT64: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.long_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::FLOAT32: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.float_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::DOUBLE: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.double_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::STRING: - for (const auto& field : value.fields()) { - str.append(fmt::format("{},", field.string_data())); - } - str.pop_back(); - break; - case dingodb::pb::common::ScalarFieldType::BYTES: - for (const auto& field : value.fields()) { - str.append(field.bytes_data()); - str.push_back(','); - } - str.pop_back(); - break; - default: - break; - } - - if (str.size() > 32) { - str = str.substr(0, 32); - } - - str.push_back(']'); - return str; - }; - - std::vector result; - for (const auto& [key, value] : scalar.scalar_data()) { - result.push_back(fmt::format("{}: {}", key, format_scalar_func(value))); - } - - return result; - } - - static std::string FormatVectorTable(const dingodb::pb::common::VectorTableData& table) { - return fmt::format("{}:{}", dingodb::Helper::StringToHex(table.table_key()), - dingodb::Helper::StringToHex(table.table_value())); - } - - static std::vector FormatDocument(const dingodb::pb::common::Document& document) { - auto format_scalar_func = [](const dingodb::pb::common::DocumentValue& value) -> std::string { - switch (value.field_type()) { - case dingodb::pb::common::ScalarFieldType::BOOL: - return fmt::format("{}", value.field_value().bool_data()); - case dingodb::pb::common::ScalarFieldType::INT8: - return fmt::format("{}", value.field_value().int_data()); - case dingodb::pb::common::ScalarFieldType::INT16: - return fmt::format("{}", value.field_value().int_data()); - case dingodb::pb::common::ScalarFieldType::INT32: - return fmt::format("{}", value.field_value().int_data()); - case dingodb::pb::common::ScalarFieldType::INT64: - return fmt::format("{}", value.field_value().long_data()); - case dingodb::pb::common::ScalarFieldType::FLOAT32: - return fmt::format("{}", value.field_value().float_data()); - case dingodb::pb::common::ScalarFieldType::DOUBLE: - return fmt::format("{}", value.field_value().double_data()); - case dingodb::pb::common::ScalarFieldType::STRING: - return fmt::format("{}", value.field_value().string_data()); - case dingodb::pb::common::ScalarFieldType::BYTES: - return fmt::format("{}", value.field_value().bytes_data()); - default: - return "Unknown Type"; - } - return ""; - }; - - std::vector result; - for (const auto& [key, value] : document.document_data()) { - result.push_back(fmt::format("{}: {}", key, format_scalar_func(value))); - } - - return result; - } - - static dingodb::pb::common::Engine GetEngine(const std::string& engine_name) { - if (engine_name == "rocksdb") { - return dingodb::pb::common::Engine::ENG_ROCKSDB; - } else if (engine_name == "bdb") { - return dingodb::pb::common::Engine::ENG_BDB; - } else { - DINGO_LOG(FATAL) << "engine_name is illegal, please input -engine=[rocksdb, bdb]"; - } - } - - static int GetCreateTableIds(dingodb::CoordinatorInteractionPtr coordinator_interaction, int64_t count, - std::vector& table_ids) { - dingodb::pb::meta::CreateTableIdsRequest request; - dingodb::pb::meta::CreateTableIdsResponse response; - - auto* schema_id = request.mutable_schema_id(); - schema_id->set_entity_type(::dingodb::pb::meta::EntityType::ENTITY_TYPE_SCHEMA); - schema_id->set_entity_id(::dingodb::pb::meta::ReservedSchemaIds::DINGO_SCHEMA); - schema_id->set_parent_entity_id(::dingodb::pb::meta::ReservedSchemaIds::ROOT_SCHEMA); - - request.set_count(count); - - auto status = coordinator_interaction->SendRequest("CreateTableIds", request, response); - DINGO_LOG(INFO) << "SendRequest status=" << status; - DINGO_LOG(INFO) << response.DebugString(); - - if (response.table_ids_size() > 0) { - for (const auto& id : response.table_ids()) { - table_ids.push_back(id.entity_id()); - } - return 0; - } else { - return -1; - } - } - - static int SetUp(std::string url) { - if (url.empty()) { - url = "file://./coor_list"; - } - - if (!url.empty()) { - std::string path = url; - path = path.replace(path.find("file://"), 7, ""); - auto addrs = Helper::GetAddrsFromFile(path); - if (addrs.empty()) { - std::cout << "coor_url not find addr, path=" << path << std::endl; - return -1; - } - - auto coordinator_interaction = std::make_shared(); - if (!coordinator_interaction->Init(addrs)) { - std::cout << "Fail to init coordinator_interaction, please check parameter --url=" << url << std::endl; - return -1; - } - - InteractionManager::GetInstance().SetCoorinatorInteraction(coordinator_interaction); - } - - // this is for legacy coordinator_client use, will be removed in the future - if (!url.empty()) { - auto coordinator_interaction = std::make_shared(); - if (!coordinator_interaction->InitByNameService( - url, dingodb::pb::common::CoordinatorServiceType::ServiceTypeCoordinator)) { - std::cout << "Fail to init coordinator_interaction, please check parameter --coor_url=" << url << std::endl; - return -1; - } - CoordinatorInteraction::GetInstance().SetCoorinatorInteraction(coordinator_interaction); - auto coordinator_interaction_meta = std::make_shared(); - if (!coordinator_interaction_meta->InitByNameService( - url, dingodb::pb::common::CoordinatorServiceType::ServiceTypeMeta)) { - std::cout << "Fail to init coordinator_interaction_meta, please check parameter --coor_url=" << url - << std::endl; - return -1; - } - CoordinatorInteraction::GetInstance().SetCoorinatorInteractionMeta(coordinator_interaction_meta); - auto coordinator_interaction_version = std::make_shared(); - if (!coordinator_interaction_version->InitByNameService( - url, dingodb::pb::common::CoordinatorServiceType::ServiceTypeVersion)) { - std::cout << "Fail to init coordinator_interaction_version, please check parameter --coor_url=" << url - << std::endl; - return -1; - } - CoordinatorInteraction::GetInstance().SetCoorinatorInteractionVersion(coordinator_interaction_version); - } - return 0; - } - - static dingodb::pb::common::RawEngine GetRawEngine(const std::string& engine_name) { - if (engine_name == "rocksdb") { - return dingodb::pb::common::RawEngine::RAW_ENG_ROCKSDB; - } else if (engine_name == "bdb") { - return dingodb::pb::common::RawEngine::RAW_ENG_BDB; - } else if (engine_name == "xdp") { - return dingodb::pb::common::RawEngine::RAW_ENG_XDPROCKS; - } else { - DINGO_LOG(FATAL) << "raw_engine_name is illegal, please input -raw-engine=[rocksdb, bdb]"; - } - - return dingodb::pb::common::RawEngine::RAW_ENG_ROCKSDB; - } - - static int GetCreateTableId(dingodb::CoordinatorInteractionPtr coordinator_interaction, int64_t& table_id) { - dingodb::pb::meta::CreateTableIdRequest request; - dingodb::pb::meta::CreateTableIdResponse response; - - auto* schema_id = request.mutable_schema_id(); - schema_id->set_entity_type(::dingodb::pb::meta::EntityType::ENTITY_TYPE_SCHEMA); - schema_id->set_entity_id(::dingodb::pb::meta::ReservedSchemaIds::DINGO_SCHEMA); - schema_id->set_parent_entity_id(::dingodb::pb::meta::ReservedSchemaIds::ROOT_SCHEMA); - - auto status = coordinator_interaction->SendRequest("CreateTableId", request, response); - DINGO_LOG(INFO) << "SendRequest status=" << status; - DINGO_LOG(INFO) << response.DebugString(); - - if (response.has_table_id()) { - table_id = response.table_id().entity_id(); - return 0; - } else { - return -1; - } - } - - static int DecodeBytes(const std::string_view& encode_key, std::string& output) { - output.reserve(256); - - const auto* data = encode_key.data(); - for (int i = 0; i < encode_key.size(); i++) { - uint8_t marker = encode_key.at(i + 8); - - int pad_count = kMarker - marker; - for (int j = 0; j < kGroupSize - pad_count; ++j) { - output.push_back(encode_key.at(i++)); - } - - if (pad_count != 0) { - for (int j = 0; j < pad_count; ++j) { - if (encode_key.at(i++) != 0) { - return -1; - } - } - - return i; - } - } - - return -1; - } - - static void PrintIndexRange(dingodb::pb::meta::IndexRange& index_range) { // NOLINT - DINGO_LOG(INFO) << "refresh route..."; - for (const auto& item : index_range.range_distribution()) { - DINGO_LOG(INFO) << fmt::format("region {} range [{}-{})", item.id().entity_id(), - dingodb::Helper::StringToHex(item.range().start_key()), - dingodb::Helper::StringToHex(item.range().end_key())); - } - } - - static dingodb::pb::common::Range DecodeRangeToPlaintext(const dingodb::pb::common::Region& region) { - const dingodb::pb::common::RegionDefinition& region_definition = region.definition(); - const auto& origin_range = region_definition.range(); - - dingodb::pb::common::Range plaintext_range; - if (region.region_type() == dingodb::pb::common::INDEX_REGION && - region.definition().index_parameter().has_vector_index_parameter()) { - int64_t min_vector_id = 0, max_vector_id = 0; - dingodb::VectorCodec::DecodeRangeToVectorId(false, origin_range, min_vector_id, max_vector_id); - - plaintext_range.set_start_key(fmt::format("{}/{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.start_key()), - dingodb::VectorCodec::UnPackagePartitionId(origin_range.start_key()), - min_vector_id)); - - plaintext_range.set_end_key(fmt::format("{}/{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.end_key()), - dingodb::VectorCodec::UnPackagePartitionId(origin_range.end_key()), - max_vector_id)); - - } else if (region.region_type() == dingodb::pb::common::DOCUMENT_REGION && - region.definition().index_parameter().has_document_index_parameter()) { - int64_t min_document_id = 0, max_document_id = 0; - dingodb::DocumentCodec::DecodeRangeToDocumentId(false, origin_range, min_document_id, max_document_id); - - plaintext_range.set_start_key(fmt::format("{}/{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.start_key()), - dingodb::DocumentCodec::UnPackagePartitionId(origin_range.start_key()), - min_document_id)); - - plaintext_range.set_end_key(fmt::format("{}/{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.end_key()), - dingodb::DocumentCodec::UnPackagePartitionId(origin_range.end_key()), - max_document_id)); - - } else if (dingodb::Helper::IsExecutorRaw(origin_range.start_key()) || - dingodb::Helper::IsExecutorTxn(origin_range.start_key())) { - } else { - std::string start_key = origin_range.start_key().substr(1, origin_range.start_key().size()); - plaintext_range.set_start_key(fmt::format("{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.start_key()), - dingodb::Helper::StringToHex(start_key))); - - std::string end_key = origin_range.end_key().substr(1, origin_range.end_key().size()); - plaintext_range.set_end_key(fmt::format("{}/{}", dingodb::Helper::GetKeyPrefix(origin_range.end_key()), - dingodb::Helper::StringToHex(end_key))); - } - return plaintext_range; - } - - static bool IsDateString(const std::string& date_str) { - struct tm time_struct = {0}; - return strptime(date_str.c_str(), "%Y-%m-%d", &time_struct) != nullptr; - } - - static int64_t StringToTimestamp(const std::string& date_str) { - struct tm time_struct = {0}; - if (!strptime(date_str.c_str(), "%Y-%m-%d", &time_struct)) { - DINGO_LOG(ERROR) << "Error converting date string to tm structure!" << std::endl; - return -1; - } - - time_t timestamp = std::mktime(&time_struct); - if (timestamp <= 0) { - DINGO_LOG(ERROR) << fmt::format("mktime failed, timestamp: {}", timestamp); - return -1; - } - // add 8 hour - return static_cast(timestamp + 28800); - } -#endif + static std::vector GetAddrsFromFile(const std::string& path); }; } // namespace br diff --git a/src/br/interaction_manager.cc b/src/br/interaction_manager.cc index 0788e7354..10bea8420 100644 --- a/src/br/interaction_manager.cc +++ b/src/br/interaction_manager.cc @@ -29,173 +29,114 @@ InteractionManager& InteractionManager::GetInstance() { } void InteractionManager::SetCoordinatorInteraction(ServerInteractionPtr interaction) { + BAIDU_SCOPED_LOCK(mutex_); coordinator_interaction_ = interaction; } -void InteractionManager::SetStoreInteraction(ServerInteractionPtr interaction) { store_interaction_ = interaction; } +void InteractionManager::SetStoreInteraction(ServerInteractionPtr interaction) { + BAIDU_SCOPED_LOCK(mutex_); + store_interaction_ = interaction; +} -void InteractionManager::SetIndexInteraction(ServerInteractionPtr interaction) { index_interaction_ = interaction; } +void InteractionManager::SetIndexInteraction(ServerInteractionPtr interaction) { + BAIDU_SCOPED_LOCK(mutex_); + index_interaction_ = interaction; +} void InteractionManager::SetDocumentInteraction(ServerInteractionPtr interaction) { + BAIDU_SCOPED_LOCK(mutex_); document_interaction_ = interaction; } -ServerInteractionPtr InteractionManager::GetCoordinatorInteraction() const { return coordinator_interaction_; } -ServerInteractionPtr InteractionManager::GetStoreInteraction() const { return store_interaction_; } -ServerInteractionPtr InteractionManager::GetIndexInteraction() const { return index_interaction_; } -ServerInteractionPtr InteractionManager::GetDocumentInteraction() const { return document_interaction_; } - -bool InteractionManager::CreateStoreInteraction(std::vector addrs) { - auto interaction = std::make_shared(); - if (!interaction->Init(addrs)) { - DINGO_LOG(ERROR) << "Fail to init store_interaction"; - return false; - } - +ServerInteractionPtr InteractionManager::GetCoordinatorInteraction() { + ServerInteractionPtr interaction; { BAIDU_SCOPED_LOCK(mutex_); - if (store_interaction_ == nullptr) { - store_interaction_ = interaction; - } + interaction = coordinator_interaction_; } - - return true; + return interaction; } - -butil::Status InteractionManager::CreateStoreInteraction(int64_t region_id) { - auto region_entry = RegionRouter::GetInstance().QueryRegionEntry(region_id); - if (region_entry == nullptr) { - std::string s = fmt::format("not found store region entry {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EREGION_NOT_FOUND, s); - } - - if (dingodb::pb::common::RegionType::STORE_REGION != region_entry->Region().region_type()) { - std::string s = fmt::format("region : {} is not a store region. region_type : {}", region_id, - dingodb::pb::common::RegionType_Name(region_entry->Region().region_type())); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); - } - - if (!CreateStoreInteraction(region_entry->GetAddrs())) { - std::string s = fmt::format("init store interaction failed, region {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); +ServerInteractionPtr InteractionManager::GetStoreInteraction() { + ServerInteractionPtr interaction; + { + BAIDU_SCOPED_LOCK(mutex_); + interaction = store_interaction_; } - - return butil::Status(); + return interaction; } - -bool InteractionManager::CreateIndexInteraction(std::vector addrs) { - auto interaction = std::make_shared(); - if (!interaction->Init(addrs)) { - DINGO_LOG(ERROR) << "Fail to init index_interaction"; - return false; - } - +ServerInteractionPtr InteractionManager::GetIndexInteraction() { + ServerInteractionPtr interaction; { BAIDU_SCOPED_LOCK(mutex_); - if (index_interaction_ == nullptr) { - index_interaction_ = interaction; - } + interaction = index_interaction_; } - - return true; + return interaction; } -butil::Status InteractionManager::CreateIndexInteraction(int64_t region_id) { - auto region_entry = RegionRouter::GetInstance().QueryRegionEntry(region_id); - if (region_entry == nullptr) { - std::string s = fmt::format("not found index region entry {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EREGION_NOT_FOUND, s); - } - - if (dingodb::pb::common::RegionType::INDEX_REGION != region_entry->Region().region_type()) { - std::string s = fmt::format("region : {} is not a index region. region_type : {}", region_id, - dingodb::pb::common::RegionType_Name(region_entry->Region().region_type())); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); - } - - if (!CreateIndexInteraction(region_entry->GetAddrs())) { - std::string s = fmt::format("init index interaction failed, region {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); +ServerInteractionPtr InteractionManager::GetDocumentInteraction() { + ServerInteractionPtr interaction; + { + BAIDU_SCOPED_LOCK(mutex_); + interaction = document_interaction_; } - - return butil::Status(); + return interaction; } -bool InteractionManager::CreateDocumentInteraction(std::vector addrs) { - auto interaction = std::make_shared(); - if (!interaction->Init(addrs)) { - DINGO_LOG(ERROR) << "Fail to init document_interaction_" << std::endl; - return false; - } - +std::pair InteractionManager::CloneCoordinatorInteraction() { + ServerInteractionPtr interaction; + std::vector addrs; { BAIDU_SCOPED_LOCK(mutex_); - if (document_interaction_ == nullptr) { - document_interaction_ = interaction; - } + addrs = coordinator_interaction_->GetAddrs(); } - return true; + return CreateInteraction(addrs); } - -butil::Status InteractionManager::CreateDocumentInteraction(int64_t region_id) { - auto region_entry = RegionRouter::GetInstance().QueryRegionEntry(region_id); - if (region_entry == nullptr) { - std::string s = fmt::format("not found document region entry {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EREGION_NOT_FOUND, s); - } - - if (dingodb::pb::common::RegionType::DOCUMENT_REGION != region_entry->Region().region_type()) { - std::string s = fmt::format("region : {} is not a document region. region_type : {}", region_id, - dingodb::pb::common::RegionType_Name(region_entry->Region().region_type())); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); - } - - if (!CreateDocumentInteraction(region_entry->GetAddrs())) { - std::string s = fmt::format("init document interaction failed, region {}", region_id); - DINGO_LOG(ERROR) << s; - return butil::Status(dingodb::pb::error::EINTERNAL, s); +std::pair InteractionManager::CloneStoreInteraction() { + ServerInteractionPtr interaction; + std::vector addrs; + { + BAIDU_SCOPED_LOCK(mutex_); + addrs = store_interaction_->GetAddrs(); } - return butil::Status(); + return CreateInteraction(addrs); } - -int64_t InteractionManager::GetCoordinatorInteractionLatency() const { - if (coordinator_interaction_ == nullptr) { - return 0; +std::pair InteractionManager::CloneIndexInteraction() { + ServerInteractionPtr interaction; + std::vector addrs; + { + BAIDU_SCOPED_LOCK(mutex_); + addrs = index_interaction_->GetAddrs(); } - return coordinator_interaction_->GetLatency(); + return CreateInteraction(addrs); } - -int64_t InteractionManager::GetStoreInteractionLatency() const { - if (store_interaction_ == nullptr) { - return 0; +std::pair InteractionManager::CloneDocumentInteraction() { + ServerInteractionPtr interaction; + std::vector addrs; + { + BAIDU_SCOPED_LOCK(mutex_); + addrs = document_interaction_->GetAddrs(); } - return store_interaction_->GetLatency(); + return CreateInteraction(addrs); } -int64_t InteractionManager::GetIndexInteractionLatency() const { - if (index_interaction_ == nullptr) { - return 0; - } +std::pair InteractionManager::CreateInteraction( + const std::vector& addrs) { + butil::Status status; - return index_interaction_->GetLatency(); -} + ServerInteractionPtr interaction = std::make_shared(); -int64_t InteractionManager::GetDocumentInteractionLatency() const { - if (document_interaction_ == nullptr) { - return 0; + if (!interaction->Init(addrs)) { + std::string s = fmt::format("Fail to init interaction, addrs"); + for (const auto& addr : addrs) { + s += fmt::format(" {}", addr); + } + DINGO_LOG(ERROR) << s; + status = butil::Status(dingodb::pb::error::EINTERNAL, s); + return {status, nullptr}; } - - return document_interaction_->GetLatency(); + return {butil::Status::OK(), interaction}; } } // namespace br \ No newline at end of file diff --git a/src/br/interaction_manager.h b/src/br/interaction_manager.h index fe25329e6..5475bddda 100644 --- a/src/br/interaction_manager.h +++ b/src/br/interaction_manager.h @@ -15,8 +15,10 @@ #ifndef DINGODB_BR_INTERATION_MANAGER_H_ #define DINGODB_BR_INTERATION_MANAGER_H_ +#include + #include "br/interation.h" -#include "br/router.h" +#include "butil/status.h" namespace br { class InteractionManager { @@ -28,47 +30,19 @@ class InteractionManager { void SetIndexInteraction(ServerInteractionPtr interaction); void SetDocumentInteraction(ServerInteractionPtr interaction); - ServerInteractionPtr GetCoordinatorInteraction() const; - ServerInteractionPtr GetStoreInteraction() const; - ServerInteractionPtr GetIndexInteraction() const; - ServerInteractionPtr GetDocumentInteraction() const; - - bool CreateStoreInteraction(std::vector addrs); - butil::Status CreateStoreInteraction(int64_t region_id); - - bool CreateIndexInteraction(std::vector addrs); - butil::Status CreateIndexInteraction(int64_t region_id); - - bool CreateDocumentInteraction(std::vector addrs); - butil::Status CreateDocumentInteraction(int64_t region_id); - - void ResetCoordinatorInteraction() { coordinator_interaction_.reset(); } - void ResetStoreInteraction() { store_interaction_.reset(); } - void ResetIndexInteraction() { index_interaction_.reset(); } - void ResetDocumentInteraction() { document_interaction_.reset(); } - - int64_t GetCoordinatorInteractionLatency() const; - int64_t GetStoreInteractionLatency() const; - int64_t GetIndexInteractionLatency() const; - int64_t GetDocumentInteractionLatency() const; + ServerInteractionPtr GetCoordinatorInteraction(); + ServerInteractionPtr GetStoreInteraction(); + ServerInteractionPtr GetIndexInteraction(); + ServerInteractionPtr GetDocumentInteraction(); - template - butil::Status SendRequestWithoutContext(const std::string& service_name, const std::string& api_name, - const Request& request, Response& response); - - template - butil::Status SendRequestWithContext(const std::string& service_name, const std::string& api_name, Request& request, - Response& response); - - template - butil::Status AllSendRequestWithoutContext(const std::string& service_name, const std::string& api_name, - const Request& request, Response& response); - - template - butil::Status AllSendRequestWithContext(const std::string& service_name, const std::string& api_name, - const Request& request, Response& response); + std::pair CloneCoordinatorInteraction(); + std::pair CloneStoreInteraction(); + std::pair CloneIndexInteraction(); + std::pair CloneDocumentInteraction(); private: + static std::pair CreateInteraction(const std::vector& addrs); + InteractionManager(); ~InteractionManager(); @@ -80,99 +54,6 @@ class InteractionManager { bthread_mutex_t mutex_; }; -template -butil::Status InteractionManager::SendRequestWithoutContext(const std::string& service_name, - const std::string& api_name, const Request& request, - Response& response) { - if (service_name == "UtilService" || service_name == "DebugService") { - if (store_interaction_ == nullptr) { - DINGO_LOG(ERROR) << "Store interaction is nullptr."; - return butil::Status(dingodb::pb::error::EINTERNAL, "Store interaction is nullptr."); - } - return store_interaction_->SendRequest(service_name, api_name, request, response); - } - return coordinator_interaction_->SendRequest(service_name, api_name, request, response); - - - - - -} - -template -butil::Status InteractionManager::SendRequestWithContext(const std::string& service_name, const std::string& api_name, - Request& request, Response& response) { - if (store_interaction_ == nullptr) { - auto status = CreateStoreInteraction(request.context().region_id()); - if (!status.ok()) { - return status; - } - } - - for (;;) { - auto status = store_interaction_->SendRequest(service_name, api_name, request, response); - if (status.ok()) { - return status; - } - - if (response.error().errcode() == dingodb::pb::error::EREGION_VERSION) { - RegionRouter::GetInstance().UpdateRegionEntry(response.error().store_region_info()); - DINGO_LOG(INFO) << "QueryRegionEntry region_id: " << request.context().region_id(); - auto region_entry = RegionRouter::GetInstance().QueryRegionEntry(request.context().region_id()); - if (region_entry == nullptr) { - return butil::Status(dingodb::pb::error::EREGION_NOT_FOUND, "Not found region %lu", - request.context().region_id()); - } - *request.mutable_context() = region_entry->GenConext(); - } else { - return status; - } - bthread_usleep(1000 * 500); - } -} - -template -butil::Status InteractionManager::AllSendRequestWithoutContext(const std::string& service_name, - const std::string& api_name, const Request& request, - Response& response) { - if (store_interaction_ == nullptr) { - return butil::Status(dingodb::pb::error::EINTERNAL, "Store interaction is nullptr."); - } - - return store_interaction_->AllSendRequest(service_name, api_name, request, response); -} - -template -butil::Status InteractionManager::AllSendRequestWithContext(const std::string& service_name, - const std::string& api_name, const Request& request, - Response& response) { - if (store_interaction_ == nullptr) { - auto status = CreateStoreInteraction(request.context().region_id()); - if (!status.ok()) { - return status; - } - } - - for (;;) { - auto status = store_interaction_->AllSendRequest(service_name, api_name, request, response); - if (status.ok()) { - return status; - } - - if (response.error().errcode() == dingodb::pb::error::EREGION_VERSION) { - RegionRouter::GetInstance().UpdateRegionEntry(response.error().store_region_info()); - auto region_entry = RegionRouter::GetInstance().QueryRegionEntry(request.context().region_id()); - if (region_entry == nullptr) { - return butil::Status(dingodb::pb::error::EREGION_NOT_FOUND, "Not found region %lu", - request.context().region_id()); - } - *request.mutable_context() = region_entry->GenConext(); - } else { - return status; - } - bthread_usleep(1000 * 500); - } -} } // namespace br #endif // DINGODB_BR_INTERATION_MANAGER_H_ \ No newline at end of file diff --git a/src/br/interation.cc b/src/br/interation.cc index 4cf6a4671..79889ed94 100644 --- a/src/br/interation.cc +++ b/src/br/interation.cc @@ -19,6 +19,7 @@ #include "br/helper.h" #include "bthread/bthread.h" +#include "butil/strings/string_split.h" #include "common/helper.h" #include "fmt/core.h" @@ -48,6 +49,8 @@ bool ServerInteraction::Init(std::vector addrs) { channels_.push_back(std::move(channel)); } + addrs_ = addrs; + return true; } @@ -62,6 +65,7 @@ bool ServerInteraction::AddAddr(const std::string& addr) { channels_.push_back(std::move(channel)); endpoints_.push_back(endpoint); + addrs_.push_back(addr); return true; } @@ -98,4 +102,34 @@ void ServerInteraction::NextLeader(const dingodb::pb::common::Location& location } } +std::vector ServerInteraction::GetAddrs() { return addrs_; } + +std::string ServerInteraction::GetAddrsAsString() { + std::string internal_addrs; + for (size_t i = 0; i < addrs_.size(); i++) { + if (0 != i) { + internal_addrs += ","; + } + internal_addrs += addrs_[i]; + } + return internal_addrs; +} + +butil::Status ServerInteraction::CreateInteraction(const std::vector& addrs, + ServerInteractionPtr& interaction) { + butil::Status status; + interaction = std::make_shared(); + if (!interaction->Init(addrs)) { + std::string s = fmt::format("Fail to init interaction, addrs"); + for (const auto& addr : addrs) { + s += fmt::format(" {}", addr); + } + DINGO_LOG(ERROR) << s; + status = butil::Status(dingodb::pb::error::EINTERNAL, s); + return status; + } + + return butil::Status::OK(); +} + } // namespace br \ No newline at end of file diff --git a/src/br/interation.h b/src/br/interation.h index 30dbb9c3a..8f26ad57e 100644 --- a/src/br/interation.h +++ b/src/br/interation.h @@ -22,6 +22,7 @@ #include #include +#include "br/parameter.h" #include "brpc/channel.h" #include "brpc/controller.h" #include "common/logging.h" @@ -37,10 +38,6 @@ namespace br { -const int kMaxRetry = 5; -const int64_t kTimeoutMs = 60000; -const bool kLogEachRequest = true; - class ServerInteraction { public: ServerInteraction() : leader_index_(0){}; @@ -51,6 +48,9 @@ class ServerInteraction { ServerInteraction(ServerInteraction&&) = delete; ServerInteraction& operator=(ServerInteraction&&) = delete; + static butil::Status CreateInteraction(const std::vector& addrs, + std::shared_ptr& interaction); + bool Init(const std::string& addrs); bool Init(std::vector addrs); @@ -70,11 +70,15 @@ class ServerInteraction { int64_t GetLatency() const { return latency_; } + std::vector GetAddrs(); + std::string GetAddrsAsString(); + private: std::atomic leader_index_; std::vector endpoints_; std::vector > channels_; int64_t latency_; + std::vector addrs_; }; using ServerInteractionPtr = std::shared_ptr; @@ -106,14 +110,15 @@ butil::Status ServerInteraction::SendRequest(const std::string& service_name, co DINGO_LOG(FATAL) << "Unknown api name: " << api_name; } + butil::Status status; int retry_count = 0; do { brpc::Controller cntl; - cntl.set_timeout_ms(kTimeoutMs); + cntl.set_timeout_ms(FLAGS_br_server_interaction_timeout_ms); cntl.set_log_id(butil::fast_rand()); const int leader_index = GetLeader(); channels_[leader_index]->CallMethod(method, &cntl, &request, &response, nullptr); - if (kLogEachRequest) { + if (FLAGS_br_server_interaction_print_each_rpc_request) { DINGO_LOG(INFO) << fmt::format("send request api [{}] {} response: {} request: {}", leader_index, api_name, response.ShortDebugString().substr(0, 256), request.ShortDebugString().substr(0, 256)); @@ -123,7 +128,8 @@ butil::Status ServerInteraction::SendRequest(const std::string& service_name, co cntl.ErrorText()); if (cntl.ErrorCode() == 112) { ++retry_count; - NextLeader(leader_index); + // NextLeader(leader_index); + status.set_error(cntl.ErrorCode(), cntl.ErrorText()); continue; } latency_ = cntl.latency_us(); @@ -149,13 +155,12 @@ butil::Status ServerInteraction::SendRequest(const std::string& service_name, co return butil::Status(); } - } while (retry_count < kMaxRetry); + } while (retry_count < FLAGS_br_server_interaction_max_retry); DINGO_LOG(ERROR) << fmt::format("{} response failed, error: {} {}", api_name, - dingodb::pb::error::Errno_Name(response.error().errcode()), - response.error().errmsg()); + dingodb::pb::error::Errno_Name(status.error_code()), status.error_cstr()); - return butil::Status(response.error().errcode(), response.error().errmsg()); + return status; } template diff --git a/src/br/main.cc b/src/br/main.cc index fb64ee5fd..514e405cb 100644 --- a/src/br/main.cc +++ b/src/br/main.cc @@ -1,4 +1,3 @@ - // Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +12,665 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include #include -int main(int argc, char* argv[]) { return 0; } +#include +#include +#include +#include +#include +#include +#include +#include + +#include "br/backup.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/parameter.h" +#include "br/utils.h" +#include "butil/status.h" +#include "common/helper.h" +#include "common/logging.h" +#include "common/version.h" +#include "fmt/format.h" +#include "gflags/gflags.h" +#include "proto/coordinator.pb.h" +#include "proto/error.pb.h" + +const std::string kProgramName = "dingodb_br"; + +static void InitLog(const std::string& log_dir) { + if (!dingodb::Helper::IsExistPath(log_dir)) { + dingodb::Helper::CreateDirectories(log_dir); + } + + FLAGS_logbufsecs = 0; + FLAGS_stop_logging_if_full_disk = true; + FLAGS_minloglevel = google::GLOG_INFO; + FLAGS_logbuflevel = google::GLOG_INFO; + FLAGS_logtostdout = false; + FLAGS_logtostderr = false; + FLAGS_alsologtostderr = false; + + google::InitGoogleLogging(kProgramName.c_str()); + google::SetLogDestination(google::GLOG_INFO, fmt::format("{}/{}.info.log.", log_dir, kProgramName).c_str()); + google::SetLogDestination(google::GLOG_WARNING, fmt::format("{}/{}.warn.log.", log_dir, kProgramName).c_str()); + google::SetLogDestination(google::GLOG_ERROR, fmt::format("{}/{}.error.log.", log_dir, kProgramName).c_str()); + google::SetLogDestination(google::GLOG_FATAL, fmt::format("{}/{}.fatal.log.", log_dir, kProgramName).c_str()); + google::SetStderrLogging(google::GLOG_FATAL); +} + +static butil::Status SetStoreInteraction() { + dingodb::pb::coordinator::GetStoreMapRequest request; + dingodb::pb::coordinator::GetStoreMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.add_filter_store_types(::dingodb::pb::common::StoreType::NODE_TYPE_STORE); + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "GetStoreMap", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to get store map, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + std::vector addrs; + + for (int i = 0; i < response.storemap().stores_size(); i++) { + const dingodb::pb::common::Store& store = response.storemap().stores(i); + const auto& location = store.server_location(); + DINGO_LOG(INFO) << "store_id=" << store.id() << ", host=" << location.host() << ", " << location.port(); + addrs.push_back(fmt::format("{}:{}", location.host(), location.port())); + } + + std::shared_ptr store_interaction = std::make_shared(); + if (!store_interaction->Init(addrs)) { + std::string s = fmt::format("Fail to init store_interaction, addrs"); + for (const auto& addr : addrs) { + s += fmt::format(" {}", addr); + } + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + return butil::Status(); +} + +static butil::Status SetIndexInteraction() { + dingodb::pb::coordinator::GetStoreMapRequest request; + dingodb::pb::coordinator::GetStoreMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.add_filter_store_types(::dingodb::pb::common::StoreType::NODE_TYPE_INDEX); + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "GetStoreMap", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to get index map, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + std::vector addrs; + + for (int i = 0; i < response.storemap().stores_size(); i++) { + const dingodb::pb::common::Store& store = response.storemap().stores(i); + const auto& location = store.server_location(); + DINGO_LOG(INFO) << "index_id=" << store.id() << ", host=" << location.host() << ", " << location.port(); + addrs.push_back(fmt::format("{}:{}", location.host(), location.port())); + } + + std::shared_ptr index_interaction = std::make_shared(); + if (!index_interaction->Init(addrs)) { + std::string s = fmt::format("Fail to init index_interaction, addrs"); + for (const auto& addr : addrs) { + s += fmt::format(" {}", addr); + } + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + return butil::Status(); +} + +static butil::Status SetDocumentInteraction() { + dingodb::pb::coordinator::GetStoreMapRequest request; + dingodb::pb::coordinator::GetStoreMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.add_filter_store_types(::dingodb::pb::common::StoreType::NODE_TYPE_DOCUMENT); + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "GetStoreMap", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to get document map, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + std::vector addrs; + + for (int i = 0; i < response.storemap().stores_size(); i++) { + const dingodb::pb::common::Store& store = response.storemap().stores(i); + const auto& location = store.server_location(); + DINGO_LOG(INFO) << "document_id=" << store.id() << ", host=" << location.host() << ", " << location.port(); + addrs.push_back(fmt::format("{}:{}", location.host(), location.port())); + } + + std::shared_ptr document_interaction = std::make_shared(); + if (!document_interaction->Init(addrs)) { + std::string s = fmt::format("Fail to init document_interaction, addrs"); + for (const auto& addr : addrs) { + s += fmt::format(" {}", addr); + } + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + return butil::Status(); +} + +struct DingoStackTraceInfo { + char* filename; + int lineno; + char* function; + uintptr_t pc; +}; + +/* Passed to backtrace callback function. */ +struct DingoBacktraceData { + struct DingoStackTraceInfo* all; + size_t index; + size_t max; + int failed; +}; + +static int BacktraceCallback(void* vdata, uintptr_t pc, const char* filename, int lineno, const char* function) { + struct DingoBacktraceData* data = (struct DingoBacktraceData*)vdata; + struct DingoStackTraceInfo* p; + + if (data->index >= data->max) { + fprintf(stderr, "callback_one: callback called too many times\n"); // NOLINT + data->failed = 1; + return 1; + } + + p = &data->all[data->index]; + + // filename + if (filename == nullptr) + p->filename = nullptr; + else { + p->filename = strdup(filename); + assert(p->filename != nullptr); + } + + // lineno + p->lineno = lineno; + + // function + if (function == nullptr) + p->function = nullptr; + else { + p->function = strdup(function); + assert(p->function != nullptr); + } + + // pc + if (pc != 0) { + p->pc = pc; + } + + ++data->index; + + return 0; +} + +/* An error callback passed to backtrace. */ + +static void ErrorCallback(void* vdata, const char* msg, int errnum) { + struct DingoBacktraceData* data = (struct DingoBacktraceData*)vdata; + + fprintf(stderr, "%s", msg); // NOLINT + if (errnum > 0) fprintf(stderr, ": %s", strerror(errnum)); // NOLINT + fprintf(stderr, "\n"); // NOLINT + data->failed = 1; +} + +// The signal handler +#ifndef MAX_STACKTRACE_SIZE +#define MAX_STACKTRACE_SIZE 128 +#endif +static void SignalHandler(int signo) { + printf("========== handle signal '%d' ==========\n", signo); + + if (signo == SIGTERM) { + // TODO: graceful shutdown + // clean temp directory + // dingodb::Helper::RemoveAllFileOrDirectory(dingodb::Server::GetInstance().GetCheckpointPath()); + // dingodb::Helper::RemoveFileOrDirectory(dingodb::Server::GetInstance().PidFilePath()); + // DINGO_LOG(WARNING) << "GRACEFUL SHUTDOWN, clean up checkpoint dir: " + // << dingodb::Server::GetInstance().GetCheckpointPath() + // << ", clean up pid_file: " << dingodb::Server::GetInstance().PidFilePath(); + _exit(0); + } + + std::cerr << "Received signal " << signo << '\n'; + std::cerr << "Stack trace:" << '\n'; + DINGO_LOG(ERROR) << "Received signal " << signo; + DINGO_LOG(ERROR) << "Stack trace:"; + + struct backtrace_state* state = backtrace_create_state(nullptr, 0, ErrorCallback, nullptr); + if (state == nullptr) { + std::cerr << "state is null" << '\n'; + } + + struct DingoStackTraceInfo all[MAX_STACKTRACE_SIZE]; + struct DingoBacktraceData data; + + data.all = &all[0]; + data.index = 0; + data.max = MAX_STACKTRACE_SIZE; + data.failed = 0; + + int i = backtrace_full(state, 0, BacktraceCallback, ErrorCallback, &data); + if (i != 0) { + std::cerr << "backtrace_full failed" << '\n'; + DINGO_LOG(ERROR) << "backtrace_full failed"; + } + + for (size_t x = 0; x < data.index; x++) { + int status; + char* nameptr = all[x].function; + char* demangled = abi::__cxa_demangle(all[x].function, nullptr, nullptr, &status); + if (status == 0 && demangled) { + nameptr = demangled; + } + + Dl_info info = {}; + + if (!dladdr((void*)all[x].pc, &info)) { + auto error_msg = butil::string_printf("#%zu source[%s:%d] symbol[%s] pc[0x%0lx]", x, all[x].filename, + all[x].lineno, nameptr, static_cast(all[x].pc)); + DINGO_LOG(ERROR) << error_msg; + std::cout << error_msg << '\n'; + } else { + auto error_msg = butil::string_printf( + "#%zu source[%s:%d] symbol[%s] pc[0x%0lx] fname[%s] fbase[0x%lx] sname[%s] saddr[0x%lx] ", x, all[x].filename, + all[x].lineno, nameptr, static_cast(all[x].pc), info.dli_fname, (uint64_t)info.dli_fbase, + info.dli_sname, (uint64_t)info.dli_saddr); + DINGO_LOG(ERROR) << error_msg; + std::cout << error_msg << '\n'; + } + if (demangled) { + free(demangled); + } + } + + // call abort() to generate core dump + DINGO_LOG(ERROR) << "call abort() to generate core dump for signo=" << signo << " " << strsignal(signo); + auto s = signal(SIGABRT, SIG_DFL); + if (s == SIG_ERR) { + std::cerr << "Failed to set signal handler to SIG_DFL for SIGABRT" << '\n'; + } + abort(); +} + +static void SignalHandlerWithoutLineno(int signo) { + printf("========== handle signal '%d' ==========\n", signo); + + if (signo == SIGTERM) { + // TODO: graceful shutdown + // // clean temp directory + // dingodb::Helper::RemoveAllFileOrDirectory(dingodb::Server::GetInstance().GetCheckpointPath()); + // dingodb::Helper::RemoveFileOrDirectory(dingodb::Server::GetInstance().PidFilePath()); + // DINGO_LOG(ERROR) << "GRACEFUL SHUTDOWN, clean up checkpoint dir: " + // << dingodb::Server::GetInstance().GetCheckpointPath() + // << ", clean up pid_file: " << dingodb::Server::GetInstance().PidFilePath(); + _exit(0); + } + + unw_context_t context; + unw_cursor_t cursor; + unw_getcontext(&context); + unw_init_local(&cursor, &context); + int i = 0; + char buffer[2048]; + + do { + unw_word_t ip, offset; + char symbol[256]; + + unw_word_t pc, sp; + unw_get_reg(&cursor, UNW_REG_IP, &pc); + unw_get_reg(&cursor, UNW_REG_SP, &sp); + Dl_info info = {}; + + // Get the instruction pointer and symbol name for this frame + unw_get_reg(&cursor, UNW_REG_IP, &ip); + + if (unw_get_proc_name(&cursor, symbol, sizeof(symbol), &offset) == 0) { + char* nameptr = symbol; + // Demangle the symbol name + int demangle_status; + char* demangled = abi::__cxa_demangle(symbol, nullptr, nullptr, &demangle_status); + if (demangled) { + nameptr = demangled; + } + // std::cout << " " << nameptr << " + " << offset << " (0x" << std::hex << pc << ")" << '\n'; + + if (!dladdr((void*)pc, &info)) { + std::stringstream string_stream; + string_stream << "Frame [" << i++ << "] symbol=[" << nameptr << " + " << offset << "] (0x" << std::hex << pc + << ") "; + std::string const error_msg = string_stream.str(); + DINGO_LOG(ERROR) << error_msg; + std::cout << error_msg << '\n'; + } else { + std::stringstream string_stream; + string_stream << "Frame [" << i++ << "] symbol=[" << nameptr << " + " << offset << "] (0x" << std::hex << pc + << ") " << " fname=[" << info.dli_fname << "] saddr=[" << info.dli_saddr << "] fbase=[" + << info.dli_fbase << "]"; + std::string const error_msg = string_stream.str(); + DINGO_LOG(ERROR) << error_msg; + std::cout << error_msg << '\n'; + } + if (demangled) { + free(demangled); + } + } + + } while (unw_step(&cursor) > 0); + + // call abort() to generate core dump + DINGO_LOG(ERROR) << "call abort() to generate core dump for signo=" << signo << " " << strsignal(signo); + auto s = signal(SIGABRT, SIG_DFL); + if (s == SIG_ERR) { + std::cerr << "Failed to set signal handler to SIG_DFL for SIGABRT" << '\n'; + } + abort(); +} + +void SetupSignalHandler() { + sighandler_t s; + s = signal(SIGTERM, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGTERM\n"); + exit(-1); + } + s = signal(SIGSEGV, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGSEGV\n"); + exit(-1); + } + s = signal(SIGFPE, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGFPE\n"); + exit(-1); + } + s = signal(SIGBUS, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGBUS\n"); + exit(-1); + } + s = signal(SIGILL, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGILL\n"); + exit(-1); + } + s = signal(SIGABRT, SignalHandler); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGABRT\n"); + exit(-1); + } + // ignore SIGPIPE + s = signal(SIGPIPE, SIG_IGN); + if (s == SIG_ERR) { + printf("Failed to setup signal handler for SIGPIPE\n"); + exit(-1); + } +} + +int main(int argc, char* argv[]) { + if (dingodb::Helper::IsExistPath("conf/gflags.conf")) { + google::SetCommandLineOption("flagfile", "conf/gflags.conf"); + } + + google::ParseCommandLineFlags(&argc, &argv, false); + + SetupSignalHandler(); + + if (dingodb::FLAGS_show_version || argc == 1) { + dingodb::DingoShowVerion(); + printf( + "Usage: --br_coor_url=[ip:port] --br_type=[backup|restore] --br_backup_type=[full] --backupts='[YYYY-MM-DD " + "HH:MM:SS ]' --storage=local://[path_dir]\n"); + printf( + "Usage: --br_coor_url=[file://./conf/coor_list] --br_type=[backup|restore] --br_backup_type=[full] " + "--backupts='[YYYY-MM-DD " + "HH:MM:SS ]' --storage=local://[path_dir]\n"); + printf("Example: \n"); + printf( + "/dingodb_br --br_coor_url=127.0.0.1:22001 --br_type=backup --br_backup_type=full --backupts='2020-01-01 " + "00:00:00 +08:00' " + "--storage=local:///opt/backup-2020-01-01\n"); + + printf( + "/dingodb_br --br_coor_url=[file://./conf/coor_list] --br_type=backup --br_backup_type=full " + "--backupts='2020-01-01 " + "00:00:00 +08:00' " + "--storage=local:///opt/backup-2020-01-01\n"); + exit(-1); + } + + InitLog(br::FLAGS_br_log_dir); + + std::cout << "Number of command line arguments : " << argc << std::endl; + DINGO_LOG(INFO) << "Number of command line arguments : " << argc; + + for (int i = 0; i < argc; i++) { + std::cout << fmt::format("args[{}]=[{}]", i, argv[i]) << std::endl; + DINGO_LOG(INFO) << fmt::format("args[{}]=[{}]", i, argv[i]); + } + + std::cout << "Detail BR log in ./log" << std::endl; + + butil::Status status; + std::shared_ptr coordinator_interaction = std::make_shared(); + if (br::FLAGS_br_coor_url.empty()) { + DINGO_LOG(WARNING) << "coordinator url is empty, try to use file://./coor_list"; + br::FLAGS_br_coor_url = "file://./coor_list"; + + std::string path = br::FLAGS_br_coor_url; + path = path.replace(path.find("file://"), 7, ""); + auto addrs = br::Helper::GetAddrsFromFile(path); + if (addrs.empty()) { + DINGO_LOG(ERROR) << "coor_url not find addr, path=" << path; + return -1; + } + if (!coordinator_interaction->Init(addrs)) { + DINGO_LOG(ERROR) << "Fail to init coordinator_interaction, please check parameter --br_coor_url=" + << br::FLAGS_br_coor_url; + return -1; + } + } else { + auto addrs = br::FLAGS_br_coor_url; + if (!coordinator_interaction->Init(addrs)) { + DINGO_LOG(ERROR) << "Fail to init coordinator_interaction, please check parameter --br_coor_url=" + << br::FLAGS_br_coor_url; + return -1; + } + } + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coordinator_interaction); + + status = SetStoreInteraction(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return -1; + } + + status = SetIndexInteraction(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return -1; + } + + status = SetDocumentInteraction(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return -1; + } + + // command parse + if (br::FLAGS_br_type == "backup") { + if (br::FLAGS_br_backup_type == "full") { + } else { + DINGO_LOG(ERROR) << "backup type not support, please check parameter --br_backup_type=" + << br::FLAGS_br_backup_type; + return -1; + } + } else if (br::FLAGS_br_type == "restore") { + // TODO : restore + DINGO_LOG(ERROR) << "br type not support, please check parameter --br_type=" << br::FLAGS_br_type; + return -1; + } else { + DINGO_LOG(ERROR) << "br type not support, please check parameter --br_type=" << br::FLAGS_br_type; + return -1; + } + + status = br::Utils::ConvertBackupTsToTso(br::FLAGS_backupts, br::FLAGS_backuptso_internal); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return -1; + } + + if (br::FLAGS_storage.empty()) { + DINGO_LOG(ERROR) << "storage is empty, please check parameter --storage=" << br::FLAGS_storage; + return -1; + } + + if (std::string(br::FLAGS_storage).find("local://") == 0) { + std::string path = br::FLAGS_storage; + path = path.replace(path.find("local://"), 8, ""); + if (!path.empty()) { + if (path.back() == '/') { + path.pop_back(); + } + } + + if (path.empty()) { + DINGO_LOG(ERROR) << "path is empty, please check parameter --storage=" << br::FLAGS_storage; + return -1; + } + + std::filesystem::path temp_path = path; + if (temp_path.is_relative()) { + DINGO_LOG(ERROR) << "storage not support relative path. use absolute path. " << br::FLAGS_storage; + return -1; + } + + br::FLAGS_storage_internal = path; + } else { + DINGO_LOG(ERROR) << "storage not support, please check parameter --storage=" << br::FLAGS_storage; + return -1; + } + + // backup + if (br::FLAGS_br_type == "backup") { + br::BackupParams params; + params.coor_url = br::FLAGS_br_coor_url; + params.br_type = br::FLAGS_br_type; + params.br_backup_type = br::FLAGS_br_backup_type; + params.backupts = br::FLAGS_backupts; + params.backuptso_internal = br::FLAGS_backuptso_internal; + params.storage = br::FLAGS_storage; + params.storage_internal = br::FLAGS_storage_internal; + + std::cout << "Full Backup Parameter :" << std::endl; + DINGO_LOG(INFO) << "Full Backup Parameter :"; + + std::cout << "coordinator url : " + << br::InteractionManager::GetInstance().GetCoordinatorInteraction()->GetAddrsAsString() << std::endl; + DINGO_LOG(INFO) << "coordinator url : " + << br::InteractionManager::GetInstance().GetCoordinatorInteraction()->GetAddrsAsString(); + + std::cout << "store url : " + << br::InteractionManager::GetInstance().GetStoreInteraction()->GetAddrsAsString() << std::endl; + DINGO_LOG(INFO) << "store url : " + << br::InteractionManager::GetInstance().GetStoreInteraction()->GetAddrsAsString(); + + std::cout << "index url : " + << br::InteractionManager::GetInstance().GetIndexInteraction()->GetAddrsAsString() << std::endl; + DINGO_LOG(INFO) << "index url : " + << br::InteractionManager::GetInstance().GetIndexInteraction()->GetAddrsAsString(); + + std::cout << "document url : " + << br::InteractionManager::GetInstance().GetDocumentInteraction()->GetAddrsAsString() << std::endl; + DINGO_LOG(INFO) << "document url : " + << br::InteractionManager::GetInstance().GetDocumentInteraction()->GetAddrsAsString(); + + std::cout << "br type : " << params.br_type << std::endl; + DINGO_LOG(INFO) << "br type : " << params.br_type; + + std::cout << "br backup type : " << params.br_backup_type << std::endl; + DINGO_LOG(INFO) << "br backup type : " << params.br_backup_type; + + std::cout << "backupts : " << params.backupts << std::endl; + DINGO_LOG(INFO) << "backupts : " << params.backupts; + + std::cout << "backuptso_internal : " << params.backuptso_internal << std::endl; + DINGO_LOG(INFO) << "backuptso_internal : " << params.backuptso_internal; + + std::cout << "storage : " << params.storage << std::endl; + DINGO_LOG(INFO) << "storage : " << params.storage; + + std::cout << "storage_internal : " << params.storage_internal << std::endl; + DINGO_LOG(INFO) << "storage_internal : " << params.storage_internal; + + std::shared_ptr backup = std::make_shared(params); + + std::cout << std::endl; + DINGO_LOG(INFO) << ""; + + std::cout << "Full Backup" << std::endl; + DINGO_LOG(INFO) << "Full Backup"; + + status = backup->Init(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + std::cout << "Backup failed" << std::endl; + DINGO_LOG(INFO) << "Backup failed"; + return -1; + } + status = backup->Run(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + std::cout << "Backup failed" << std::endl; + DINGO_LOG(INFO) << "Backup failed"; + return -1; + } + + status = backup->Finish(); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + std::cout << "Backup failed" << std::endl; + DINGO_LOG(INFO) << "Backup failed"; + return -1; + } + + DINGO_LOG(INFO) << "Backup finish"; + + } else { + DINGO_LOG(ERROR) << "br type not support, please check parameter --br_type=" << br::FLAGS_br_type; + return -1; + } + + return 0; +} diff --git a/src/br/parameter.cc b/src/br/parameter.cc new file mode 100644 index 000000000..41b0cb0bf --- /dev/null +++ b/src/br/parameter.cc @@ -0,0 +1,55 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/parameter.h" + +namespace br { + +DEFINE_string(br_coor_url, "", "coordinator url"); + +DEFINE_string(br_type, "backup", "backup restore type. default: backup"); + +DEFINE_string(br_backup_type, "full", "backup type. default: full."); + +DEFINE_string(backupts, "", "backup ts. like: 2022-09-08 13:30:00 +08:00"); +DEFINE_int64(backuptso_internal, 0, "backup tso. like: convert 2022-09-08 13:30:00 +08:00 to tso"); + +DEFINE_string(storage, "", "storage. like: local:///br_data"); +DEFINE_string(storage_internal, "", "storage. like: /br_data. remove local://"); + +// backup watch interval in seconds. default 5s +DEFINE_uint32(backup_watch_interval_s, 5, "backup watch interval in seconds. default 5s"); + +// backup task timeout in seconds. default 100s +DEFINE_uint32(backup_task_timeout_s, 100, "backup task timeout in seconds. default 100s"); + +// backup task max retry times. default 5 +DEFINE_uint32(backup_task_max_retry, 5, "backup task max retry times. default 5"); + +DEFINE_bool(br_server_interaction_print_each_rpc_request, false, + "br server interaction log switch rpc request. default is false"); + +DEFINE_int32(br_server_interaction_max_retry, 5, "br server interaction max retry. default 5"); + +// DEFINE_int64(br_server_interaction_timeout_ms, 60000, "br server interaction connect timeout . default 60000 ms"); +DEFINE_int64(br_server_interaction_timeout_ms, 0x7fffffff, + "br server interaction connect timeout . default 0x7fffffff ms"); + +DEFINE_bool(br_log_switch_backup_detail, true, "backup detail log"); + +DEFINE_bool(br_log_switch_backup_detail_detail, false, "backup detail detail log"); + +DEFINE_string(br_log_dir, "./log", "backup log dir. default ./log"); + +} // namespace br \ No newline at end of file diff --git a/src/br/parameter.h b/src/br/parameter.h new file mode 100644 index 000000000..18981290b --- /dev/null +++ b/src/br/parameter.h @@ -0,0 +1,70 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_PARAMETER_H_ +#define DINGODB_BR_PARAMETER_H_ + +#include "gflags/gflags.h" +#include "gflags/gflags_declare.h" + +namespace br { + +DECLARE_string(br_coor_url); + +DECLARE_string(br_type); + +DECLARE_string(br_backup_type); + +DECLARE_string(backupts); +DECLARE_int64(backuptso_internal); + +DECLARE_string(storage); +DECLARE_string(storage_internal); + +// backup watch interval in seconds. default 10s +DECLARE_uint32(backup_watch_interval_s); + +// backup task timeout in seconds. default 100s +DECLARE_uint32(backup_task_timeout_s); + +// backup task max retry times. default 5 +DECLARE_uint32(backup_task_max_retry); + +struct BackupParams { + std::string coor_url; + std::string br_type; + std::string br_backup_type; + std::string backupts; + int64_t backuptso_internal; + std::string storage; + std::string storage_internal; +}; + +inline const std::string kBackupFileLock = "backup.lock"; + +DECLARE_bool(br_server_interaction_print_each_rpc_request); + +DECLARE_int32(br_server_interaction_max_retry); + +DECLARE_int64(br_server_interaction_timeout_ms); + +DECLARE_bool(br_log_switch_backup_detail); + +DECLARE_bool(br_log_switch_backup_detail_detail); + +DECLARE_string(br_log_dir); + +} // namespace br + +#endif // DINGODB_BR_PARAMETER_H_ \ No newline at end of file diff --git a/src/br/router.cc b/src/br/router.cc index 6cebe412c..712899d1c 100644 --- a/src/br/router.cc +++ b/src/br/router.cc @@ -28,7 +28,7 @@ static dingodb::pb::common::Region SendQueryRegion(int64_t region_id) { request.set_region_id(region_id); - InteractionManager::GetInstance().SendRequestWithoutContext("CoordinatorService", "QueryRegion", request, response); + //InteractionManager::GetInstance().SendRequestWithoutContext("CoordinatorService", "QueryRegion", request, response); return response.region(); } @@ -83,8 +83,8 @@ dingodb::pb::store::Context RegionEntry::GenConext() { return ctx; } -RegionRouter::RegionRouter() { bthread_mutex_init(&mutex_, nullptr); } -RegionRouter::~RegionRouter() { bthread_mutex_destroy(&mutex_); } +RegionRouter::RegionRouter() = default; +RegionRouter::~RegionRouter() = default; RegionRouter& RegionRouter::GetInstance() { static RegionRouter instance; @@ -92,13 +92,13 @@ RegionRouter& RegionRouter::GetInstance() { } void RegionRouter::AddRegionEntry(const dingodb::pb::common::Region& region) { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockWriteGuard guard(&rw_lock_); route_map_.insert_or_assign(region.definition().range().start_key(), RegionEntry::New(region)); } void RegionRouter::AddRegionEntry(RegionEntryPtr region) { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockWriteGuard guard(&rw_lock_); route_map_.insert_or_assign(region->Range().start_key(), region); } @@ -124,7 +124,7 @@ void RegionRouter::UpdateRegionEntry(const dingodb::pb::error::StoreRegionInfo& } RegionEntryPtr RegionRouter::QueryRegionEntry(const std::string& key) { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockReadGuard guard(&rw_lock_); auto it = route_map_.lower_bound(key); if (it == route_map_.end()) { @@ -146,7 +146,7 @@ RegionEntryPtr RegionRouter::QueryRegionEntry(int64_t region_id) { RegionEntryPtr region_entry; { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockReadGuard guard(&rw_lock_); for (auto& [_, temp_region_entry] : route_map_) { if (temp_region_entry->RegionId() == region_id) { @@ -168,7 +168,7 @@ RegionEntryPtr RegionRouter::QueryRegionEntry(int64_t region_id) { } std::vector RegionRouter::QueryRegionEntryByTable(int64_t table_id) { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockReadGuard guard(&rw_lock_); std::vector region_entries; for (auto& [_, region_entry] : route_map_) { @@ -184,7 +184,7 @@ std::vector RegionRouter::QueryRegionEntryByTable(int64_t table_ } std::vector RegionRouter::QueryRegionEntryByPartition(int64_t partition_id) { - BAIDU_SCOPED_LOCK(mutex_); + dingodb::RWLockReadGuard guard(&rw_lock_); std::vector region_entries; for (auto& [_, region_entry] : route_map_) { diff --git a/src/br/router.h b/src/br/router.h index 3cbfd0b74..7af32886b 100644 --- a/src/br/router.h +++ b/src/br/router.h @@ -21,7 +21,7 @@ #include #include -#include "bthread/mutex.h" +#include "common/synchronization.h" #include "proto/common.pb.h" #include "proto/error.pb.h" #include "proto/store.pb.h" @@ -89,7 +89,7 @@ class RegionRouter { // key: the start_key of region range // value: RegionEntry std::map route_map_; - bthread_mutex_t mutex_; + dingodb::RWLock rw_lock_; }; } // namespace br diff --git a/src/br/sst_file_writer.cc b/src/br/sst_file_writer.cc new file mode 100644 index 000000000..0ea801f15 --- /dev/null +++ b/src/br/sst_file_writer.cc @@ -0,0 +1,48 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/sst_file_writer.h" + +#include +#include + +#include "fmt/core.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/write_batch.h" + +namespace br { + +butil::Status SstFileWriter::SaveFile(const std::map& kvs, const std::string& filename) { + auto status = sst_writer_->Open(filename); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + + for (const auto& [key, value] : kvs) { + status = sst_writer_->Put(key, value); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + } + + status = sst_writer_->Finish(); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + + return butil::Status(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/sst_file_writer.h b/src/br/sst_file_writer.h new file mode 100644 index 000000000..ef33ca405 --- /dev/null +++ b/src/br/sst_file_writer.h @@ -0,0 +1,49 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_SST_FILE_WRITER_H_ +#define DINGODB_BR_SST_FILE_WRITER_H_ + +#include +#include + +#include "butil/status.h" +#include "proto/common.pb.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" + +namespace br { +class SstFileWriter { + public: + SstFileWriter(const rocksdb::Options& options) + : options_(options), + sst_writer_(std::make_unique(rocksdb::EnvOptions(), options_, nullptr, true)) {} + ~SstFileWriter() = default; + + SstFileWriter(SstFileWriter&& rhs) = delete; + SstFileWriter& operator=(SstFileWriter&& rhs) = delete; + + butil::Status SaveFile(const std::map& kvs, const std::string& filename); + + int64_t GetSize() { return sst_writer_->FileSize(); } + + private: + rocksdb::Options options_; + std::unique_ptr sst_writer_; +}; +using SstFileWriterPtr = std::shared_ptr; +} // namespace br + +#endif // DINGODB_BR_SST_FILE_WRITER_H_ \ No newline at end of file diff --git a/src/br/utils.cc b/src/br/utils.cc new file mode 100644 index 000000000..f1a962cd1 --- /dev/null +++ b/src/br/utils.cc @@ -0,0 +1,264 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "br/utils.h" + +#include +#include +#include +#include + +#include "common/logging.h" +#include "coordinator/tso_control.h" +#include "fmt/core.h" +#include "proto/error.pb.h" + +namespace br { + +butil::Status Utils::FileExistsAndRegular(const std::string& file_path) { + std::filesystem::path file_path_check(file_path); + if (std::filesystem::exists(file_path_check)) { + std::error_code ec; + if (std::filesystem::is_symlink(file_path)) { + file_path_check = std::filesystem::read_symlink(file_path); + } + if (!std::filesystem::is_regular_file(file_path_check, ec)) { + std::string s = fmt::format("file_path : {} is not regular file : {} {}", file_path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_REGULAR, s); + } + + auto perms = std::filesystem::status(file_path_check, ec).permissions(); + if (ec) { + std::string s = fmt::format("file_path : {} does not have owner read permission", file_path); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_OWNER_READABLE, s); + } + // Check if the file has owner read permission + bool has_owner_read = (perms & std::filesystem::perms::owner_read) != std::filesystem::perms::none; + if (!has_owner_read) { + std::string s = fmt::format("file_path : {} does not have owner read permission", file_path); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_OWNER_READABLE, s); + } + // TODO : Check if the file has owner write permission + } else { // data_path not exist + std::string s = fmt::format("file_path : {} not exist", file_path); + // DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_EXIST, s); + } + return butil::Status::OK(); +} + +butil::Status Utils::DirExists(const std::string& dir_path) { + std::filesystem::path dir_path_check(dir_path); + if (std::filesystem::exists(dir_path_check)) { + std::error_code ec; + if (std::filesystem::is_symlink(dir_path)) { + dir_path_check = std::filesystem::read_symlink(dir_path); + } + + if (!std::filesystem::is_directory(dir_path_check, ec)) { + std::string s = fmt::format("dir_path : {} is not directory, {}, {}", dir_path, ec.value(), ec.message()); + // DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_DIRECTORY, s); + } + auto perms = std::filesystem::status(dir_path_check, ec).permissions(); + if (ec) { + std::string s = fmt::format("dir_path : {} does not have owner read permission", dir_path); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_OWNER_READABLE, s); + } + // Check if the file has owner read permission + bool has_owner_read = (perms & std::filesystem::perms::owner_read) != std::filesystem::perms::none; + if (!has_owner_read) { + std::string s = fmt::format("dir_path : {} does not have owner read permission", dir_path); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_OWNER_READABLE, s); + } + // TODO : Check if the file has owner write permission + } else { // index_path_prefix not exist + std::string s = fmt::format("dir_path : {} not exist", dir_path); + // DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EFILE_NOT_EXIST, s); + } + return butil::Status::OK(); +} + +butil::Status Utils::ClearDir(const std::string& dir_path) { + butil::Status status = DirExists(dir_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + return RemoveAllDir(dir_path, false); +} + +butil::Status Utils::CreateDir(const std::string& dir_path) { + butil::Status status = DirExists(dir_path); + if (status.ok()) { + return butil::Status::OK(); + } else { + std::error_code ec; + std::filesystem::create_directory(dir_path, ec); + if (0 != ec.value()) { + std::string s = fmt::format("create_directory dir_path : {} failed, {} {}", dir_path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + if (ec.value() == EACCES) { + return butil::Status(dingodb::pb::error::Errno::EFILE_PERMISSION_DENIED, s); + } else { + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + } + } + + return butil::Status::OK(); +} + +butil::Status Utils::CreateDirRecursion(const std::string& dir_path) { + butil::Status status = DirExists(dir_path); + if (status.ok()) { + return butil::Status::OK(); + } else { + std::error_code ec; + std::filesystem::create_directories(dir_path, ec); + if (0 != ec.value()) { + std::string s = fmt::format("create_directories dir_path : {} failed, {} {}", dir_path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + if (ec.value() == EACCES) { + return butil::Status(dingodb::pb::error::Errno::EFILE_PERMISSION_DENIED, s); + } else { + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + } + } + + return butil::Status::OK(); +} + +butil::Status Utils::CreateFile(std::ofstream& writer, const std::string& filename) { + writer.exceptions(std::ofstream::failbit | std::ofstream::badbit); + writer.open(filename, std::ios::binary | std::ios::out); + + if (writer.fail()) { + char buff[1024]; + auto ret = std::string(strerror_r(errno, buff, 1024)); + std::string s = std::string("Failed to open file") + filename + " for write because " + buff + ", ret=" + ret; + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + + return butil::Status::OK(); +} + +butil::Status Utils::RemoveAllDir(const std::string& dir_path, bool remove_self) { + std::error_code ec; + + try { + std::filesystem::path dir_path_check(dir_path); + std::filesystem::path dir_path_link_check(dir_path); + + if (std::filesystem::is_symlink(dir_path)) { + dir_path_check = std::filesystem::read_symlink(dir_path); + } + + if (!remove_self) { + for (const auto& entry : std::filesystem::directory_iterator(dir_path_check)) { + std::filesystem::remove_all(entry, ec); + if (0 != ec.value()) { + std::string s = + fmt::format("remove_all dir_path : {} failed, {} {}", entry.path().string(), ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + if (ec.value() == EACCES) { + return butil::Status(dingodb::pb::error::Errno::EFILE_PERMISSION_DENIED, s); + } else { + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + } + } + } else { + std::filesystem::remove_all(dir_path_check, ec); + if (0 != ec.value()) { + std::string s = + fmt::format("remove_all dir_path : {} failed, {} {}", dir_path_check.string(), ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + if (ec.value() == EACCES) { + return butil::Status(dingodb::pb::error::Errno::EFILE_PERMISSION_DENIED, s); + } else { + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + } + } + + if (remove_self) { + if (std::filesystem::is_symlink(dir_path)) { + std::filesystem::remove(dir_path_link_check, ec); + if (0 != ec.value()) { + std::string s = fmt::format("remove_all dir_path : {} failed, {} {}", dir_path_link_check.string(), + ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + if (ec.value() == EACCES) { + return butil::Status(dingodb::pb::error::Errno::EFILE_PERMISSION_DENIED, s); + } else { + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + } + } + } + } catch (const std::filesystem::filesystem_error& e) { + std::string s = fmt::format("clear index_path_prefix : {} failed, {}", dir_path, e.what()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::Errno::EINTERNAL, s); + } + return butil::Status::OK(); +} + +int64_t Utils::ConvertToMilliseconds(const std::string& datetime) { + std::istringstream ss(datetime); + + std::tm tm = {}; + ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); + + if (ss.fail()) { + throw std::runtime_error("Failed to parse date-time"); + } + + std::chrono::system_clock::time_point tp = std::chrono::system_clock::from_time_t(std::mktime(&tm)); + + auto milliseconds = std::chrono::duration_cast(tp.time_since_epoch()).count(); + + return milliseconds; +} + +// std::string backup_ts = "2022-09-08 13:30:00 +08:00"; +// TODO : Ignore the time zone for now and use Boost.DateTime or date.h provided by Howard Hinnant later. +butil::Status Utils::ConvertBackupTsToTso(const std::string& backup_ts, int64_t& tso) { + int64_t milliseconds = 0; + try { + milliseconds = ConvertToMilliseconds(backup_ts); + + } catch (const std::exception& e) { + std::string s = fmt::format("Failed to parse backup_ts, {} backup_ts : {}", e.what(), backup_ts); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + int64_t internal_milliseconds = milliseconds - dingodb::kBaseTimestampMs; + tso = 0; + tso = internal_milliseconds << dingodb::kLogicalBits; + + return butil::Status(); +} + +} // namespace br \ No newline at end of file diff --git a/src/br/utils.h b/src/br/utils.h new file mode 100644 index 000000000..f416f5362 --- /dev/null +++ b/src/br/utils.h @@ -0,0 +1,44 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGODB_BR_UTILS_H_ +#define DINGODB_BR_UTILS_H_ + +#include +#include + +#include "butil/status.h" + +namespace br { + +class Utils { + public: + static butil::Status FileExistsAndRegular(const std::string& file_path); + static butil::Status DirExists(const std::string& dir_path); + static butil::Status ClearDir(const std::string& dir_path); + static butil::Status CreateDir(const std::string& dir_path); + static butil::Status CreateDirRecursion(const std::string& dir_path); + static butil::Status CreateFile(std::ofstream& writer, const std::string& filename); + static butil::Status RemoveAllDir(const std::string& dir_path, bool remove_self); + + static int64_t ConvertToMilliseconds(const std::string& datetime); + static butil::Status ConvertBackupTsToTso(const std::string& backup_ts, int64_t& tso); + + private: + Utils() = default; + ~Utils() = default; +}; +} // namespace br + +#endif // DINGODB_BR_UTILS_H_ \ No newline at end of file diff --git a/src/common/constant.h b/src/common/constant.h index 3866da37a..c81f62876 100644 --- a/src/common/constant.h +++ b/src/common/constant.h @@ -274,6 +274,50 @@ class Constant { // tenant inline static const int64_t kDefaultTenantId = 0; + + // backup & restore + inline static const std::string kStoreRegionName = "store"; + inline static const std::string kIndexRegionName = "index"; + inline static const std::string kDocumentRegionName = "document"; + inline static const std::string kCoordinatorRegionName = "coordinator"; + inline static const std::string kBackupRegionName = "backup"; + + inline static const std::string kSdkData = "sdk"; + inline static const std::string kSqlData = "sql"; + + inline static const std::string kCoordinatorSdkMetaKeyName = "SDK_META"; + inline static const std::string kCoordinatorSdkMetaSstName = "coordinator_sdk_meta.sst"; + + inline static const std::string kStoreRegionSqlMetaSstName = "store_region_sql_meta.sst"; + inline static const std::string kStoreCfSstMetaSqlMetaSstName = "store_cf_sst_meta_sql_meta.sst"; + + // sdk data + inline static const std::string kStoreRegionSdkDataSstName = "store_region_sdk_data.sst"; + inline static const std::string kStoreCfSstMetaSdkDataSstName = "store_cf_sst_meta_sdk_data.sst"; + inline static const std::string kIndexRegionSdkDataSstName = "index_region_sdk_data.sst"; + inline static const std::string kIndexCfSstMetaSdkDataSstName = "index_cf_sst_meta_sdk_data.sst"; + inline static const std::string kDocumentRegionSdkDataSstName = "document_region_sdk_data.sst"; + inline static const std::string kDocumentCfSstMetaSdkDataSstName = "document_cf_sst_meta_sdk_data.sst"; + + // sql data + inline static const std::string kStoreRegionSqlDataSstName = "store_region_sql_data.sst"; + inline static const std::string kStoreCfSstMetaSqlDataSstName = "store_cf_sst_meta_sql_data.sst"; + inline static const std::string kIndexRegionSqlDataSstName = "index_region_sql_data.sst"; + inline static const std::string kIndexCfSstMetaSqlDataSstName = "index_cf_sst_meta_sql_data.sst"; + inline static const std::string kDocumentRegionSqlDataSstName = "document_region_sql_data.sst"; + inline static const std::string kDocumentCfSstMetaSqlDataSstName = "document_cf_sst_meta_sql_data.sst"; + + inline static const std::string kBackupMetaDataFileName = "backupmeta.datafile"; + inline static const std::string kBackupMetaSchemaName = "backupmeta.schema"; + inline static const std::string kBackupMetaName = "backupmeta"; + inline static const std::string kBackupMetaEncryptionName = "backupmeta.encryption"; + inline static const std::string kBackupMetaDebugName = "backupmeta.debug"; + + inline static const std::string kIdEpochTypeAndValueKey = "dingodb::pb::meta::IdEpochTypeAndValue"; + inline static const std::string kTableIncrementKey = "dingodb::pb::meta::TableIncrementGroup"; + + inline static const std::string kBackupVersionKey = "pb::common::VersionInfo"; + inline static const std::string kBackupBackupParamKey = "pb::common::BackupParam"; }; } // namespace dingodb diff --git a/src/common/helper.cc b/src/common/helper.cc index fb32383a2..24a81aa82 100644 --- a/src/common/helper.cc +++ b/src/common/helper.cc @@ -14,6 +14,8 @@ #include "common/helper.h" +#include +#include #include #include #include @@ -57,6 +59,7 @@ #include "fmt/core.h" #include "glog/logging.h" #include "google/protobuf/util/json_util.h" +#include "openssl/sha.h" #include "proto/common.pb.h" #include "proto/error.pb.h" #include "proto/node.pb.h" @@ -2233,4 +2236,164 @@ bool Helper::IsSupportSplitAndMerge(const pb::common::RegionDefinition& definiti pb::common::VectorIndexType::VECTOR_INDEX_TYPE_DISKANN; } +void Helper::CalSha1CodeWithString(const std::string& source, std::string& hash_code) { + unsigned char hash[SHA_DIGEST_LENGTH]; + SHA1(reinterpret_cast(source.c_str()), source.size(), hash); + + hash_code.clear(); + for (unsigned char& i : hash) { + hash_code += fmt::format("{:02x}", i); + } +} + +butil::Status Helper::CalSha1CodeWithFile(const std::string& path, std::string& hash_code) { + std::filesystem::path file_path_check(path); + if (std::filesystem::exists(path)) { + std::error_code ec; + if (!std::filesystem::is_regular_file(path, ec)) { + std::string s = fmt::format("file_path : {} is not regular file : {} {}", path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_NOT_REGULAR, s); + } + } else { // data_path not exist + std::string s = fmt::format("file_path : {} not exist", path); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_NOT_EXIST, s); + } + + SHA_CTX s; + unsigned char hash[SHA_DIGEST_LENGTH]; + + SHA1_Init(&s); + + try { + std::ifstream reader(path.c_str(), std::ios::binary); + std::vector buffer; + int64_t remain = 0; + size_t buffer_size = 1024 * 1024; + buffer.resize(buffer_size); + uint32_t real_count = 0; + + while (true) { + reader.read(reinterpret_cast(buffer.data()), buffer_size); + int64_t num = reader.gcount(); + if (0 == num) { + break; + } + SHA1_Update(&s, reinterpret_cast(buffer.data()), num); + } + } catch (const std::exception& e) { + std::string s = fmt::format("read error : {} {}", path, e.what()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_READ, s); + } + + SHA1_Final(hash, &s); + + hash_code.clear(); + for (unsigned char& i : hash) { + hash_code += fmt::format("{:02x}", i); + } + + return butil::Status::OK(); +} + +butil::Status Helper::CalSha1CodeWithFileEx(const std::string& path, std::string& hash_code) { + std::filesystem::path file_path_check(path); + if (std::filesystem::exists(path)) { + std::error_code ec; + if (!std::filesystem::is_regular_file(path, ec)) { + std::string s = fmt::format("file_path : {} is not regular file : {} {}", path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_NOT_REGULAR, s); + } + } else { // data_path not exist + std::string s = fmt::format("file_path : {} not exist", path); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_NOT_EXIST, s); + } + + EVP_MD_CTX* ctx = nullptr; + unsigned char hash[SHA_DIGEST_LENGTH]; + + ctx = EVP_MD_CTX_new(); + EVP_DigestInit_ex(ctx, EVP_sha1(), nullptr); + + try { + std::ifstream reader(path.c_str(), std::ios::binary); + std::vector buffer; + int64_t remain = 0; + size_t buffer_size = 1024 * 1024; + buffer.resize(buffer_size); + uint32_t real_count = 0; + + while (true) { + reader.read(reinterpret_cast(buffer.data()), buffer_size); + int64_t num = reader.gcount(); + if (0 == num) { + break; + } + + EVP_DigestUpdate(ctx, reinterpret_cast(buffer.data()), num); + } + } catch (const std::exception& e) { + EVP_DigestFinal_ex(ctx, hash, nullptr); + EVP_MD_CTX_free(ctx); + std::string s = fmt::format("read error : {} {}", path, e.what()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_READ, s); + } + + EVP_DigestFinal_ex(ctx, hash, nullptr); + EVP_MD_CTX_free(ctx); + + hash_code.clear(); + for (unsigned char& i : hash) { + hash_code += fmt::format("{:02x}", i); + } + + return butil::Status::OK(); +} + +bool Helper::StringConvertTrue(const std::string& str) { + return std::string("true") == str || std::string("TRUE") == str || std::string("True") == str || + std::string("1") == str; +} + +bool Helper::StringConvertFalse(const std::string& str) { + return std::string("false") == str || std::string("FALSE") == str || std::string("False") == str || + std::string("0") == str; +} + +void Helper::HandleBoolControlConfigVariable(const pb::common::ControlConfigVariable& variable, + pb::common::ControlConfigVariable& config, bool& gflags_var) { + bool is_true = Helper::StringConvertTrue(variable.value()); + bool is_false = Helper::StringConvertFalse(variable.value()); + + if (!is_true && !is_false) { + config.set_is_already_set(false); + config.set_is_error_occurred(true); + DINGO_LOG(ERROR) << "ControlConfig variable: " << variable.name() << " value: " << variable.value() + << " is not bool type, skip."; + return; + } + + if (is_true) { + if (gflags_var) { + config.set_is_already_set(true); + } else { + config.set_is_already_set(false); + gflags_var = true; + } + } else if (is_false) { + if (gflags_var) { + config.set_is_already_set(false); + gflags_var = false; + } else { + config.set_is_already_set(true); + } + } + config.set_is_error_occurred(false); +} + } // namespace dingodb diff --git a/src/common/helper.h b/src/common/helper.h index d052f33fb..2e3537a62 100644 --- a/src/common/helper.h +++ b/src/common/helper.h @@ -415,6 +415,15 @@ class Helper { static pb::common::Schema::Type TransformSchemaType(const std::string& name); static bool IsSupportSplitAndMerge(const pb::common::RegionDefinition& definition); + + static void CalSha1CodeWithString(const std::string& source, std::string& hash_code); + static butil::Status CalSha1CodeWithFile(const std::string& path, std::string& hash_code); + static butil::Status CalSha1CodeWithFileEx(const std::string& path, std::string& hash_code); + + static bool StringConvertTrue(const std::string& str); + static bool StringConvertFalse(const std::string& str); + static void HandleBoolControlConfigVariable(const pb::common::ControlConfigVariable& variable, + pb::common::ControlConfigVariable& config, bool& gflags_var); }; } // namespace dingodb diff --git a/src/coordinator/coordinator_control.cc b/src/coordinator/coordinator_control.cc index bc6549ede..2109d4fed 100644 --- a/src/coordinator/coordinator_control.cc +++ b/src/coordinator/coordinator_control.cc @@ -612,4 +612,92 @@ int64_t CoordinatorControl::UpdatePresentId(const pb::coordinator::IdEpochType& return new_id; } +butil::Status CoordinatorControl::GetAllPresentId( + std::vector>& id_epoch_type_values) { + int64_t value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_COORINATOR); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_COORINATOR, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_STORE); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_STORE, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_EXECUTOR); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_EXECUTOR, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_SCHEMA); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_SCHEMA, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_REGION); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_REGION, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_TABLE); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_TABLE, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_INDEX); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_INDEX, + value); // deprecated for index id auto_increment + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_MDS); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_MDS, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_TENANT); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_TENANT, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_COORINATOR); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_COORINATOR, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_STORE); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_STORE, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_EXECUTOR); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_EXECUTOR, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_SCHEMA); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_SCHEMA, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_REGION); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_REGION, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_TABLE); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_TABLE, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::EPOCH_INDEX); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::EPOCH_INDEX, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::RAFT_APPLY_TERM); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::RAFT_APPLY_TERM, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::RAFT_APPLY_INDEX); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::RAFT_APPLY_INDEX, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_REGION_CMD); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_REGION_CMD, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_TASK_LIST); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_TASK_LIST, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_LEASE); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_LEASE, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_REVISION); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_REVISION, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_NEXT_META_WATCH); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_NEXT_META_WATCH, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_GC_SAFE_POINT); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_GC_SAFE_POINT, value); + + // // for online ddl + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_SCHEMA_VERSION); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_SCHEMA_VERSION, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_DDL_JOB); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_DDL_JOB, value); + + value = id_epoch_map_safe_temp_.GetPresentId(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT); + id_epoch_type_values.emplace_back(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT, value); + + return butil::Status::OK(); +} + } // namespace dingodb diff --git a/src/coordinator/coordinator_control.h b/src/coordinator/coordinator_control.h index 3bfa2ad4f..d7399e2a1 100644 --- a/src/coordinator/coordinator_control.h +++ b/src/coordinator/coordinator_control.h @@ -312,7 +312,7 @@ class CoordinatorControl : public MetaControl { butil::Status ChangePeerRegionWithTaskList(int64_t region_id, std::vector &new_store_ids, pb::coordinator_internal::MetaIncrement &meta_increment); butil::Status ChangePairPeerRegionWithTaskList(int64_t region_id, std::vector &new_store_ids, - pb::coordinator_internal::MetaIncrement &meta_increment); + pb::coordinator_internal::MetaIncrement &meta_increment); // transfer leader region butil::Status TransferLeaderRegionWithTaskList(int64_t region_id, int64_t new_leader_store_id, bool is_force, @@ -897,10 +897,14 @@ class CoordinatorControl : public MetaControl { // GC butil::Status UpdateGCSafePoint(int64_t safe_point, pb::coordinator::UpdateGCSafePointRequest::GcFlagType gc_flag, int64_t &new_safe_point, bool &gc_stop, - std::map &tenant_safe_points, + std::map &tenant_safe_points, int64_t resolve_lock_safe_point, + std::map &tenant_resolve_lock_safe_points, + int64_t &new_resolve_lock_safe_point, pb::coordinator_internal::MetaIncrement &meta_increment); butil::Status GetGCSafePoint(int64_t &safe_point, bool &gc_stop, const std::vector &tenant_ids, - bool get_all_tenant, std::map &tenant_safe_points); + bool get_all_tenant, std::map &tenant_safe_points, + int64_t &resolve_lock_safe_point, const std::vector &tenant_resolve_lock_ids, + std::map &resolve_lock_tenant_safe_points); // meta watch static butil::Status MetaWatchSendEvents(int64_t watch_id, std::bitset watch_bitset, @@ -934,11 +938,19 @@ class CoordinatorControl : public MetaControl { butil::Status CreateTenant(pb::meta::Tenant &tenant, pb::coordinator_internal::MetaIncrement &meta_increment); butil::Status DropTenant(int64_t tenant_id, pb::coordinator_internal::MetaIncrement &meta_increment); butil::Status UpdateTenant(pb::meta::Tenant &tenant, pb::coordinator_internal::MetaIncrement &meta_increment); - butil::Status UpdateTenantSafepoint(int64_t tenant_id, int64_t safe_point_ts, + butil::Status UpdateTenantSafepoint(int64_t tenant_id, int64_t safe_point_ts, int64_t resolve_lock_safe_point_ts, pb::coordinator_internal::MetaIncrement &meta_increment); butil::Status GetTenants(std::vector tenant_ids, std::vector &tenants); butil::Status GetAllTenants(std::vector &tenants); + // backup & restore + static butil::Status RegisterBackup(const std::string &backup_name, const std::string &backup_path, + int64_t backup_start_timestamp, int64_t backup_current_timestamp, + int64_t backup_timeout_s); + static butil::Status UnRegisterBackup(const std::string &backup_name); + + butil::Status GetAllPresentId(std::vector> &id_epoch_type_values); + private: butil::Status ValidateTaskListConflict(int64_t region_id, int64_t second_region_id); @@ -1140,6 +1152,42 @@ class MetaWatchCleanTask : public TaskRunnable { int64_t max_outdate_time_ms_; }; +// backup & restore + +struct BrBackupWatchDogInfo { + std::string backup_name; + std::string backup_path; + int64_t backup_start_timestamp; + int64_t backup_current_timestamp; + int64_t backup_timeout_s; + + BrBackupWatchDogInfo(const std::string &backup_name, const std::string &backup_path, int64_t backup_start_timestamp, + int64_t backup_current_timestamp, int64_t backup_timeout_s) + : backup_name(backup_name), + backup_path(backup_path), + backup_start_timestamp(backup_start_timestamp), + backup_current_timestamp(backup_current_timestamp), + backup_timeout_s(backup_timeout_s) {} + + ~BrBackupWatchDogInfo() = default; +}; +class BrWatchDogManager { + public: + static BrWatchDogManager *Instance(); + + butil::Status RegisterBackup(const std::string &backup_name, const std::string &backup_path, + int64_t backup_start_timestamp, int64_t backup_current_timestamp, + int64_t backup_timeout_s); + + butil::Status UnRegisterBackup(const std::string &backup_name); + + private: + BrWatchDogManager(); + ~BrWatchDogManager(); + std::shared_ptr br_backup_watch_dog_info_; + bthread_mutex_t mutex_; +}; + } // namespace dingodb #endif // DINGODB_COORDINATOR_CONTROL_H_ diff --git a/src/coordinator/coordinator_control_coor.cc b/src/coordinator/coordinator_control_coor.cc index 213d5a8bb..4d2d22246 100644 --- a/src/coordinator/coordinator_control_coor.cc +++ b/src/coordinator/coordinator_control_coor.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "config/config_helper.h" #include "coordinator/coordinator_control.h" #include "fmt/core.h" +#include "fmt/format.h" #include "gflags/gflags.h" #include "metrics/coordinator_bvar_metrics.h" #include "proto/common.pb.h" @@ -982,7 +984,7 @@ void CoordinatorControl::GetRegionMap(pb::common::RegionMap& region_map, int64_t region_map_.GetRawMapCopy(region_internal_map_copy); for (auto& element : region_internal_map_copy) { - if (tenant_id == element.second.definition().tenant_id()) { + if (tenant_id == element.second.definition().tenant_id() || tenant_id == -1) { auto* tmp_region = region_map.add_regions(); GenRegionSlim(element.second, *tmp_region); } @@ -5979,14 +5981,16 @@ butil::Status CoordinatorControl::ScanRegions(const std::string& start_key, cons return butil::Status::OK(); } -butil::Status CoordinatorControl::UpdateGCSafePoint(int64_t safe_point, - pb::coordinator::UpdateGCSafePointRequest::GcFlagType gc_flag, - int64_t& new_safe_point, bool& gc_stop, - std::map& tenant_safe_points, - pb::coordinator_internal::MetaIncrement& meta_increment) { +butil::Status CoordinatorControl::UpdateGCSafePoint( + int64_t safe_point, pb::coordinator::UpdateGCSafePointRequest::GcFlagType gc_flag, int64_t& new_safe_point, + bool& gc_stop, std::map& tenant_safe_points, int64_t resolve_lock_safe_point, + std::map& tenant_resolve_lock_safe_points, int64_t& new_resolve_lock_safe_point, + pb::coordinator_internal::MetaIncrement& meta_increment) { DINGO_LOG(INFO) << "UpdateGCSafePoint safe_point=" << safe_point << ", gc_flag=" << pb::coordinator::UpdateGCSafePointRequest::GcFlagType_Name(gc_flag) - << ", tenant_safe_points.count: " << tenant_safe_points.size(); + << ", tenant_safe_points.count: " << tenant_safe_points.size() + << ", resolve_lock_safe_point=" << resolve_lock_safe_point + << ", tenant_resolve_lock_safe_points.count: " << tenant_resolve_lock_safe_points.size(); // update gc_stop pb::coordinator_internal::CommonInternal gc_stop_element; @@ -6041,8 +6045,22 @@ butil::Status CoordinatorControl::UpdateGCSafePoint(int64_t safe_point, UpdatePresentId(pb::coordinator::IdEpochType::ID_GC_SAFE_POINT, new_safe_point, meta_increment); } + // update resolve_lock_safe_point + int64_t now_resolve_lock_safe_point = GetPresentId(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT); + + if (now_resolve_lock_safe_point >= resolve_lock_safe_point) { + DINGO_LOG(WARNING) << "UpdateGCSafePoint now_resolve_lock_safe_point=" << now_resolve_lock_safe_point + << " >= resolve_lock_safe_point=" << resolve_lock_safe_point << ", skip update default"; + new_resolve_lock_safe_point = now_resolve_lock_safe_point; + } else { + new_resolve_lock_safe_point = resolve_lock_safe_point; + UpdatePresentId(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT, new_resolve_lock_safe_point, + meta_increment); + } + // check and update tenant safe_points std::map new_tenant_safe_points; + std::map new_tenant_resolve_lock_safe_points; for (const auto [tenant_id, safe_point_in_map] : tenant_safe_points) { if (tenant_id == 0) { @@ -6068,13 +6086,53 @@ butil::Status CoordinatorControl::UpdateGCSafePoint(int64_t safe_point, } } + for (const auto [tenant_id, resolve_lock_safe_point_in_map] : tenant_resolve_lock_safe_points) { + if (tenant_id == 0) { + continue; + } + + pb::coordinator_internal::TenantInternal tenant; + auto ret = tenant_map_.Get(tenant_id, tenant); + if (ret < 0) { + std::string s = "tenant_map_.Get failed, tenant_id=" + std::to_string(tenant_id); + return butil::Status(pb::error::EINTERNAL, s); + } + + if (tenant.resolve_lock_safe_point() < resolve_lock_safe_point_in_map) { + new_tenant_resolve_lock_safe_points[tenant_id] = tenant.resolve_lock_safe_point(); + auto lambda_find_tenant_function = + [temp_tenant_id = tenant_id](pb::coordinator_internal::MetaIncrementTenant& meta_increment_tenant) { + return meta_increment_tenant.id() == temp_tenant_id; + }; + auto iter = std::find_if(meta_increment.mutable_tenants()->begin(), meta_increment.mutable_tenants()->end(), + lambda_find_tenant_function); + + if (iter != meta_increment.tenants().end()) { + iter->mutable_tenant()->set_update_timestamp(butil::gettimeofday_ms()); + iter->mutable_tenant()->set_resolve_lock_safe_point(resolve_lock_safe_point_in_map); + } else { + auto* tenant_increment = meta_increment.add_tenants(); + tenant_increment->set_id(tenant_id); + tenant_increment->set_op_type(pb::coordinator_internal::MetaIncrementOpType::UPDATE); + auto* increment_tenant = tenant_increment->mutable_tenant(); + *increment_tenant = tenant; + increment_tenant->set_update_timestamp(butil::gettimeofday_ms()); + increment_tenant->set_resolve_lock_safe_point(resolve_lock_safe_point_in_map); + } + } + } + return butil::Status::OK(); } butil::Status CoordinatorControl::GetGCSafePoint(int64_t& safe_point, bool& gc_stop, const std::vector& tenant_ids, bool get_all_tenant, - std::map& tenant_safe_points) { + std::map& tenant_safe_points, + int64_t& resolve_lock_safe_point, + const std::vector& tenant_resolve_lock_ids, + std::map& resolve_lock_tenant_safe_points) { safe_point = GetPresentId(pb::coordinator::IdEpochType::ID_GC_SAFE_POINT); + resolve_lock_safe_point = GetPresentId(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT); pb::coordinator_internal::CommonInternal common; common_disk_meta_->Get(Constant::kGcStopKey, common); gc_stop = (common.value() == Constant::kGcStopValueTrue); @@ -6091,6 +6149,7 @@ butil::Status CoordinatorControl::GetGCSafePoint(int64_t& safe_point, bool& gc_s for (const auto& tenant : tenants) { if (tenant.id() != 0) { tenant_safe_points[tenant.id()] = tenant.safe_point_ts(); + resolve_lock_tenant_safe_points[tenant.id()] = tenant.resolve_lock_safe_point_ts(); } } @@ -6112,6 +6171,23 @@ butil::Status CoordinatorControl::GetGCSafePoint(int64_t& safe_point, bool& gc_s tenant_safe_points[tenant_id] = tenant.safe_point_ts(); } + + // get resolve_lock_safe_points for tenant_resolve_lock_ids + for (const auto tenant_id : tenant_resolve_lock_ids) { + if (tenant_id == 0) { + resolve_lock_tenant_safe_points[tenant_id] = resolve_lock_safe_point; + continue; + } + + pb::coordinator_internal::TenantInternal tenant; + auto ret = tenant_map_.Get(tenant_id, tenant); + if (ret < 0) { + std::string s = "tenant_map_.Get failed, tenant_id=" + std::to_string(tenant_id); + return butil::Status(pb::error::EINTERNAL, s); + } + + resolve_lock_tenant_safe_points[tenant_id] = tenant.resolve_lock_safe_point(); + } } return butil::Status::OK(); @@ -6887,4 +6963,80 @@ butil::Status CoordinatorControl::CreateIds(pb::coordinator::IdEpochType id_epoc return butil::Status::OK(); } +// backup & restore + +BrWatchDogManager::BrWatchDogManager() { bthread_mutex_init(&mutex_, nullptr); } +BrWatchDogManager::~BrWatchDogManager() { bthread_mutex_destroy(&mutex_); } + +BrWatchDogManager* BrWatchDogManager::Instance() { + static BrWatchDogManager instance; + return &instance; +} + +butil::Status BrWatchDogManager::RegisterBackup(const std::string& backup_name, const std::string& backup_path, + int64_t backup_start_timestamp, int64_t backup_current_timestamp, + int64_t backup_timeout_s) { + BAIDU_SCOPED_LOCK(mutex_); + // check exist + if (!br_backup_watch_dog_info_) { + br_backup_watch_dog_info_ = std::make_shared(backup_name, backup_path, backup_start_timestamp, + backup_current_timestamp, backup_timeout_s); + } else { + // get current timestamp + int64_t current_timestamp = Helper::Timestamp(); + // update + if (br_backup_watch_dog_info_->backup_name == backup_name) { + br_backup_watch_dog_info_->backup_current_timestamp = backup_current_timestamp; + } else { // backup name not match + if (current_timestamp > + (br_backup_watch_dog_info_->backup_current_timestamp + br_backup_watch_dog_info_->backup_timeout_s)) { + // timeout, reset + br_backup_watch_dog_info_ = std::make_shared( + backup_name, backup_path, backup_start_timestamp, backup_current_timestamp, backup_timeout_s); + } else { + // not timeout, update backup_current_timestamp + std::string s = fmt::format( + "register backup failed, backup exist. backup name not match, input backup_name={} not match current " + "already exist backup_name={}", + backup_name, br_backup_watch_dog_info_->backup_name); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::EBACKUP_TASK_EXIST, s); + } + } + } + return butil::Status::OK(); +} + +butil::Status BrWatchDogManager::UnRegisterBackup(const std::string& backup_name) { + BAIDU_SCOPED_LOCK(mutex_); + // check exist + if (!br_backup_watch_dog_info_) { + return butil::Status::OK(); + } else { + // backup name equal + if (br_backup_watch_dog_info_->backup_name == backup_name) { + br_backup_watch_dog_info_.reset(); + } else { // backup name not match + std::string s = fmt::format( + "unregister backup failed. backup name not match, input backup_name={} not match current already exist " + "backup_name={}", + backup_name, br_backup_watch_dog_info_->backup_name); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::EBACKUP_TASK_NAME_NOT_MATCH, s); + } + } + return butil::Status::OK(); +} + +butil::Status CoordinatorControl::RegisterBackup(const std::string& backup_name, const std::string& backup_path, + int64_t backup_start_timestamp, int64_t backup_current_timestamp, + int64_t backup_timeout_s) { + return BrWatchDogManager::Instance()->RegisterBackup(backup_name, backup_path, backup_start_timestamp, + backup_current_timestamp, backup_timeout_s); +} + +butil::Status CoordinatorControl::UnRegisterBackup(const std::string& backup_name) { + return BrWatchDogManager::Instance()->UnRegisterBackup(backup_name); +} + } // namespace dingodb diff --git a/src/coordinator/coordinator_control_fsm.cc b/src/coordinator/coordinator_control_fsm.cc index cfc39b2f6..1e5cf898f 100644 --- a/src/coordinator/coordinator_control_fsm.cc +++ b/src/coordinator/coordinator_control_fsm.cc @@ -1037,8 +1037,7 @@ void CoordinatorControl::ApplyMetaIncrement(pb::coordinator_internal::MetaIncrem if (index <= applied_index && term <= applied_term) { DINGO_LOG(WARNING) << "SKIP ApplyMetaIncrement index <= applied_index && term <<= applied_term, just return, [index=" << index - << "][applied_index=" << applied_index << "]" - << "[term=" << term << "][applied_term=" << applied_term << "]"; + << "][applied_index=" << applied_index << "]" << "[term=" << term << "][applied_term=" << applied_term << "]"; return; } else if (meta_increment.ByteSizeLong() > 0) { DINGO_LOG(INFO) << "NORMAL ApplyMetaIncrement [index=" << index << "][applied_index=" << applied_index << "]" @@ -1270,7 +1269,7 @@ void CoordinatorControl::ApplyMetaIncrement(pb::coordinator_internal::MetaIncrem DINGO_LOG(INFO) << "ApplyMetaIncrement tenant_mem UPDATE, but safe_point_ts <= old, [id=" << tenant_increment->id() << "], old_safe_point_ts=" << tenant_internal.safe_point_ts() << ", new_safe_point_ts=" << tenant_increment->tenant().safe_point_ts(); - continue; + goto RESOLVE_LOCK_; } tenant_internal.set_safe_point_ts(tenant_increment->tenant().safe_point_ts()); @@ -1281,6 +1280,26 @@ void CoordinatorControl::ApplyMetaIncrement(pb::coordinator_internal::MetaIncrem tenant_internal.set_update_timestamp(tenant_increment->tenant().update_timestamp()); } + RESOLVE_LOCK_: + // if resolve_lock > 0, update resolve_lock only + // else do not update resolve_lock + if (tenant_increment->tenant().resolve_lock_safe_point() > 0) { + if (tenant_increment->tenant().resolve_lock_safe_point() <= tenant_internal.resolve_lock_safe_point()) { + DINGO_LOG(INFO) << "ApplyMetaIncrement tenant_mem UPDATE, but resolve_lock_safe_point <= old, [id=" + << tenant_increment->id() + << "], old_resolve_lock_safe_point=" << tenant_internal.resolve_lock_safe_point() + << ", new_resolve_lock_safe_point=" << tenant_increment->tenant().resolve_lock_safe_point(); + continue; + } + + tenant_internal.set_resolve_lock_safe_point(tenant_increment->tenant().resolve_lock_safe_point()); + tenant_internal.set_update_timestamp(tenant_increment->tenant().update_timestamp()); + } else { + tenant_internal.set_name(tenant_increment->tenant().name()); + tenant_internal.set_comment(tenant_increment->tenant().comment()); + tenant_internal.set_update_timestamp(tenant_increment->tenant().update_timestamp()); + } + tenant_internal.set_revision(meta_revision); auto ret = tenant_meta_->Put(tenant_increment->id(), tenant_internal); diff --git a/src/coordinator/coordinator_control_meta.cc b/src/coordinator/coordinator_control_meta.cc index 2ea166ff8..3161fada3 100644 --- a/src/coordinator/coordinator_control_meta.cc +++ b/src/coordinator/coordinator_control_meta.cc @@ -3461,6 +3461,8 @@ void CoordinatorControl::GetDefaultTenant(pb::coordinator_internal::TenantIntern tenant_internal.set_update_timestamp(kBaseTimestampMs); tenant_internal.set_safe_point_ts(GetPresentId(pb::coordinator::IdEpochType::ID_GC_SAFE_POINT)); + tenant_internal.set_resolve_lock_safe_point( + GetPresentId(pb::coordinator::IdEpochType::ID_GC_RESOLVE_LOCK_SAFE_POINT)); } butil::Status CoordinatorControl::CreateTenant(pb::meta::Tenant& tenant, @@ -3576,6 +3578,7 @@ butil::Status CoordinatorControl::UpdateTenant(pb::meta::Tenant& tenant, } butil::Status CoordinatorControl::UpdateTenantSafepoint(int64_t tenant_id, int64_t safe_point_ts, + int64_t resolve_lock_safe_point_ts, pb::coordinator_internal::MetaIncrement& meta_increment) { pb::coordinator_internal::TenantInternal tenant_internal; @@ -3599,6 +3602,7 @@ butil::Status CoordinatorControl::UpdateTenantSafepoint(int64_t tenant_id, int64 *increment_tenant = tenant_internal; increment_tenant->set_update_timestamp(butil::gettimeofday_ms()); increment_tenant->set_safe_point_ts(safe_point_ts); + increment_tenant->set_resolve_lock_safe_point(resolve_lock_safe_point_ts); } else { GetDefaultTenant(tenant_internal); } @@ -3661,6 +3665,7 @@ butil::Status CoordinatorControl::GetAllTenants(std::vector& t tenant.set_delete_timestamp(tenant_internal.delete_timestamp()); tenant.set_safe_point_ts(tenant_internal.safe_point_ts()); tenant.set_revision(tenant_internal.revision()); + tenant.set_resolve_lock_safe_point_ts(tenant_internal.resolve_lock_safe_point()); tenants.push_back(std::move(tenant)); @@ -3674,6 +3679,7 @@ butil::Status CoordinatorControl::GetAllTenants(std::vector& t tenant.set_delete_timestamp(tenant_internal.delete_timestamp()); tenant.set_safe_point_ts(tenant_internal.safe_point_ts()); tenant.set_revision(tenant_internal.revision()); + tenant.set_resolve_lock_safe_point_ts(tenant_internal.resolve_lock_safe_point()); tenants.push_back(std::move(tenant)); } diff --git a/src/engine/rocks_raw_engine.cc b/src/engine/rocks_raw_engine.cc index 938fb2e07..f3200fd03 100644 --- a/src/engine/rocks_raw_engine.cc +++ b/src/engine/rocks_raw_engine.cc @@ -117,6 +117,27 @@ butil::Status Iterator::Status() const { return butil::Status(pb::error::EINTERNAL, "Internal iterator error"); } +butil::Status SstFileWriter::SaveFile(const std::map& kvs, const std::string& filename) { + auto status = sst_writer_->Open(filename); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + + for (const auto& [key, value] : kvs) { + status = sst_writer_->Put(key, value); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + } + + status = sst_writer_->Finish(); + if (!status.ok()) { + return butil::Status(status.code(), status.ToString()); + } + + return butil::Status(); +} + butil::Status SstFileWriter::SaveFile(const std::vector& kvs, const std::string& filename) { auto status = sst_writer_->Open(filename); if (!status.ok()) { diff --git a/src/engine/rocks_raw_engine.h b/src/engine/rocks_raw_engine.h index 548fc7824..1ae0771e0 100644 --- a/src/engine/rocks_raw_engine.h +++ b/src/engine/rocks_raw_engine.h @@ -145,6 +145,7 @@ class SstFileWriter { SstFileWriter(SstFileWriter&& rhs) = delete; SstFileWriter& operator=(SstFileWriter&& rhs) = delete; + butil::Status SaveFile(const std::map& kvs, const std::string& filename); butil::Status SaveFile(const std::vector& kvs, const std::string& filename); butil::Status SaveFile(std::shared_ptr iter, const std::string& filename); diff --git a/src/engine/storage.cc b/src/engine/storage.cc index 0030ab047..88097677a 100644 --- a/src/engine/storage.cc +++ b/src/engine/storage.cc @@ -46,6 +46,9 @@ namespace dingodb { +DECLARE_bool(region_enable_auto_split); +DECLARE_bool(region_enable_auto_merge); + Storage::Storage(std::shared_ptr raft_engine, std::shared_ptr mono_engine, mvcc::TsProviderPtr ts_provider) : raft_engine_(raft_engine), mono_engine_(mono_engine), ts_provider_(ts_provider) {} @@ -1650,4 +1653,49 @@ butil::Status Storage::CommitMerge(std::shared_ptr ctx, int64_t job_id, } } +butil::Status Storage::BackupData(std::shared_ptr ctx, store::RegionPtr region, + const pb::common::RegionType& region_type, std::string backup_ts, int64_t backup_tso, + const std::string& storage_path, const pb::common::StorageBackend& storage_backend, + const pb::common::CompressionType& compression_type, int32_t compression_level, + dingodb::pb::store::BackupDataResponse* response) { + RawEnginePtr raw_engine = GetRawEngine(ctx->StoreEngineType(), ctx->RawEngineType()); + + return TxnEngineHelper::BackupData(ctx, raw_engine, region, region_type, backup_ts, backup_tso, storage_path, + storage_backend, compression_type, compression_level, response); +} + +butil::Status Storage::BackupMeta(std::shared_ptr ctx, store::RegionPtr region, + const pb::common::RegionType& region_type, std::string backup_ts, int64_t backup_tso, + const std::string& storage_path, const pb::common::StorageBackend& storage_backend, + const pb::common::CompressionType& compression_type, int32_t compression_level, + dingodb::pb::store::BackupMetaResponse* response) { + RawEnginePtr raw_engine = GetRawEngine(ctx->StoreEngineType(), ctx->RawEngineType()); + + return TxnEngineHelper::BackupMeta(ctx, raw_engine, region, region_type, backup_ts, backup_tso, storage_path, + storage_backend, compression_type, compression_level, response); +} + +butil::Status Storage::ControlConfig(std::shared_ptr /*ctx*/, + const std::vector& variables, + dingodb::pb::store::ControlConfigResponse* response) { + for (const auto& variable : variables) { + pb::common::ControlConfigVariable config; + config.set_name(variable.name()); + config.set_value(variable.value()); + + if ("FLAGS_region_enable_auto_split" == variable.name()) { + Helper::HandleBoolControlConfigVariable(variable, config, FLAGS_region_enable_auto_split); + } else if ("FLAGS_region_enable_auto_merge" == variable.name()) { + Helper::HandleBoolControlConfigVariable(variable, config, FLAGS_region_enable_auto_merge); + } else { + config.set_is_already_set(false); + config.set_is_error_occurred(true); + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << variable.name() << " skip."; + } + + response->mutable_control_config_variable()->Add(std::move(config)); + } + return butil::Status(); +} + } // namespace dingodb diff --git a/src/engine/storage.h b/src/engine/storage.h index 0ab2c8393..6a0254557 100644 --- a/src/engine/storage.h +++ b/src/engine/storage.h @@ -232,6 +232,22 @@ class Storage { const pb::common::RegionDefinition& region_definition, int64_t prepare_merge_log_id, const std::vector& entries); + butil::Status BackupData(std::shared_ptr ctx, store::RegionPtr region, + const pb::common::RegionType& region_type, std::string backup_ts, int64_t backup_tso, + const std::string& storage_path, const pb::common::StorageBackend& storage_backend, + const pb::common::CompressionType& compression_type, int32_t compression_level, + dingodb::pb::store::BackupDataResponse* response); + + butil::Status BackupMeta(std::shared_ptr ctx, store::RegionPtr region, + const pb::common::RegionType& region_type, std::string backup_ts, int64_t backup_tso, + const std::string& storage_path, const pb::common::StorageBackend& storage_backend, + const pb::common::CompressionType& compression_type, int32_t compression_level, + dingodb::pb::store::BackupMetaResponse* response); + + static butil::Status ControlConfig(std::shared_ptr ctx, + const std::vector& variables, + dingodb::pb::store::ControlConfigResponse* response); + private: std::shared_ptr raft_engine_; std::shared_ptr mono_engine_; diff --git a/src/engine/txn_engine_helper.cc b/src/engine/txn_engine_helper.cc index 81465508c..4dfa5c52b 100644 --- a/src/engine/txn_engine_helper.cc +++ b/src/engine/txn_engine_helper.cc @@ -14,10 +14,13 @@ #include "engine/txn_engine_helper.h" +#include + #include #include #include #include +#include #include #include #include @@ -32,9 +35,11 @@ #include "common/helper.h" #include "common/logging.h" #include "common/stream.h" +#include "common/uuid.h" #include "coprocessor/coprocessor_v2.h" #include "document/codec.h" #include "engine/gc_safe_point.h" +#include "engine/rocks_raw_engine.h" #include "fmt/core.h" #include "fmt/format.h" #include "glog/logging.h" @@ -61,6 +66,7 @@ DEFINE_int64(gc_delete_batch_count, 32768, "gc delete batch count"); DEFINE_bool(dingo_log_switch_txn_detail, false, "txn detail log"); DEFINE_bool(dingo_log_switch_txn_gc_detail, false, "txn gc detail log"); +DEFINE_bool(dingo_log_switch_backup_detail, false, "backup detail log"); DECLARE_int64(stream_message_max_bytes); DECLARE_int64(stream_message_max_limit_size); @@ -3217,9 +3223,9 @@ butil::Status TxnEngineHelper::CheckTxnStatus(RawEnginePtr raw_engine, std::shar if (force_sync_commit) { DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_txn_detail) << fmt::format("[txn][region({})] CheckTxnStatus,", region->Id()) - << "fallback is set, check_txn_status treats it as a non-async-commit txn" - << "primary_key: " << primary_key << ", lock_info: " << lock_info.ShortDebugString() - << ", lock_ts: " << lock_ts << ", caller_start_ts: " << caller_start_ts << ", current_ts: " << current_ts; + << "fallback is set, check_txn_status treats it as a non-async-commit txn" << "primary_key: " << primary_key + << ", lock_info: " << lock_info.ShortDebugString() << ", lock_ts: " << lock_ts + << ", caller_start_ts: " << caller_start_ts << ", current_ts: " << current_ts; } else { *txn_result->mutable_locked() = lock_info; // async-commit locks can't be resolved until they expire. @@ -3889,7 +3895,7 @@ butil::Status TxnEngineHelper::ResolveLock(RawEnginePtr raw_engine, std::shared_ } } } // end while iter - } // end scan lock + } // end scan lock if (!lock_infos_to_commit.empty()) { auto ret = DoTxnCommit(raw_engine, raft_engine, ctx, region, lock_infos_to_commit, start_ts, commit_ts); @@ -5342,4 +5348,737 @@ void TxnEngineHelper::RegularDoGcHandler(void * /*arg*/) { DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_txn_gc_detail) << fmt::format("[txn_gc] gc task end."); } +// backup & restore +butil::Status TxnEngineHelper::BackupData(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + std::string backup_ts, int64_t backup_tso, const std::string &storage_path, + const pb::common::StorageBackend &storage_backend, + const pb::common::CompressionType &compression_type, + int32_t compression_level, dingodb::pb::store::BackupDataResponse *response) { + butil::Status status; + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) << fmt::format( + "[backupdata][region({})][type({})] backup data. backup_ts : {} backup_tso : {} storage_path : {} " + "storage_backend : {} compression_type : {} compression_level : {}", + ctx->RegionId(), pb::common::RegionType_Name(region_type), backup_ts, backup_tso, storage_path, + storage_backend.DebugString(), pb::common::CompressionType_Name(compression_type), compression_level); + + if (region == nullptr) { + std::string s = fmt::format("[backupdata][region({})] region not found.", ctx->RegionId()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EREGION_NOT_FOUND, s); + } + + bool is_txn = region->IsTxn(); + + // txn + std::map kv_data; + std::map kv_write; + + // non txn + std::map kv_default; + std::map kv_scalar; + std::map kv_table; + std::map kv_scalar_speedup; + + std::string region_type_name; + + if (region_type == pb::common::RegionType::STORE_REGION && is_txn) { + region_type_name = Constant::kStoreRegionName; + status = DoBackupDataForStoreTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); + } else if (region_type == pb::common::RegionType::STORE_REGION && !is_txn) { + region_type_name = Constant::kStoreRegionName; + status = DoBackupDataForStoreNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, + kv_table, kv_scalar_speedup); + } else if (region_type == pb::common::RegionType::INDEX_REGION && is_txn) { + region_type_name = Constant::kIndexRegionName; + status = DoBackupDataForIndexTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); + } else if (region_type == pb::common::RegionType::INDEX_REGION && !is_txn) { + region_type_name = Constant::kIndexRegionName; + status = DoBackupDataForIndexNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, + kv_table, kv_scalar_speedup); + } else if (region_type == pb::common::RegionType::DOCUMENT_REGION && is_txn) { + region_type_name = Constant::kDocumentRegionName; + status = DoBackupDataForDocumentTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); + } else if (region_type == pb::common::RegionType::DOCUMENT_REGION && !is_txn) { + region_type_name = Constant::kDocumentRegionName; + status = DoBackupDataForDocumentNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, + kv_table, kv_scalar_speedup); + } else { + std::string s = fmt::format("[backupdata][region({})][region_type({})] BackupData invalid region type and txn", + region->Id(), pb::common::RegionType_Name(region_type), (is_txn ? "true" : "false")); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::ENOT_SUPPORT, s); + } + + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + + Helper::CalSha1CodeWithString(region->Range().start_key(), hash_code); + + // second timestamp + int64_t second_timestamp = Helper::Timestamp(); + + std::string backup_file_prefix = + fmt::format("{}_{}_{}_{}", region->Id(), region->EpochToString(), hash_code, second_timestamp); + + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group = response->mutable_sst_metas(); + + int64_t instance_id = Server::GetInstance().Id(); + + if (is_txn) { + status = WriteSstFileForTxn(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + kv_data, kv_write, sst_meta_group); + + } else { + status = + WriteSstFileForNonTxn(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + kv_default, kv_scalar, kv_table, kv_scalar_speedup, sst_meta_group); + } + + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status(); +} + +butil::Status TxnEngineHelper::DoBackupDataCoreTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_data, + std::map &kv_write) { + int64_t start_time_ms = Helper::TimestampMs(); + int64_t end_time_ms = 0; + int64_t total_iter_count = 0; + int64_t region_id = ctx->RegionId(); + std::string region_start_key = region->Range().start_key(); + std::string region_end_key = region->Range().end_key(); + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) + << fmt::format("[backupdata][region({})][type({})][txn] start region start_key: {} end_key: {} backup_tso : {}", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key), backup_tso); + + RawEngine::ReaderPtr reader = raw_engine->Reader(); + std::shared_ptr snapshot = raw_engine->GetSnapshot(); + + IteratorOptions write_iter_options; + write_iter_options.lower_bound = mvcc::Codec::EncodeKey(region_start_key, Constant::kMaxVer); + write_iter_options.upper_bound = mvcc::Codec::EncodeKey(region_end_key, Constant::kMaxVer); + + auto write_iter = reader->NewIterator(Constant::kTxnWriteCF, snapshot, write_iter_options); + if (nullptr == write_iter) { + std::string s = + fmt::format("[backupdata][write][region({})][type({})][txn] NewIterator failed. start_key: {} end_key: {} ", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key)); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EINTERNAL, s); + } + + write_iter->Seek(write_iter_options.lower_bound); + + // this var is too long. this is important var. do not cut down it. + bool is_continue_scan_in_this_write_key = false; + + std::string write_key; + std::string last_write_key; + + while (write_iter->Valid()) { + total_iter_count++; + std::string_view write_iter_key = write_iter->Key(); + std::string_view write_iter_value = write_iter->Value(); + + int64_t write_ts = 0; + + auto ret = mvcc::Codec::DecodeKey(write_iter_key, write_key, write_ts); + if (!ret) { + std::string s = + fmt::format("[backupdata][write][region({})][type({})][txn] decode txn key failed, write_iter->key : {} ", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(write_iter_key)); + DINGO_LOG(FATAL) << s; + } + + // reset for first put + if (last_write_key != write_key) { + is_continue_scan_in_this_write_key = true; + last_write_key = write_key; + } + + // write_ts > backup_tso key value + if (write_ts > backup_tso) { + write_iter->Next(); + continue; + } + + if (!is_continue_scan_in_this_write_key) { + write_iter->Next(); + continue; + } + + pb::store::WriteInfo write_info; + bool parse_success = write_info.ParseFromArray(write_iter_value.data(), write_iter_value.size()); + if (!parse_success) { + std::string s = fmt::format( + "[backupdata][write][region({})][type({})][txn] ParseFromArray failed, write_iter->key : {} " + "write_key : {} write_ts : {}, write_iter->value : {} ", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(write_iter_key), + Helper::StringToHex(write_key), write_ts, Helper::StringToHex(write_iter_value)); + DINGO_LOG(FATAL) << s; + } + + int64_t start_ts = write_info.start_ts(); + pb::store::Op op = write_info.op(); + + // handle write_ts <= backup_tso key value + switch (op) { + case pb::store::Put: + [[fallthrough]]; + case pb::store::Delete: { + butil::Status status = + DoWriteDataAndCheckForTxn(reader, snapshot, region_id, region_type, write_info, write_iter_key, + write_iter_value, write_key, start_ts, write_ts, kv_data, kv_write); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + is_continue_scan_in_this_write_key = false; + break; + } + case pb::store::Rollback: { + // continue scan + break; + } + case pb::store::Lock: + [[fallthrough]]; + case pb::store::PutIfAbsent: + [[fallthrough]]; + case pb::store::None: + [[fallthrough]]; + case pb::store::Op_INT_MIN_SENTINEL_DO_NOT_USE_: + [[fallthrough]]; + case pb::store::Op_INT_MAX_SENTINEL_DO_NOT_USE_: + [[fallthrough]]; + default: { + std::string s = fmt::format( + "[backupdata][write][region({})][type({})][txn] invalid pb::store::Op type : {} type string : " + "{} , write_iter->key : {} write_key : {} write_ts : {} write_iter->value : {} ignore. ", + region_id, pb::common::RegionType_Name(region_type), static_cast(op), pb::store::Op_Name(op), + Helper::StringToHex(write_iter_key), Helper::StringToHex(write_key), write_ts, + write_info.ShortDebugString()); + DINGO_LOG(ERROR) << s; + break; + } + } + + write_iter->Next(); + } + + end_time_ms = Helper::TimestampMs(); + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) << fmt::format( + "[backupdata][region({})][type({})][txn] end region start_key: {} end_key: {} backup_tso " + ": {} time consuming : {} ms total_data_count : {} total_write_count : {} total_iter_count : {} ", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key), backup_tso, (end_time_ms - start_time_ms), kv_data.size(), kv_write.size(), + total_iter_count); + + return butil::Status(); +} + +butil::Status TxnEngineHelper::DoBackupDataCoreNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup) { + int64_t start_time_ms = Helper::TimestampMs(); + int64_t end_time_ms = 0; + int64_t total_delete_count = 0; + int64_t total_iter_count = 0; + int64_t region_id = ctx->RegionId(); + std::string region_start_key = region->Range().start_key(); + std::string region_end_key = region->Range().end_key(); + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) << fmt::format( + "[backupdata][region({})][type({})][nontxn] start region start_key: {} end_key: {} " + "backup_tso : {}", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key), backup_tso); + + RawEngine::ReaderPtr reader = raw_engine->Reader(); + std::shared_ptr snapshot = raw_engine->GetSnapshot(); + + IteratorOptions default_iter_options; + default_iter_options.lower_bound = mvcc::Codec::EncodeKey(region_start_key, Constant::kMaxVer); + default_iter_options.upper_bound = mvcc::Codec::EncodeKey(region_end_key, Constant::kMaxVer); + + const std::string &cf_name = + (region_type == pb::common::RegionType::INDEX_REGION ? Constant::kVectorDataCF : Constant::kStoreDataCF); + + auto default_iter = reader->NewIterator(cf_name, snapshot, default_iter_options); + if (nullptr == default_iter) { + std::string s = fmt::format( + "[backupdata][default][region({})][type({})][nontxn] NewIterator failed. start_key: {} end_key: {} ", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key)); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EINTERNAL, s); + } + + default_iter->Seek(default_iter_options.lower_bound); + + // this var is too long. this is important var. do not cut down it. + bool is_continue_scan_in_this_default_key = false; + + std::string default_key; + std::string last_default_key; + + char prefix = '\0'; + int64_t region_part_id = 0; + if (region_type == pb::common::RegionType::INDEX_REGION) { + prefix = region->GetKeyPrefix(); + region_part_id = region->PartitionId(); + } + + auto lambda_emplace_back_function = [®ion, &prefix, ®ion_part_id, &kv_default, &kv_scalar, &kv_table, + &kv_scalar_speedup, &snapshot, &reader]( + pb::common::RegionType type, std::string_view default_iter_key, + std::string_view default_iter_value, int64_t vector_id, int64_t default_ts) { + // push back to kv_default + { kv_default.insert({std::string(default_iter_key), std::string(default_iter_value)}); } + + if (type == pb::common::RegionType::INDEX_REGION) { + std::string scalar_key = std::string(default_iter_key); + std::string scalar_value; + auto status = reader->KvGet(Constant::kVectorScalarCF, snapshot, scalar_key, scalar_value); + if (status.ok()) { + kv_scalar.insert({scalar_key, scalar_value}); + } + + std::string table_key = std::string(default_iter_key); + std::string table_value; + status = reader->KvGet(Constant::kVectorTableCF, snapshot, table_key, table_value); + if (status.ok()) { + kv_table.insert({table_key, table_value}); + } + + pb::common::ScalarSchema scalar_schema = region->ScalarSchema(); + for (const auto &fields : scalar_schema.fields()) { + if (fields.enable_speed_up()) { + std::string scalar_speedup_key = + VectorCodec::EncodeVectorKey(prefix, region_part_id, vector_id, fields.key(), default_ts); + std::string scalar_speedup_value; + status = + reader->KvGet(Constant::kVectorScalarKeySpeedUpCF, snapshot, scalar_speedup_key, scalar_speedup_value); + if (status.ok()) { + kv_scalar_speedup.insert({scalar_speedup_key, scalar_speedup_value}); + } + } + } + } + }; + + while (default_iter->Valid()) { + total_iter_count++; + std::string_view default_iter_key = default_iter->Key(); + std::string_view default_iter_value = default_iter->Value(); + int64_t default_ts = 0; + + auto ret = mvcc::Codec::DecodeKey(default_iter_key, default_key, default_ts); + if (!ret) { + std::string s = + fmt::format("[backupdata][decode][region({})][type({})][nontxn] decode key failed, key : {} ", region_id, + pb::common::RegionType_Name(region_type), Helper::StringToHex(default_iter_key)); + DINGO_LOG(FATAL) << s; + } + + int64_t vector_id = 0; + int64_t partition_id = 0; + + if (region_type == pb::common::RegionType::INDEX_REGION) { + VectorCodec::DecodeFromEncodeKeyWithTs(std::string(default_iter_key), partition_id, vector_id); + if (region_part_id != partition_id) { + std::string s = fmt::format( + "[backupdata][decode][region({})][type({})][nontxn] decode region_part_id({}) != " + "partition_id({}), " + "key : {} ", + region_id, pb::common::RegionType_Name(region_type), region_part_id, partition_id, + Helper::StringToHex(default_iter_key)); + DINGO_LOG(FATAL) << s; + } + } + + if (last_default_key != default_key) { + is_continue_scan_in_this_default_key = true; + last_default_key = default_key; + } + + // write_ts > backup_tso key value + if (default_ts > backup_tso) { + default_iter->Next(); + continue; + } + + if (!is_continue_scan_in_this_default_key) { + default_iter->Next(); + continue; + } + + mvcc::ValueFlag value_flag = mvcc::Codec::GetValueFlag(default_iter_value); + + switch (value_flag) { + case mvcc::ValueFlag::kPut: + [[fallthrough]]; + case mvcc::ValueFlag::kPutTTL: + [[fallthrough]]; + case mvcc::ValueFlag::kDelete: { + lambda_emplace_back_function(region_type, default_iter_key, default_iter_value, vector_id, default_ts); + is_continue_scan_in_this_default_key = false; + break; + } + + default: { + std::string s = + fmt::format("[backupdata][region({})][type({})][nontxn] invalid mvcc value flag : {}, value : {} ", + region_id, pb::common::RegionType_Name(region_type), static_cast(value_flag), + Helper::StringToHex(default_iter_value)); + DINGO_LOG(FATAL) << s; + } + } + + default_iter->Next(); + } + + end_time_ms = Helper::TimestampMs(); + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) << fmt::format( + "[backupdata][region({})][type({})][nontxn] end region start_key: {} end_key: {} " + "backup_tso " + ": {} time consuming : {} ms total_default_count : {} total_scalar_count : {} total_table_count : {} " + "total_scalar_speedup_count : {} total_iter_count : {}", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(region_start_key), + Helper::StringToHex(region_end_key), backup_tso, (end_time_ms - start_time_ms), kv_default.size(), + kv_scalar.size(), kv_table.size(), kv_scalar_speedup.size(), total_iter_count); + + return butil::Status(); +} + +butil::Status TxnEngineHelper::DoBackupDataForStoreTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_data, + std::map &kv_write) { + return DoBackupDataCoreTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); +} + +butil::Status TxnEngineHelper::DoBackupDataForStoreNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup) { + return DoBackupDataCoreNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, kv_table, + kv_scalar_speedup); +} + +butil::Status TxnEngineHelper::DoBackupDataForIndexTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_data, + std::map &kv_write) { + return DoBackupDataCoreTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); +} + +butil::Status TxnEngineHelper::DoBackupDataForIndexNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup) { + return DoBackupDataCoreNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, kv_table, + kv_scalar_speedup); +} + +butil::Status TxnEngineHelper::DoBackupDataForDocumentTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, + std::map &kv_data, + std::map &kv_write) { + return DoBackupDataCoreTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); +} + +butil::Status TxnEngineHelper::DoBackupDataForDocumentNonTxn( + std::shared_ptr ctx, RawEnginePtr raw_engine, store::RegionPtr region, + const pb::common::RegionType ®ion_type, int64_t backup_tso, std::map &kv_default, + std::map &kv_scalar, std::map &kv_table, + std::map &kv_scalar_speedup) { + return DoBackupDataCoreNonTxn(ctx, raw_engine, region, region_type, backup_tso, kv_default, kv_scalar, kv_table, + kv_scalar_speedup); +} + +butil::Status TxnEngineHelper::DoWriteDataAndCheckForTxn( + RawEngine::ReaderPtr reader, std::shared_ptr snapshot, int64_t region_id, + const pb::common::RegionType ®ion_type, const pb::store::WriteInfo &write_info, std::string_view write_iter_key, + std::string_view write_iter_value, const std::string &write_key, int64_t start_ts, int64_t write_ts, + std::map &kv_data, std::map &kv_write) { + std::string lock_key = mvcc::Codec::EncodeKey(write_key, Constant::kLockVer); + std::string lock_value; + butil::Status status = reader->KvGet(Constant::kTxnLockCF, snapshot, lock_key, lock_value); + if (status.ok()) { + pb::store::LockInfo lock_info; + + if (!lock_value.empty()) { + auto ret = lock_info.ParseFromString(lock_value); + if (!ret) { + DINGO_LOG(FATAL) << "[backupdata] GetLockInfo parse lock info failed, lock_key: " + << Helper::StringToHex(lock_key) << ", lock_value: " << Helper::StringToHex(lock_value); + } + + if (lock_info.lock_ts() == start_ts) { + // exist lock conflict + std::string s = fmt::format( + "[backupdata][write][region({})][type({})][txn] lock conflict, write_key : {} " + "write_ts : {} lock_key : {} lock_ts : {} lock_info : {}", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(write_key), write_ts, + Helper::StringToHex(lock_key), lock_info.lock_ts(), lock_info.ShortDebugString()); + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(pb::error::Errno::EBACKUP_TXN_FOUND_LOCK, s); + } + } + + } else if (status.error_code() != pb::error::Errno::EKEY_NOT_FOUND) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + { kv_write.insert({std::string(write_iter_key), std::string(write_iter_value)}); } + + // try get key from data column family. if not exist , ignore. + std::string data_key = mvcc::Codec::EncodeKey(write_key, write_info.start_ts()); + std::string data_value; + status = reader->KvGet(Constant::kTxnDataCF, snapshot, data_key, data_value); + if (status.ok()) { + kv_data.insert({data_key, data_value}); + } else { + if (pb::error::Errno::EKEY_NOT_FOUND != status.error_code()) { + // other error + std::string s = fmt::format( + "[backupdata][data][region({})][type({})][txn] get key failed. key: {} raw_key : {} " + "start_ts : {} status : {} write_info: {}", + region_id, pb::common::RegionType_Name(region_type), Helper::StringToHex(data_key), + Helper::StringToHex(write_key), write_info.start_ts(), status.error_cstr(), write_info.ShortDebugString()); + DINGO_LOG(ERROR) << s; + return status; + } + } + return butil::Status(); +} + +butil::Status TxnEngineHelper::WriteSstFileForTxn( + store::RegionPtr region, int64_t instance_id, const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, const std::string &backup_file_prefix, + const std::string ®ion_type_name, const std::map &kv_data, + const std::map &kv_write, pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group) { + butil::Status status; + + status = DoWriteSstFile(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + Constant::kTxnWriteCF, kv_write, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = DoWriteSstFile(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + Constant::kTxnDataCF, kv_data, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status(); +} + +butil::Status TxnEngineHelper::WriteSstFileForNonTxn( + store::RegionPtr region, int64_t instance_id, const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, const std::string &backup_file_prefix, + const std::string ®ion_type_name, const std::map &kv_default, + const std::map &kv_scalar, const std::map &kv_table, + const std::map &kv_scalar_speedup, + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group) { + butil::Status status; + + status = DoWriteSstFile( + region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + (region_type == pb::common::RegionType::INDEX_REGION ? Constant::kVectorDataCF : Constant::kStoreDataCF), + kv_default, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = DoWriteSstFile(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + Constant::kVectorScalarCF, kv_scalar, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = DoWriteSstFile(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + Constant::kVectorTableCF, kv_table, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + status = DoWriteSstFile(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + Constant::kVectorScalarKeySpeedUpCF, kv_scalar_speedup, sst_meta_group); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status(); +} + +butil::Status TxnEngineHelper::DoWriteSstFile(store::RegionPtr region, int64_t instance_id, + const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, + const std::string &backup_file_prefix, + const std::string ®ion_type_name, const std::string &cf, + const std::map &kvs, + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group) { + butil::Status status; + + if (kvs.empty()) { + std::string s = fmt::format("[backupdata][region({})][region_type({})] empty. ignore.", region->Id(), + pb::common::RegionType_Name(region_type)); + DINGO_LOG(INFO) << s; + return butil::Status(); + } + + rocksdb::Options options; + rocks::SstFileWriter sst(options); + std::string base_path = storage_backend.local().path(); + std::string dir_name = fmt::format("{}-{}", region_type_name, instance_id); + std::string dir_path = base_path + "/" + dir_name; + + if (std::filesystem::exists(dir_path)) { + std::error_code ec; + if (!std::filesystem::is_directory(dir_path, ec)) { + std::string s = fmt::format("dir_path : {} is not directory, {}, {}", dir_path, ec.value(), ec.message()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EFILE_NOT_DIRECTORY, s); + } + } else { + bool is_success = Helper::CreateDirectory(dir_path); + if (!is_success) { + std::string s = fmt::format("[backupdata][region({})][region_type({})] CreateDirectory failed. dir_path : {}", + region->Id(), pb::common::RegionType_Name(region_type), dir_path); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EBACKUP_CREATE_REMOTE_DIR, s); + } + } + + std::string data_file_name = fmt::format("{}_{}.sst", backup_file_prefix, cf); + std::string data_file_path = dir_path + "/" + data_file_name; + status = sst.SaveFile(kvs, data_file_path); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + int64_t data_file_size = sst.GetSize(); + std::string data_file_hash_code; + status = Helper::CalSha1CodeWithFileEx(data_file_path, data_file_hash_code); + + pb::common::BackupDataFileValueSstMeta *sst_meta = sst_meta_group->add_backup_data_file_value_sst_metas(); + sst_meta->set_cf(cf); + sst_meta->set_region_id(region->Id()); + sst_meta->set_dir_name(dir_name); + sst_meta->set_file_size(data_file_size); + sst_meta->set_encryption(data_file_hash_code); + sst_meta->set_file_name(data_file_name); + + return butil::Status(); +} + +butil::Status TxnEngineHelper::BackupMeta(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + std::string backup_ts, int64_t backup_tso, const std::string &storage_path, + const pb::common::StorageBackend &storage_backend, + const pb::common::CompressionType &compression_type, + int32_t compression_level, dingodb::pb::store::BackupMetaResponse *response) { + butil::Status status; + + DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_backup_detail) << fmt::format( + "[backupmeta][region({})][type({})] backup meta. backup_ts : {} backup_tso : {} storage_path : {} " + "storage_backend : {} compression_type : {} compression_level : {}", + ctx->RegionId(), pb::common::RegionType_Name(region_type), backup_ts, backup_tso, storage_path, + storage_backend.DebugString(), pb::common::CompressionType_Name(compression_type), compression_level); + + if (region == nullptr) { + std::string s = fmt::format("[backupmeta][region({})] region not found.", ctx->RegionId()); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::EREGION_NOT_FOUND, s); + } + + bool is_txn = region->IsTxn(); + + // txn + std::map kv_data; + std::map kv_write; + + std::string region_type_name; + + if (region_type == pb::common::RegionType::STORE_REGION && is_txn) { + region_type_name = Constant::kStoreRegionName; + status = DoBackupDataForStoreTxn(ctx, raw_engine, region, region_type, backup_tso, kv_data, kv_write); + } else { + std::string s = fmt::format("[backupmeta][region({})][region_type({})] backupmeta invalid region type and txn", + region->Id(), pb::common::RegionType_Name(region_type), (is_txn ? "true" : "false")); + DINGO_LOG(ERROR) << s; + return butil::Status(pb::error::Errno::ENOT_SUPPORT, s); + } + + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + std::string hash_code; + + Helper::CalSha1CodeWithString(region->Range().start_key(), hash_code); + + // second timestamp + int64_t second_timestamp = Helper::Timestamp(); + + std::string backup_file_prefix = + fmt::format("{}_{}_{}_{}", region->Id(), region->EpochToString(), hash_code, second_timestamp); + + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group = response->mutable_sst_metas(); + + int64_t instance_id = Server::GetInstance().Id(); + + if (is_txn) { + status = WriteSstFileForTxn(region, instance_id, region_type, storage_backend, backup_file_prefix, region_type_name, + kv_data, kv_write, sst_meta_group); + } + + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + return butil::Status(); +} + } // namespace dingodb diff --git a/src/engine/txn_engine_helper.h b/src/engine/txn_engine_helper.h index 8c497da66..d773f78d7 100644 --- a/src/engine/txn_engine_helper.h +++ b/src/engine/txn_engine_helper.h @@ -331,6 +331,100 @@ class TxnEngineHelper { int64_t start_ts, int64_t mutation_size, std::vector &kv_puts_data, std::vector &kv_puts_lock); + + // backup & restore + static butil::Status BackupData(std::shared_ptr ctx, RawEnginePtr raw_engine, store::RegionPtr region, + const pb::common::RegionType ®ion_type, std::string backup_ts, int64_t backup_tso, + const std::string &storage_path, const pb::common::StorageBackend &storage_backend, + const pb::common::CompressionType &compression_type, int32_t compression_level, + dingodb::pb::store::BackupDataResponse *response); + + static butil::Status DoBackupDataCoreTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_data, + std::map &kv_write); + + static butil::Status DoBackupDataCoreNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup); + + static butil::Status DoBackupDataForStoreTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_data, + std::map &kv_write); + + static butil::Status DoBackupDataForStoreNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup); + + static butil::Status DoBackupDataForIndexTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_data, + std::map &kv_write); + + static butil::Status DoBackupDataForIndexNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup); + + static butil::Status DoBackupDataForDocumentTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_data, + std::map &kv_write); + + static butil::Status DoBackupDataForDocumentNonTxn(std::shared_ptr ctx, RawEnginePtr raw_engine, + store::RegionPtr region, const pb::common::RegionType ®ion_type, + int64_t backup_tso, std::map &kv_default, + std::map &kv_scalar, + std::map &kv_table, + std::map &kv_scalar_speedup); + + static butil::Status DoWriteDataAndCheckForTxn(RawEngine::ReaderPtr reader, std::shared_ptr snapshot, + int64_t region_id, const pb::common::RegionType ®ion_type, + const pb::store::WriteInfo &write_info, + std::string_view write_iter_key, std::string_view write_iter_value, + const std::string &write_key, int64_t start_ts, int64_t write_ts, + std::map &kv_data, + std::map &kv_write); + + static butil::Status WriteSstFileForTxn(store::RegionPtr region, int64_t instance_id, + const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, + const std::string &backup_file_prefix, const std::string ®ion_type_name, + const std::map &kv_data, + const std::map &kv_write, + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group); + + static butil::Status WriteSstFileForNonTxn(store::RegionPtr region, int64_t instance_id, + const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, + const std::string &backup_file_prefix, const std::string ®ion_type_name, + const std::map &kv_default, + const std::map &kv_scalar, + const std::map &kv_table, + const std::map &kv_scalar_speedup, + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group); + + static butil::Status DoWriteSstFile(store::RegionPtr region, int64_t instance_id, + const pb::common::RegionType ®ion_type, + const pb::common::StorageBackend &storage_backend, + const std::string &backup_file_prefix, const std::string ®ion_type_name, + const std::string &cf, const std::map &kvs, + pb::common::BackupDataFileValueSstMetaGroup *sst_meta_group); + + static butil::Status BackupMeta(std::shared_ptr ctx, RawEnginePtr raw_engine, store::RegionPtr region, + const pb::common::RegionType ®ion_type, std::string backup_ts, int64_t backup_tso, + const std::string &storage_path, const pb::common::StorageBackend &storage_backend, + const pb::common::CompressionType &compression_type, int32_t compression_level, + dingodb::pb::store::BackupMetaResponse *response); }; } // namespace dingodb diff --git a/src/server/coordinator_service.cc b/src/server/coordinator_service.cc index de31ec657..dd2e036f8 100644 --- a/src/server/coordinator_service.cc +++ b/src/server/coordinator_service.cc @@ -59,6 +59,9 @@ BRPC_VALIDATE_GFLAG(hello_latency_ms, brpc::NonNegativeInteger); DECLARE_int32(default_replica_num); +DECLARE_bool(enable_balance_leader); +DECLARE_bool(enable_balance_region); + void DoCoordinatorHello(google::protobuf::RpcController * /*controller*/, const pb::coordinator::HelloRequest *request, pb::coordinator::HelloResponse *response, TrackClosure *done, std::shared_ptr coordinator_control, @@ -2280,8 +2283,16 @@ void DoUpdateGCSafePoint(google::protobuf::RpcController * /*controller*/, tenant_safe_points.insert({tenant_id, safe_point}); } - auto ret = coordinator_control->UpdateGCSafePoint(request->safe_point(), request->gc_flag(), new_gc_safe_point, - gc_stop, tenant_safe_points, meta_increment); + std::map tenant_resolve_lock_safe_points; + for (const auto &[tenant_id, resolve_lock_safe_point] : request->tenant_resolve_lock_safe_points()) { + tenant_resolve_lock_safe_points.insert({tenant_id, resolve_lock_safe_point}); + } + + int64_t new_resolve_lock_safe_point = 0; + + auto ret = coordinator_control->UpdateGCSafePoint( + request->safe_point(), request->gc_flag(), new_gc_safe_point, gc_stop, tenant_safe_points, + request->resolve_lock_safe_point(), tenant_resolve_lock_safe_points, new_resolve_lock_safe_point, meta_increment); if (!ret.ok()) { DINGO_LOG(ERROR) << "UpdateGCSafePoint failed, gc_safe_point:" << request->safe_point() << ", errcode:" << ret.error_code() << ", errmsg:" << ret.error_str(); @@ -2292,6 +2303,7 @@ void DoUpdateGCSafePoint(google::protobuf::RpcController * /*controller*/, response->set_new_safe_point(new_gc_safe_point); response->set_gc_stop(gc_stop); + response->set_new_safe_point(new_resolve_lock_safe_point); if (meta_increment.ByteSizeLong() == 0) { return; @@ -2325,6 +2337,7 @@ void DoGetGCSafePoint(google::protobuf::RpcController * /*controller*/, } int64_t gc_safe_point = 0; + int64_t gc_resolve_lock_safe_point = 0; bool gc_stop = false; std::vector tenant_ids; @@ -2335,10 +2348,20 @@ void DoGetGCSafePoint(google::protobuf::RpcController * /*controller*/, } } + std::vector tenant_resolve_lock_ids; + if (!request->tenant_resolve_lock_ids().empty()) { + tenant_resolve_lock_ids.reserve(request->tenant_resolve_lock_ids().size()); + for (const auto &tenant_id : request->tenant_resolve_lock_ids()) { + tenant_resolve_lock_ids.push_back(tenant_id); + } + } + std::map tenant_safe_points; + std::map resolve_lock_tenant_safe_points; auto ret = coordinator_control->GetGCSafePoint(gc_safe_point, gc_stop, tenant_ids, request->get_all_tenant(), - tenant_safe_points); + tenant_safe_points, gc_resolve_lock_safe_point, + tenant_resolve_lock_ids, resolve_lock_tenant_safe_points); if (!ret.ok()) { DINGO_LOG(ERROR) << "GetGCSafePoint failed, errcode:" << ret.error_code() << ", errmsg:" << ret.error_str(); response->mutable_error()->set_errcode(static_cast(ret.error_code())); @@ -2346,6 +2369,7 @@ void DoGetGCSafePoint(google::protobuf::RpcController * /*controller*/, } response->set_safe_point(gc_safe_point); + response->set_resolve_lock_safe_point(gc_resolve_lock_safe_point); response->set_gc_stop(gc_stop); if (!tenant_safe_points.empty()) { @@ -2355,6 +2379,13 @@ void DoGetGCSafePoint(google::protobuf::RpcController * /*controller*/, } } + if (!resolve_lock_tenant_safe_points.empty()) { + auto *new_tenant_resolve_lock_safe_points = response->mutable_tenant_resolve_lock_safe_points(); + for (const auto [tenant_id, resolve_lock_safe_point] : resolve_lock_tenant_safe_points) { + new_tenant_resolve_lock_safe_points->insert({tenant_id, resolve_lock_safe_point}); + } + } + DINGO_LOG(INFO) << "Response GetGCSafePoint Request:" << response->ShortDebugString(); } @@ -3839,4 +3870,144 @@ void CoordinatorServiceImpl::CreateIds(google::protobuf::RpcController *controll } } +// backup & restore +void DoRegisterBackup(google::protobuf::RpcController * /*controller*/, + const pb::coordinator::RegisterBackupRequest *request, + pb::coordinator::RegisterBackupResponse *response, TrackClosure *done, + std::shared_ptr coordinator_control, + std::shared_ptr /*raft_engine*/) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(INFO) << request->ShortDebugString(); + + auto ret = coordinator_control->RegisterBackup(request->backup_name(), request->backup_path(), + request->backup_start_timestamp(), request->backup_current_timestamp(), + request->backup_timeout_s()); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "RegisterBackup failed in coordinator_service"; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + DINGO_LOG(INFO) << "RegisterBackup Success. backup_name = " << request->backup_name(); +} +void CoordinatorServiceImpl::RegisterBackup(google::protobuf::RpcController *controller, + const pb::coordinator::RegisterBackupRequest *request, + pb::coordinator::RegisterBackupResponse *response, + google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(DEBUG) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoRegisterBackup(controller, request, response, svr_done, coordinator_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + +void DoUnRegisterBackup(google::protobuf::RpcController * /*controller*/, + const pb::coordinator::UnRegisterBackupRequest *request, + pb::coordinator::UnRegisterBackupResponse *response, TrackClosure *done, + std::shared_ptr coordinator_control, + std::shared_ptr /*raft_engine*/) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(DEBUG) << request->ShortDebugString(); + + auto ret = coordinator_control->UnRegisterBackup(request->backup_name()); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "UnRegisterBackup failed in coordinator_service"; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + DINGO_LOG(INFO) << "UnRegisterBackup Success. backup_name = " << request->backup_name(); +} + +void CoordinatorServiceImpl::UnRegisterBackup(google::protobuf::RpcController *controller, + const pb::coordinator::UnRegisterBackupRequest *request, + pb::coordinator::UnRegisterBackupResponse *response, + google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(DEBUG) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoUnRegisterBackup(controller, request, response, svr_done, coordinator_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + +void DoControlConfig(google::protobuf::RpcController * /*controller*/, + const pb::coordinator::ControlConfigRequest *request, + pb::coordinator::ControlConfigResponse *response, TrackClosure *done, + std::shared_ptr /*coordinator_control*/, + std::shared_ptr /*raft_engine*/) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(DEBUG) << request->ShortDebugString(); + + for (const auto &variable : request->control_config_variable()) { + pb::common::ControlConfigVariable config; + config.set_name(variable.name()); + config.set_value(variable.value()); + + if ("FLAGS_enable_balance_leader" == variable.name()) { + Helper::HandleBoolControlConfigVariable(variable, config, FLAGS_enable_balance_leader); + } else if ("FLAGS_enable_balance_region" == variable.name()) { + Helper::HandleBoolControlConfigVariable(variable, config, FLAGS_enable_balance_region); + } else { + config.set_is_already_set(false); + config.set_is_error_occurred(true); + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << variable.name() << " skip."; + } + + response->mutable_control_config_variable()->Add(std::move(config)); + } + + butil::Status ret; + if (!ret.ok()) { + DINGO_LOG(ERROR) << "ControlConfig failed in coordinator_service"; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + DINGO_LOG(INFO) << "ControlConfig Success. " << response->DebugString(); +} + +void CoordinatorServiceImpl::ControlConfig(google::protobuf::RpcController *controller, + const pb::coordinator::ControlConfigRequest *request, + pb::coordinator::ControlConfigResponse *response, + google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(DEBUG) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoControlConfig(controller, request, response, svr_done, coordinator_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + } // namespace dingodb diff --git a/src/server/coordinator_service.h b/src/server/coordinator_service.h index d823d7c4c..14e48bdc0 100644 --- a/src/server/coordinator_service.h +++ b/src/server/coordinator_service.h @@ -310,6 +310,18 @@ class CoordinatorServiceImpl : public pb::coordinator::CoordinatorService { void CreateIds(google::protobuf::RpcController* controller, const pb::coordinator::CreateIdsRequest* request, pb::coordinator::CreateIdsResponse* response, google::protobuf::Closure* done) override; + // backup & restore + void RegisterBackup(google::protobuf::RpcController* controller, + const pb::coordinator::RegisterBackupRequest* request, + pb::coordinator::RegisterBackupResponse* response, google::protobuf::Closure* done) override; + + void UnRegisterBackup(google::protobuf::RpcController* controller, + const pb::coordinator::UnRegisterBackupRequest* request, + pb::coordinator::UnRegisterBackupResponse* response, google::protobuf::Closure* done) override; + + void ControlConfig(google::protobuf::RpcController* controller, const pb::coordinator::ControlConfigRequest* request, + pb::coordinator::ControlConfigResponse* response, google::protobuf::Closure* done) override; + void SetWorkSet(WorkerSetPtr worker_set) { worker_set_ = worker_set; } private: diff --git a/src/server/document_service.cc b/src/server/document_service.cc index c637cfae7..c092f1645 100644 --- a/src/server/document_service.cc +++ b/src/server/document_service.cc @@ -2441,6 +2441,107 @@ void DocumentServiceImpl::TxnDeleteRange(google::protobuf::RpcController* contro "WorkerSet queue is full, please wait and retry"); } } +static butil::Status ValidateBackupDataRangeRequest(const dingodb::pb::store::BackupDataRequest* request, + store::RegionPtr region) { + // check if region_epoch is match + auto status = ServiceHelper::ValidateRegionEpoch(request->context().region_epoch(), region); + if (!status.ok()) { + return status; + } + + pb::common::Range req_range; + req_range.set_start_key(request->start_key()); + req_range.set_end_key(request->end_key()); + + status = ServiceHelper::ValidateRange(req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRangeInRange(region->Range(false), req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRegionState(region); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateClusterReadOnly(); + if (!status.ok()) { + return status; + } + + return butil::Status(); +} + +static void DoBackupData(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto region = done->GetRegion(); + + auto status = ValidateBackupDataRangeRequest(request, region); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + ServiceHelper::GetStoreRegionInfo(region, response->mutable_error()); + return; + } + + // check leader if need + if (request->need_leader()) { + status = storage->ValidateLeader(region); + if (!status.ok()) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + return; + } + } + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetRegionId(request->context().region_id()); + ctx->SetTracker(tracker); + ctx->SetCfName(Constant::kStoreDataCF); + ctx->SetRegionEpoch(request->context().region_epoch()); + ctx->SetIsolationLevel(request->context().isolation_level()); + ctx->SetRawEngineType(region->GetRawEngineType()); + ctx->SetStoreEngineType(region->GetStoreEngineType()); + + status = storage->BackupData(ctx, region, request->region_type(), request->backup_ts(), request->backup_tso(), + request->storage_path(), request->storage_backend(), request->compression_type(), + request->compression_level(), response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} + +void DocumentServiceImpl::BackupData(google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, + google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure(__func__, done, request, response); + + if (BAIDU_UNLIKELY(svr_done->GetRegion() == nullptr)) { + brpc::ClosureGuard done_guard(svr_done); + return; + } + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoBackupData(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} static butil::Status ValidateTxnDumpRequest(const pb::store::TxnDumpRequest* request, store::RegionPtr region) { // check if region_epoch is match @@ -2552,7 +2653,8 @@ void DoHello(google::protobuf::RpcController* controller, const dingodb::pb::doc void DocumentServiceImpl::Hello(google::protobuf::RpcController* controller, const pb::document::HelloRequest* request, pb::document::HelloResponse* response, google::protobuf::Closure* done) { // Run in queue. - auto* svr_done = new ServiceClosure(__func__, done, request, response); + auto* svr_done = + new ServiceClosure(__func__, done, request, response); auto task = std::make_shared([=]() { DoHello(controller, request, response, svr_done); }); @@ -2568,7 +2670,7 @@ void DocumentServiceImpl::GetMemoryInfo(google::protobuf::RpcController* control const pb::document::HelloRequest* request, pb::document::HelloResponse* response, google::protobuf::Closure* done) { // Run in queue. - auto* svr_done = new ServiceClosure(__func__, done, request, response); + auto* svr_done = new ServiceClosure(__func__, done, request, response); auto task = std::make_shared([=]() { DoHello(controller, request, response, svr_done, true); }); diff --git a/src/server/document_service.h b/src/server/document_service.h index 6278a4af7..ee0727ac7 100644 --- a/src/server/document_service.h +++ b/src/server/document_service.h @@ -99,6 +99,10 @@ class DocumentServiceImpl : public pb::document::DocumentService { void TxnDeleteRange(google::protobuf::RpcController* controller, const pb::store::TxnDeleteRangeRequest* request, pb::store::TxnDeleteRangeResponse* response, google::protobuf::Closure* done) override; + // backup & restore + void BackupData(google::protobuf::RpcController* controller, const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, google::protobuf::Closure* done) override; + void SetStorage(StoragePtr storage) { storage_ = storage; } void SetReadWorkSet(WorkerSetPtr worker_set) { read_worker_set_ = worker_set; } void SetWriteWorkSet(WorkerSetPtr worker_set) { write_worker_set_ = worker_set; } diff --git a/src/server/index_service.cc b/src/server/index_service.cc index 6cff70d0c..d83d33aa7 100644 --- a/src/server/index_service.cc +++ b/src/server/index_service.cc @@ -3601,6 +3601,147 @@ void IndexServiceImpl::TxnDeleteRange(google::protobuf::RpcController* controlle } } +static butil::Status ValidateBackupDataRangeRequest(const dingodb::pb::store::BackupDataRequest* request, + store::RegionPtr region) { + // check if region_epoch is match + auto status = ServiceHelper::ValidateRegionEpoch(request->context().region_epoch(), region); + if (!status.ok()) { + return status; + } + + pb::common::Range req_range; + req_range.set_start_key(request->start_key()); + req_range.set_end_key(request->end_key()); + + status = ServiceHelper::ValidateRange(req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRangeInRange(region->Range(false), req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRegionState(region); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateClusterReadOnly(); + if (!status.ok()) { + return status; + } + + return butil::Status(); +} + +static void DoBackupData(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto region = done->GetRegion(); + + auto status = ValidateBackupDataRangeRequest(request, region); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + ServiceHelper::GetStoreRegionInfo(region, response->mutable_error()); + return; + } + + // check leader if need + if (request->need_leader()) { + status = storage->ValidateLeader(region); + if (!status.ok()) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + return; + } + } + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetRegionId(request->context().region_id()); + ctx->SetTracker(tracker); + ctx->SetCfName(Constant::kStoreDataCF); + ctx->SetRegionEpoch(request->context().region_epoch()); + ctx->SetIsolationLevel(request->context().isolation_level()); + ctx->SetRawEngineType(region->GetRawEngineType()); + ctx->SetStoreEngineType(region->GetStoreEngineType()); + + status = storage->BackupData(ctx, region, request->region_type(), request->backup_ts(), request->backup_tso(), + request->storage_path(), request->storage_backend(), request->compression_type(), + request->compression_level(), response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} + +void IndexServiceImpl::BackupData(google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure(__func__, done, request, response); + + if (BAIDU_UNLIKELY(svr_done->GetRegion() == nullptr)) { + brpc::ClosureGuard done_guard(svr_done); + return; + } + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoBackupData(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} + +static void DoControlConfig(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::ControlConfigRequest* request, + dingodb::pb::store::ControlConfigResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetTracker(tracker); + + std::vector variables; + for (const auto& variable : request->control_config_variable()) { + variables.push_back(variable); + } + + auto status = storage->ControlConfig(ctx, variables, response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} +void IndexServiceImpl::ControlConfig(google::protobuf::RpcController* controller, + const pb::store::ControlConfigRequest* request, + pb::store::ControlConfigResponse* response, google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure( + __func__, done, request, response); + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoControlConfig(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} + static butil::Status ValidateTxnDumpRequest(const pb::store::TxnDumpRequest* request, store::RegionPtr region) { // check if region_epoch is match auto epoch_ret = ServiceHelper::ValidateRegionEpoch(request->context().region_epoch(), region); diff --git a/src/server/index_service.h b/src/server/index_service.h index 5be8ebe89..fa386acfd 100644 --- a/src/server/index_service.h +++ b/src/server/index_service.h @@ -118,6 +118,13 @@ class IndexServiceImpl : public pb::index::IndexService { void TxnDeleteRange(google::protobuf::RpcController* controller, const pb::store::TxnDeleteRangeRequest* request, pb::store::TxnDeleteRangeResponse* response, google::protobuf::Closure* done) override; + // backup & restore + void BackupData(google::protobuf::RpcController* controller, const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, google::protobuf::Closure* done) override; + + void ControlConfig(google::protobuf::RpcController* controller, const pb::store::ControlConfigRequest* request, + pb::store::ControlConfigResponse* response, google::protobuf::Closure* done) override; + void SetStorage(StoragePtr storage) { storage_ = storage; } void SetReadWorkSet(WorkerSetPtr worker_set) { read_worker_set_ = worker_set; } void SetWriteWorkSet(WorkerSetPtr worker_set) { write_worker_set_ = worker_set; } diff --git a/src/server/meta_service.cc b/src/server/meta_service.cc index 3df8b4f63..9ab19d5f9 100644 --- a/src/server/meta_service.cc +++ b/src/server/meta_service.cc @@ -27,6 +27,7 @@ #include "coordinator/auto_increment_control.h" #include "coordinator/coordinator_control.h" #include "coordinator/tso_control.h" +#include "fmt/format.h" #include "gflags/gflags.h" #include "proto/common.pb.h" #include "proto/coordinator_internal.pb.h" @@ -810,6 +811,52 @@ void DoCreateAutoIncrement(google::protobuf::RpcController *controller, } } +void DoCreateAutoIncrements(google::protobuf::RpcController *controller, + const pb::meta::CreateAutoIncrementsRequest *request, + pb::meta::CreateAutoIncrementsResponse *response, TrackClosure *done, + std::shared_ptr auto_increment_control, + std::shared_ptr raft_engine) { + brpc::ClosureGuard done_guard(done); + + DINGO_LOG(INFO) << request->ShortDebugString(); + + pb::coordinator_internal::MetaIncrement meta_increment; + + for (const auto &table_increment : request->table_increment_group().table_increments()) { + if (!auto_increment_control->IsLeader()) { + return auto_increment_control->RedirectResponse(response); + } + + auto table_id = table_increment.table_id(); + auto start_id = table_increment.start_id(); + + auto ret = auto_increment_control->CreateAutoIncrement(table_id, start_id, meta_increment); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "failed, " << table_id << " | " << start_id; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + } + + std::shared_ptr ctx = + std::make_shared(static_cast(controller), nullptr, response); + ctx->SetRegionId(Constant::kAutoIncrementRegionId); + ctx->SetTracker(done->Tracker()); + + // this is a async operation will be block by closure + auto ret2 = raft_engine->Write(ctx, WriteDataBuilder::BuildWrite(ctx->CfName(), meta_increment)); + if (!ret2.ok()) { + DINGO_LOG(ERROR) << "failed, " << " | " << ret2.error_str(); + ServiceHelper::SetError(response->mutable_error(), ret2.error_code(), ret2.error_str()); + + if (ret2.error_code() == pb::error::Errno::ERAFT_NOTLEADER) { + auto_increment_control->RedirectResponse(response); + } + return; + } +} + void DoUpdateAutoIncrement(google::protobuf::RpcController * /*controller*/, const ::dingodb::pb::meta::UpdateAutoIncrementRequest *request, pb::meta::UpdateAutoIncrementResponse *response, TrackClosure *done, @@ -2619,6 +2666,29 @@ void MetaServiceImpl::CreateAutoIncrement(google::protobuf::RpcController *contr } } +void MetaServiceImpl::CreateAutoIncrements(google::protobuf::RpcController *controller, + const pb::meta::CreateAutoIncrementsRequest *request, + pb::meta::CreateAutoIncrementsResponse *response, + google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + if (!auto_increment_control_->IsLeader()) { + return RedirectAutoIncrementResponse(response); + } + DINGO_LOG(INFO) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoCreateAutoIncrements(controller, request, response, svr_done, auto_increment_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + void MetaServiceImpl::UpdateAutoIncrement(google::protobuf::RpcController *controller, const ::dingodb::pb::meta::UpdateAutoIncrementRequest *request, pb::meta::UpdateAutoIncrementResponse *response, @@ -3496,6 +3566,177 @@ void MetaServiceImpl::ListWatch(google::protobuf::RpcController *controller, con } } +void DoExportMeta(google::protobuf::RpcController * /*controller*/, const pb::meta::ExportMetaRequest *request, + pb::meta::ExportMetaResponse *response, TrackClosure *done, + std::shared_ptr coordinator_control, std::shared_ptr /*raft_engine*/) { + brpc::ClosureGuard done_guard(done); + + if (!coordinator_control->IsLeader()) { + return coordinator_control->RedirectResponse(response); + } + + DINGO_LOG(INFO) << request->ShortDebugString(); + + // 1. get all tenants + std::vector tenants; + auto ret = coordinator_control->GetAllTenants(tenants); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "GetTenants failed in meta_service, error code=" << ret; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + // 2. get all schema + for (auto &tenant : tenants) { + std::vector schemas; + ret = coordinator_control->GetSchemas(tenant.id(), schemas); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "GetSchemas failed in meta_service, error code=" << ret; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + pb::meta::Schemas pb_schemas; + for (const auto &schema : schemas) { + // Schema DingoCommonId + std::string schema_dingo_common_id = fmt::format("{}|{}|{}", static_cast(schema.id().entity_type()), + schema.id().parent_entity_id(), schema.id().entity_id()); + pb::meta::TablesAndIndexes tables_and_indexes; + + // 3.1 get all table + std::vector table_definition_with_ids; + ret = coordinator_control->GetTables(schema.id().entity_id(), table_definition_with_ids); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "GetTables failed in meta_service, error code=" << ret; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + for (const auto &table_definition_with_id : table_definition_with_ids) { + tables_and_indexes.add_tables()->CopyFrom(table_definition_with_id); + } + + table_definition_with_ids.clear(); + + // 3.2 get all index + auto ret = coordinator_control->GetIndexes(schema.id().entity_id(), table_definition_with_ids); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "GetIndexes failed in meta_service, error code=" << ret; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + // 3.2.1 add index_definition_with_id + std::vector index_definition_with_ids; + for (const auto &table_definition_with_id : table_definition_with_ids) { + pb::meta::IndexDefinitionWithId index_definition_with_id; + *(index_definition_with_id.mutable_index_id()) = table_definition_with_id.table_id(); + + MetaServiceImpl::TableDefinitionToIndexDefinition(table_definition_with_id.table_definition(), + *(index_definition_with_id.mutable_index_definition())); + index_definition_with_id.set_tenant_id(tenant.id()); + index_definition_with_ids.emplace_back(index_definition_with_id); + } + + for (const auto &index_definition_with_id : index_definition_with_ids) { + tables_and_indexes.add_indexes()->CopyFrom(index_definition_with_id); + } + + (*response->mutable_meta_all()->mutable_tables_and_indexes())[schema_dingo_common_id] = tables_and_indexes; + + // copy schema + pb_schemas.add_schemas()->CopyFrom(schema); + } + + (*response->mutable_meta_all()->mutable_schemas())[tenant.id()] = pb_schemas; + + response->mutable_meta_all()->add_tenants()->CopyFrom(tenant); + } + + DINGO_LOG(INFO) << "ExportMeta Success. response: " << response->ShortDebugString(); +} + +void MetaServiceImpl::ExportMeta(google::protobuf::RpcController *controller, + const pb::meta::ExportMetaRequest *request, pb::meta::ExportMetaResponse *response, + google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + if (!coordinator_control_->IsLeader()) { + return RedirectResponse(response); + } + + DINGO_LOG(INFO) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoExportMeta(controller, request, response, svr_done, coordinator_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + +void DoSaveIdEpochType(google::protobuf::RpcController * /*controller*/, + const pb::meta::SaveIdEpochTypeRequest *request, pb::meta::SaveIdEpochTypeResponse *response, + TrackClosure *done, std::shared_ptr coordinator_control, + std::shared_ptr /*raft_engine*/) { + brpc::ClosureGuard done_guard(done); + + if (!coordinator_control->IsLeader()) { + return coordinator_control->RedirectResponse(response); + } + + DINGO_LOG(INFO) << request->ShortDebugString(); + + std::vector> id_epoch_type_values; + butil::Status ret = coordinator_control->GetAllPresentId(id_epoch_type_values); + if (!ret.ok()) { + DINGO_LOG(ERROR) << "GetAllGetPresentId failed."; + response->mutable_error()->set_errcode(static_cast(ret.error_code())); + response->mutable_error()->set_errmsg(ret.error_str()); + return; + } + + for (const auto &[type, value] : id_epoch_type_values) { + pb::meta::IdEpochTypeAndValueItem type_value; + type_value.set_type(type); + type_value.set_value(value); + response->mutable_id_epoch_type_and_value()->mutable_items()->Add()->CopyFrom(type_value); + } + + DINGO_LOG(INFO) << "SaveIdEpochType Success. response: " << response->ShortDebugString(); +} + +void MetaServiceImpl::SaveIdEpochType(google::protobuf::RpcController *controller, + const pb::meta::SaveIdEpochTypeRequest *request, + pb::meta::SaveIdEpochTypeResponse *response, google::protobuf::Closure *done) { + brpc::ClosureGuard done_guard(done); + + if (!coordinator_control_->IsLeader()) { + return RedirectResponse(response); + } + + DINGO_LOG(INFO) << request->ShortDebugString(); + + // Run in queue. + auto *svr_done = new CoordinatorServiceClosure(__func__, done_guard.release(), request, response); + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoSaveIdEpochType(controller, request, response, svr_done, coordinator_control_, engine_); + }); + bool ret = worker_set_->ExecuteRR(task); + if (!ret) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, "Commit execute queue failed"); + } +} + void DoCreateTenant(google::protobuf::RpcController * /*controller*/, const pb::meta::CreateTenantRequest *request, pb::meta::CreateTenantResponse *response, TrackClosure *done, std::shared_ptr coordinator_control, std::shared_ptr /*raft_engine*/) { diff --git a/src/server/meta_service.h b/src/server/meta_service.h index 6ac8a1246..f52f1d731 100644 --- a/src/server/meta_service.h +++ b/src/server/meta_service.h @@ -162,6 +162,9 @@ class MetaServiceImpl : public pb::meta::MetaService { void CreateAutoIncrement(google::protobuf::RpcController* controller, const pb::meta::CreateAutoIncrementRequest* request, pb::meta::CreateAutoIncrementResponse* response, google::protobuf::Closure* done) override; + void CreateAutoIncrements(google::protobuf::RpcController* controller, + const pb::meta::CreateAutoIncrementsRequest* request, + pb::meta::CreateAutoIncrementsResponse* response, google::protobuf::Closure* done) override; void UpdateAutoIncrement(google::protobuf::RpcController* controller, const ::dingodb::pb::meta::UpdateAutoIncrementRequest* request, pb::meta::UpdateAutoIncrementResponse* response, google::protobuf::Closure* done) override; @@ -214,6 +217,12 @@ class MetaServiceImpl : public pb::meta::MetaService { void ListWatch(google::protobuf::RpcController* controller, const pb::meta::ListWatchRequest* request, pb::meta::ListWatchResponse* response, google::protobuf::Closure* done) override; + void ExportMeta(google::protobuf::RpcController* controller, const pb::meta::ExportMetaRequest* request, + pb::meta::ExportMetaResponse* response, google::protobuf::Closure* done) override; + + void SaveIdEpochType(google::protobuf::RpcController* controller, const pb::meta::SaveIdEpochTypeRequest* request, + pb::meta::SaveIdEpochTypeResponse* response, google::protobuf::Closure* done) override; + void SetWorkSet(WorkerSetPtr worker_set) { worker_set_ = worker_set; } // table and index definition convertor diff --git a/src/server/store_service.cc b/src/server/store_service.cc index 6a1982566..39e7c0e75 100644 --- a/src/server/store_service.cc +++ b/src/server/store_service.cc @@ -3349,6 +3349,248 @@ void StoreServiceImpl::TxnDeleteRange(google::protobuf::RpcController* controlle } } +static butil::Status ValidateBackupDataRangeRequest(const dingodb::pb::store::BackupDataRequest* request, + store::RegionPtr region) { + // check if region_epoch is match + auto status = ServiceHelper::ValidateRegionEpoch(request->context().region_epoch(), region); + if (!status.ok()) { + return status; + } + + pb::common::Range req_range; + req_range.set_start_key(request->start_key()); + req_range.set_end_key(request->end_key()); + + status = ServiceHelper::ValidateRange(req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRangeInRange(region->Range(false), req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRegionState(region); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateClusterReadOnly(); + if (!status.ok()) { + return status; + } + + return butil::Status(); +} + +static void DoBackupData(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto region = done->GetRegion(); + + auto status = ValidateBackupDataRangeRequest(request, region); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + ServiceHelper::GetStoreRegionInfo(region, response->mutable_error()); + return; + } + + // check leader if need + if (request->need_leader()) { + status = storage->ValidateLeader(region); + if (!status.ok()) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + return; + } + } + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetRegionId(request->context().region_id()); + ctx->SetTracker(tracker); + ctx->SetCfName(Constant::kStoreDataCF); + ctx->SetRegionEpoch(request->context().region_epoch()); + ctx->SetIsolationLevel(request->context().isolation_level()); + ctx->SetRawEngineType(region->GetRawEngineType()); + ctx->SetStoreEngineType(region->GetStoreEngineType()); + + status = storage->BackupData(ctx, region, request->region_type(), request->backup_ts(), request->backup_tso(), + request->storage_path(), request->storage_backend(), request->compression_type(), + request->compression_level(), response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} + +void StoreServiceImpl::BackupData(google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure(__func__, done, request, response); + + if (BAIDU_UNLIKELY(svr_done->GetRegion() == nullptr)) { + brpc::ClosureGuard done_guard(svr_done); + return; + } + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoBackupData(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} + +static butil::Status ValidateBackupMetaRangeRequest(const dingodb::pb::store::BackupMetaRequest* request, + store::RegionPtr region) { + // check if region_epoch is match + auto status = ServiceHelper::ValidateRegionEpoch(request->context().region_epoch(), region); + if (!status.ok()) { + return status; + } + + pb::common::Range req_range; + req_range.set_start_key(request->start_key()); + req_range.set_end_key(request->end_key()); + + status = ServiceHelper::ValidateRange(req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRangeInRange(region->Range(false), req_range); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateRegionState(region); + if (!status.ok()) { + return status; + } + + status = ServiceHelper::ValidateClusterReadOnly(); + if (!status.ok()) { + return status; + } + + return butil::Status(); +} + +static void DoBackupMeta(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupMetaRequest* request, + dingodb::pb::store::BackupMetaResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto region = done->GetRegion(); + + auto status = ValidateBackupMetaRangeRequest(request, region); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + ServiceHelper::GetStoreRegionInfo(region, response->mutable_error()); + return; + } + + // check leader if need + if (request->need_leader()) { + status = storage->ValidateLeader(region); + if (!status.ok()) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + return; + } + } + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetRegionId(request->context().region_id()); + ctx->SetTracker(tracker); + ctx->SetCfName(Constant::kStoreDataCF); + ctx->SetRegionEpoch(request->context().region_epoch()); + ctx->SetIsolationLevel(request->context().isolation_level()); + ctx->SetRawEngineType(region->GetRawEngineType()); + ctx->SetStoreEngineType(region->GetStoreEngineType()); + + status = storage->BackupMeta(ctx, region, request->region_type(), request->backup_ts(), request->backup_tso(), + request->storage_path(), request->storage_backend(), request->compression_type(), + request->compression_level(), response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} + +void StoreServiceImpl::BackupMeta(google::protobuf::RpcController* controller, + const dingodb::pb::store::BackupMetaRequest* request, + dingodb::pb::store::BackupMetaResponse* response, google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure(__func__, done, request, response); + + if (BAIDU_UNLIKELY(svr_done->GetRegion() == nullptr)) { + brpc::ClosureGuard done_guard(svr_done); + return; + } + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoBackupMeta(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} + +static void DoControlConfig(StoragePtr storage, google::protobuf::RpcController* controller, + const dingodb::pb::store::ControlConfigRequest* request, + dingodb::pb::store::ControlConfigResponse* response, TrackClosure* done, bool is_sync) { + brpc::Controller* cntl = (brpc::Controller*)controller; + brpc::ClosureGuard done_guard(done); + auto tracker = done->Tracker(); + tracker->SetServiceQueueWaitTime(); + + auto ctx = std::make_shared(cntl, is_sync ? nullptr : done_guard.release(), request, response); + ctx->SetTracker(tracker); + + std::vector variables; + for (const auto& variable : request->control_config_variable()) { + variables.push_back(variable); + } + + auto status = storage->ControlConfig(ctx, variables, response); + if (BAIDU_UNLIKELY(!status.ok())) { + ServiceHelper::SetError(response->mutable_error(), status.error_code(), status.error_str()); + if (!is_sync) done->Run(); + } +} +void StoreServiceImpl::ControlConfig(google::protobuf::RpcController* controller, + const pb::store::ControlConfigRequest* request, + pb::store::ControlConfigResponse* response, google::protobuf::Closure* done) { + auto* svr_done = new ServiceClosure( + __func__, done, request, response); + + // Run in queue. + auto task = std::make_shared([this, controller, request, response, svr_done]() { + DoControlConfig(storage_, controller, request, response, svr_done, true); + }); + bool ret = write_worker_set_->ExecuteRR(task); + if (BAIDU_UNLIKELY(!ret)) { + brpc::ClosureGuard done_guard(svr_done); + ServiceHelper::SetError(response->mutable_error(), pb::error::EREQUEST_FULL, + "WorkerSet queue is full, please wait and retry"); + } +} + static butil::Status ValidateTxnDumpRequest(const dingodb::pb::store::TxnDumpRequest* request, store::RegionPtr region) { // check if region_epoch is match diff --git a/src/server/store_service.h b/src/server/store_service.h index f29fe0f2f..ecedabb15 100644 --- a/src/server/store_service.h +++ b/src/server/store_service.h @@ -131,6 +131,16 @@ class StoreServiceImpl : public pb::store::StoreService { void TxnDeleteRange(google::protobuf::RpcController* controller, const pb::store::TxnDeleteRangeRequest* request, pb::store::TxnDeleteRangeResponse* response, google::protobuf::Closure* done) override; + // backup & restore + void BackupData(google::protobuf::RpcController* controller, const dingodb::pb::store::BackupDataRequest* request, + dingodb::pb::store::BackupDataResponse* response, google::protobuf::Closure* done) override; + + void BackupMeta(google::protobuf::RpcController* controller, const dingodb::pb::store::BackupMetaRequest* request, + dingodb::pb::store::BackupMetaResponse* response, google::protobuf::Closure* done) override; + + void ControlConfig(google::protobuf::RpcController* controller, const pb::store::ControlConfigRequest* request, + pb::store::ControlConfigResponse* response, google::protobuf::Closure* done) override; + void SetStorage(StoragePtr storage) { storage_ = storage; } void SetReadWorkSet(WorkerSetPtr worker_set) { read_worker_set_ = worker_set; } diff --git a/test/unit_test/CMakeLists.txt b/test/unit_test/CMakeLists.txt index a87935ae7..32ee980b1 100644 --- a/test/unit_test/CMakeLists.txt +++ b/test/unit_test/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(vector) add_subdirectory(misc) add_subdirectory(document) add_subdirectory(txn) +add_subdirectory(br) set(UNIT_TEST_BIN "dingodb_unit_test") @@ -27,7 +28,8 @@ set(UNIT_TEST_LIBS $ $ $ - $) + $ + $) set(UNIT_TEST_LIBS ${UNIT_TEST_LIBS} ${GTEST_LIBRARIES} ${GMOCK_LIBRARIES} "-Xlinker \"-(\"" ${BLAS_LIBRARIES} "-Xlinker \"-)\"") diff --git a/test/unit_test/br/CMakeLists.txt b/test/unit_test/br/CMakeLists.txt new file mode 100644 index 000000000..d0d00fcb0 --- /dev/null +++ b/test/unit_test/br/CMakeLists.txt @@ -0,0 +1,7 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/) + +file(GLOB UNIT_TEST_BR_SRCS "./*.cc") + +add_library(UNIT_TEST_BR_OBJS OBJECT ${UNIT_TEST_BR_SRCS}) + +add_dependencies(UNIT_TEST_BR_OBJS ${DEPEND_LIBS}) diff --git a/test/unit_test/br/test_backup.cc b/test/unit_test/br/test_backup.cc new file mode 100644 index 000000000..c9726ea87 --- /dev/null +++ b/test/unit_test/br/test_backup.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/parameter.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-31 14:39:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + br::BackupParams backup_params; + backup_params.coor_url = "127.0.0.1:32001, 127.0.0.1:32002, 127.0.0.1:32003"; + backup_params.br_type = "backup"; + backup_params.br_backup_type = "full"; + backup_params.backupts = backupts; + backup_params.backuptso_internal = backuptso_internal; + backup_params.storage = storage; + backup_params.storage_internal = storage_internal; + + backup = std::make_shared(backup_params); + + status = backup->Init(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupTest, Run) { + butil::Status status = backup->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupTest, Finish) { + butil::Status status = backup->Finish(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} diff --git a/test/unit_test/br/test_backup_data.cc b/test/unit_test/br/test_backup_data.cc new file mode 100644 index 000000000..4dd53291d --- /dev/null +++ b/test/unit_test/br/test_backup_data.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_data.h" +#include "br/backup_sql_meta.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupDataTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-31 14:39:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_data; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupDataTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_data = + std::make_shared(coor_interaction, store_interaction, index_interaction, document_interaction, + backupts, backuptso_internal, storage, storage_internal); + + auto backup_sql_meta = std::make_shared(coor_interaction, store_interaction, backupts, + backuptso_internal, storage, storage_internal); + status = backup_sql_meta->GetSqlMetaRegionFromCoordinator(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + std::vector region_meta_list; + backup_sql_meta->GetSqlMetaRegionList(region_meta_list); + + status = backup_data->Init(region_meta_list); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupDataTest, Run) { + butil::Status status = backup_data->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupDataTest, Finish) { + butil::Status status = backup_data->Finish(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupDataTest, GetRegionMap) { + std::shared_ptr region_map; + region_map = backup_data->GetRegionMap(); + EXPECT_NE(region_map.get(), nullptr); + + for (const auto& region : region_map->regions()) { + DINGO_LOG(INFO) << fmt::format("region: {}", region.DebugString()); + } +} + +TEST_F(BrBackupDataTest, GetBackupMeta) { + std::shared_ptr backup_meta; + backup_meta = backup_data->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); + DINGO_LOG(INFO) << fmt::format("backup_meta: {}", backup_meta->DebugString()); +} diff --git a/test/unit_test/br/test_backup_meta.cc b/test/unit_test/br/test_backup_meta.cc new file mode 100644 index 000000000..3ea90111e --- /dev/null +++ b/test/unit_test/br/test_backup_meta.cc @@ -0,0 +1,155 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_meta.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupMetaTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-31 14:39:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_meta; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupMetaTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_meta = + std::make_shared(coor_interaction, store_interaction, index_interaction, document_interaction, + backupts, backuptso_internal, storage, storage_internal); + + status = backup_meta->Init(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupMetaTest, SetRegionMap) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneCoordinatorInteraction(); + + dingodb::pb::coordinator::GetRegionMapRequest request; + dingodb::pb::coordinator::GetRegionMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_tenant_id(-1); // get all tenants region map + + status = interaction->SendRequest("CoordinatorService", "GetRegionMap", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + } + + region_maps = std::make_shared(response.regionmap()); + + DINGO_LOG(INFO) << fmt::format("region_maps size : {}", region_maps->regions_size()); + + for (const auto& region : region_maps->regions()) { + DINGO_LOG(INFO) << fmt::format("region_id: {}, region: {}", region.id(), region.DebugString()); + } +} + +TEST_F(BrBackupMetaTest, Run) { + butil::Status status = backup_meta->Run(region_maps); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupMetaTest, Finish) { + butil::Status status = backup_meta->Finish(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupMetaTest, GetSqlMetaRegionList) { + std::vector sql_meta_region_list; + sql_meta_region_list = backup_meta->GetSqlMetaRegionList(); + + for (const auto& region_id : sql_meta_region_list) { + DINGO_LOG(INFO) << fmt::format("region: {}", region_id); + } +} + +TEST_F(BrBackupMetaTest, GetBackupMeta) { + std::shared_ptr backup_meta_item; + backup_meta_item = backup_meta->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); + DINGO_LOG(INFO) << fmt::format("backup_meta: {}", backup_meta_item->DebugString()); +} + +TEST_F(BrBackupMetaTest, GetIdEpochTypeAndValue) { + const auto& [status, id_epoch_type_and_value] = backup_meta->GetIdEpochTypeAndValue(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + DINGO_LOG(INFO) << fmt::format("id_epoch_type_and_value: {}", id_epoch_type_and_value->DebugString()); +} + +TEST_F(BrBackupMetaTest, GetAllTableIncrement) { + const auto& [status, table_increment_group] = backup_meta->GetAllTableIncrement(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + DINGO_LOG(INFO) << fmt::format("id_epoch_type_and_value: {}", table_increment_group->DebugString()); +} \ No newline at end of file diff --git a/test/unit_test/br/test_backup_misc.cc b/test/unit_test/br/test_backup_misc.cc new file mode 100644 index 000000000..80897923f --- /dev/null +++ b/test/unit_test/br/test_backup_misc.cc @@ -0,0 +1,488 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include + +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "common/helper.h" +#include "common/logging.h" + +class BrBackupMiscTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-31 14:39:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static bool region_auto_split_enable_after_finish = false; + inline static bool region_auto_merge_enable_after_finish = false; + + inline static bool balance_leader_enable_after_finish = false; + inline static bool balance_region_enable_after_finish = false; + + inline static bool is_gc_stop = false; + inline static bool is_gc_enable_after_finish = false; +}; + +TEST_F(BrBackupMiscTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); +} + +static butil::Status RegisterBackupToCoordinator(bool is_first, const std::string &task_id, + const std::string &storage_internal, + br::ServerInteractionPtr coordinator_interaction) { + dingodb::pb::coordinator::RegisterBackupRequest request; + dingodb::pb::coordinator::RegisterBackupResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + request.set_backup_name(task_id); + request.set_backup_path(storage_internal); + int64_t current_now_s = dingodb::Helper::Timestamp(); + if (is_first) { + request.set_backup_start_timestamp(current_now_s); + } + request.set_backup_current_timestamp(current_now_s); + request.set_backup_timeout_s(3000); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "RegisterBackup", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set RegisterBackup, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set RegisterBackup, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(response.error().errcode(), s); + } + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, RegisterBackup) { + butil::Status status; + + status = RegisterBackupToCoordinator(true, "task_id", storage_internal, + br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + // status = RegisterBackupToCoordinator(true, "task_id2", storage_internal, + // br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + + // EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status UnregisterBackupToCoordinator(const std::string &task_id, + br::ServerInteractionPtr coordinator_interaction) { + dingodb::pb::coordinator::UnRegisterBackupRequest request; + dingodb::pb::coordinator::UnRegisterBackupResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + request.set_backup_name(task_id); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "UnRegisterBackup", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set UnRegisterBackup, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set UnRegisterBackup, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(response.error().errcode(), s); + } + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, UnregisterBackup) { + butil::Status status; + status = UnregisterBackupToCoordinator("task_id2", br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + EXPECT_EQ(status.error_code(), dingodb::pb::error::EBACKUP_TASK_NAME_NOT_MATCH); + + status = UnregisterBackupToCoordinator("task_id", br::InteractionManager::GetInstance().GetCoordinatorInteraction()); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status DisableSplitAndMergeToStoreAndIndex(br::ServerInteractionPtr store_interaction, + br::ServerInteractionPtr index_interaction, + bool ®ion_auto_split_enable_after_finish, + bool ®ion_auto_merge_enable_after_finish) { + dingodb::pb::store::ControlConfigRequest request; + dingodb::pb::store::ControlConfigResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + dingodb::pb::common::ControlConfigVariable config_auto_split; + config_auto_split.set_name("FLAGS_region_enable_auto_split"); + config_auto_split.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_split)); + + dingodb::pb::common::ControlConfigVariable config_auto_merge; + config_auto_merge.set_name("FLAGS_region_enable_auto_merge"); + config_auto_merge.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_merge)); + + butil::Status status = store_interaction->AllSendRequest("StoreService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + for (const auto &config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_split") { + region_auto_split_enable_after_finish = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_merge") { + region_auto_merge_enable_after_finish = true; + } + } + + DINGO_LOG(INFO) << "DisableSplitAndMergeToStoreAndIndex success." << response.DebugString(); + + status = index_interaction->AllSendRequest("IndexService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + for (const auto &config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_split") { + region_auto_split_enable_after_finish = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_region_enable_auto_merge") { + region_auto_merge_enable_after_finish = true; + } + } + + DINGO_LOG(INFO) << "DisableSplitAndMergeToStoreAndIndex success." << response.DebugString(); + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, DisableSplitAndMerge) { + butil::Status status; + + status = + DisableSplitAndMergeToStoreAndIndex(br::InteractionManager::GetInstance().GetStoreInteraction(), + br::InteractionManager::GetInstance().GetIndexInteraction(), + region_auto_split_enable_after_finish, region_auto_merge_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status EnableSplitAndMergeToStoreAndIndex(br::ServerInteractionPtr store_interaction, + br::ServerInteractionPtr index_interaction, + bool region_auto_split_enable_after_finish, + bool region_auto_merge_enable_after_finish) { + dingodb::pb::store::ControlConfigRequest request; + dingodb::pb::store::ControlConfigResponse response; + + if (region_auto_split_enable_after_finish) { + dingodb::pb::common::ControlConfigVariable config_auto_split; + config_auto_split.set_name("FLAGS_region_enable_auto_split"); + config_auto_split.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_auto_split)); + } + + if (region_auto_merge_enable_after_finish) { + dingodb::pb::common::ControlConfigVariable config_auto_merge; + config_auto_merge.set_name("FLAGS_region_enable_auto_merge"); + config_auto_merge.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_auto_merge)); + } + + if (!request.control_config_variable().empty()) { + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + butil::Status status = store_interaction->AllSendRequest("StoreService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + DINGO_LOG(INFO) << "EnableSplitAndMergeToStoreAndIndex success." << response.DebugString(); + + status = index_interaction->AllSendRequest("IndexService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + DINGO_LOG(INFO) << "EnableSplitAndMergeToStoreAndIndex success." << response.DebugString(); + } + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, EnableSplitAndMerge) { + butil::Status status; + status = + EnableSplitAndMergeToStoreAndIndex(br::InteractionManager::GetInstance().GetStoreInteraction(), + br::InteractionManager::GetInstance().GetIndexInteraction(), + region_auto_split_enable_after_finish, region_auto_merge_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status DisableBalanceToCoordinator(br::ServerInteractionPtr coordinator_interaction, + bool &balance_leader_enable_after_finish, + bool &balance_region_enable_after_finish) { + dingodb::pb::coordinator::ControlConfigRequest request; + dingodb::pb::coordinator::ControlConfigResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + dingodb::pb::common::ControlConfigVariable config_balance_leader; + config_balance_leader.set_name("FLAGS_enable_balance_leader"); + config_balance_leader.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_balance_leader)); + + dingodb::pb::common::ControlConfigVariable config_balance_region; + config_balance_region.set_name("FLAGS_enable_balance_region"); + config_balance_region.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_balance_region)); + + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + + for (const auto &config : response.control_config_variable()) { + if (config.is_error_occurred()) { + DINGO_LOG(ERROR) << "ControlConfig not support variable: " << config.name() << " skip."; + return butil::Status(dingodb::pb::error::EINTERNAL, "ControlConfig not support variable: %s skip.", + config.name().c_str()); + } + + if (!config.is_already_set() && config.name() == "FLAGS_enable_balance_leader") { + balance_leader_enable_after_finish = true; + } + + if (!config.is_already_set() && config.name() == "FLAGS_enable_balance_region") { + balance_region_enable_after_finish = true; + } + } + + DINGO_LOG(INFO) << "DisableBalanceToCoordinator success." << response.DebugString(); + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, DisableBalance) { + butil::Status status; + status = DisableBalanceToCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction(), + balance_leader_enable_after_finish, balance_region_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status EnableBalanceToCoordinator(br::ServerInteractionPtr coordinator_interaction, + bool balance_leader_enable_after_finish, + bool balance_region_enable_after_finish) { + dingodb::pb::coordinator::ControlConfigRequest request; + dingodb::pb::coordinator::ControlConfigResponse response; + + if (balance_leader_enable_after_finish) { + dingodb::pb::common::ControlConfigVariable config_balance_leader; + config_balance_leader.set_name("FLAGS_enable_balance_leader"); + config_balance_leader.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_balance_leader)); + } + + if (balance_region_enable_after_finish) { + dingodb::pb::common::ControlConfigVariable config_balance_region; + config_balance_region.set_name("FLAGS_enable_balance_region"); + config_balance_region.set_value("true"); + request.mutable_control_config_variable()->Add(std::move(config_balance_region)); + } + + if (!request.control_config_variable().empty()) { + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + butil::Status status = + coordinator_interaction->AllSendRequest("CoordinatorService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return status; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return butil::Status(status.error_code(), "%s", status.error_cstr()); + } + } + + DINGO_LOG(INFO) << "EnableBalanceToCoordinator success." << response.DebugString(); + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, EnableBalance) { + butil::Status status; + status = EnableBalanceToCoordinator(br::InteractionManager::GetInstance().GetCoordinatorInteraction(), + balance_leader_enable_after_finish, balance_region_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status SetGcStop(bool &is_gc_stop, bool &is_gc_enable_after_finish) { + if (is_gc_stop) { + return butil::Status::OK(); + } + + DINGO_LOG(INFO) << "Set GC stop ..."; + + dingodb::pb::coordinator::UpdateGCSafePointRequest request; + dingodb::pb::coordinator::UpdateGCSafePointResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_gc_flag( + ::dingodb::pb::coordinator::UpdateGCSafePointRequest_GcFlagType::UpdateGCSafePointRequest_GcFlagType_GC_STOP); + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "UpdateGCSafePoint", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set GC stop, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set GC stop, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + is_gc_stop = true; + is_gc_enable_after_finish = true; + + DINGO_LOG(INFO) << "GC is stopped. Backup will enable GC. if backup is finished."; + + return butil::Status::OK( + + ); +} + +TEST_F(BrBackupMiscTest, GcStop) { + butil::Status status; + status = SetGcStop(is_gc_stop, is_gc_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +static butil::Status SetGcStart(bool &is_gc_stop, bool &is_gc_enable_after_finish) { + if (!is_gc_enable_after_finish) { + return butil::Status::OK(); + } + DINGO_LOG(INFO) << "Set GC start ..."; + + dingodb::pb::coordinator::UpdateGCSafePointRequest request; + dingodb::pb::coordinator::UpdateGCSafePointResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_gc_flag( + ::dingodb::pb::coordinator::UpdateGCSafePointRequest_GcFlagType::UpdateGCSafePointRequest_GcFlagType_GC_START); + butil::Status status = br::InteractionManager::GetInstance().GetCoordinatorInteraction()->SendRequest( + "CoordinatorService", "UpdateGCSafePoint", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set GC stop, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + return status; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set GC stop, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + return butil::Status(dingodb::pb::error::EINTERNAL, s); + } + + is_gc_stop = false; + is_gc_enable_after_finish = false; + + DINGO_LOG(INFO) << "Set GC start success."; + + return butil::Status::OK(); +} + +TEST_F(BrBackupMiscTest, GcStart) { + butil::Status status; + status = SetGcStart(is_gc_stop, is_gc_enable_after_finish); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} \ No newline at end of file diff --git a/test/unit_test/br/test_backup_sdk_data.cc b/test/unit_test/br/test_backup_sdk_data.cc new file mode 100644 index 000000000..df41e69b1 --- /dev/null +++ b/test/unit_test/br/test_backup_sdk_data.cc @@ -0,0 +1,150 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_sdk_data.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupSdkDataTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-30 09:35:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_sdk_data; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupSdkDataTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_sdk_data = + std::make_shared(coor_interaction, store_interaction, index_interaction, document_interaction, + backupts, backuptso_internal, storage, storage_internal); +} + +TEST_F(BrBackupSdkDataTest, SetRegionMap) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneCoordinatorInteraction(); + + dingodb::pb::coordinator::GetRegionMapRequest request; + dingodb::pb::coordinator::GetRegionMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_tenant_id(-1); // get all tenants region map + + status = interaction->SendRequest("CoordinatorService", "GetRegionMap", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + } + + region_maps = std::make_shared(response.regionmap()); + + DINGO_LOG(INFO) << fmt::format("region_maps size : {}", region_maps->regions_size()); + + for (const auto& region : region_maps->regions()) { + DINGO_LOG(INFO) << fmt::format("region_id: {}, region: {}", region.id(), region.DebugString()); + } + + auto iter = region_maps->mutable_regions()->begin(); + while (iter != region_maps->mutable_regions()->end()) { + if (iter->id() != 80046 && iter->id() != 80045 && iter->id() != 80047) { + iter = region_maps->mutable_regions()->erase(iter); + } else { + ++iter; + } + } + + backup_sdk_data->SetRegionMap(region_maps); +} + +TEST_F(BrBackupSdkDataTest, Filter) { + butil::Status status = backup_sdk_data->Filter(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkDataTest, Run) { + butil::Status status = backup_sdk_data->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkDataTest, Backup) { + butil::Status status = backup_sdk_data->Backup(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkDataTest, GetBackupMeta) { + std::shared_ptr> backup_meta; + backup_meta = backup_sdk_data->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); + + for (const auto& meta : *backup_meta) { + DINGO_LOG(INFO) << fmt::format("BackupMeta: {}", meta.DebugString()); + } +} \ No newline at end of file diff --git a/test/unit_test/br/test_backup_sdk_meta.cc b/test/unit_test/br/test_backup_sdk_meta.cc new file mode 100644 index 000000000..94b9c13f3 --- /dev/null +++ b/test/unit_test/br/test_backup_sdk_meta.cc @@ -0,0 +1,106 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_sdk_meta.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "common/logging.h" +#include "fmt/core.h" + +class BrBackupSdkMetaTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-30 13:58:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_sdk_meta; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupSdkMetaTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_sdk_meta = std::make_shared(coor_interaction, storage_internal); + + status = backup_sdk_meta->GetSdkMetaFromCoordinator(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkMetaTest, Run) { + butil::Status status = backup_sdk_meta->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkMetaTest, Backup) { + butil::Status status = backup_sdk_meta->Backup(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSdkMetaTest, GetBackupMeta) { + std::shared_ptr backup_meta; + backup_meta = backup_sdk_meta->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); + + DINGO_LOG(INFO) << backup_meta->DebugString(); +} diff --git a/test/unit_test/br/test_backup_sql_data.cc b/test/unit_test/br/test_backup_sql_data.cc new file mode 100644 index 000000000..4979dbb51 --- /dev/null +++ b/test/unit_test/br/test_backup_sql_data.cc @@ -0,0 +1,154 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_sql_data.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupSqlDataTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-27 20:56:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_sql_data; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupSqlDataTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_sql_data = + std::make_shared(coor_interaction, store_interaction, index_interaction, document_interaction, + backupts, backuptso_internal, storage, storage_internal); +} + +TEST_F(BrBackupSqlDataTest, SetRegionMap) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneCoordinatorInteraction(); + + dingodb::pb::coordinator::GetRegionMapRequest request; + dingodb::pb::coordinator::GetRegionMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_tenant_id(-1); // get all tenants region map + + status = interaction->SendRequest("CoordinatorService", "GetRegionMap", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + } + + region_maps = std::make_shared(response.regionmap()); + + DINGO_LOG(INFO) << fmt::format("region_maps size : {}", region_maps->regions_size()); + + for (const auto& region : region_maps->regions()) { + DINGO_LOG(INFO) << fmt::format("region_id: {}, region: {}", region.id(), region.DebugString()); + } + + backup_sql_data->SetRegionMap(region_maps); +} + +TEST_F(BrBackupSqlDataTest, RemoveSqlMeta) { + std::vector meta_region_list; + for (const auto& region : region_maps->regions()) { + if (region.id() >= 80039 && region.id() <= 80044) { + continue; + } + meta_region_list.push_back(region.id()); + } + + butil::Status status = backup_sql_data->RemoveSqlMeta(meta_region_list); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlDataTest, Filter) { + butil::Status status = backup_sql_data->Filter(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlDataTest, Run) { + butil::Status status = backup_sql_data->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlDataTest, Backup) { + butil::Status status = backup_sql_data->Backup(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlDataTest, GetBackupMeta) { + std::shared_ptr> backup_meta; + backup_meta = backup_sql_data->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); + + for (const auto& meta : *backup_meta) { + DINGO_LOG(INFO) << fmt::format("BackupMeta: {}", meta.DebugString()); + } +} \ No newline at end of file diff --git a/test/unit_test/br/test_backup_sql_meta.cc b/test/unit_test/br/test_backup_sql_meta.cc new file mode 100644 index 000000000..1ec4bb73a --- /dev/null +++ b/test/unit_test/br/test_backup_sql_meta.cc @@ -0,0 +1,147 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/backup_sql_meta.h" +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrBackupSqlMetaTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string backupts = "2024-12-30 13:58:00 +08:00"; + inline static int64_t backuptso_internal = 0; + inline static std::string storage = "local://./backup"; + inline static std::string storage_internal = "./backup"; + + inline static std::shared_ptr backup_sql_meta; + + inline static std::shared_ptr region_maps; +}; + +TEST_F(BrBackupSqlMetaTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + br::ServerInteractionPtr coor_interaction = std::make_shared(); + br::ServerInteractionPtr store_interaction = std::make_shared(); + br::ServerInteractionPtr index_interaction = std::make_shared(); + br::ServerInteractionPtr document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(coor_interaction); + br::InteractionManager::GetInstance().SetStoreInteraction(store_interaction); + br::InteractionManager::GetInstance().SetIndexInteraction(index_interaction); + br::InteractionManager::GetInstance().SetDocumentInteraction(document_interaction); + + std::filesystem::path temp_storage_internal = std::filesystem::absolute(storage_internal); + if (temp_storage_internal.has_relative_path()) { + auto filename = temp_storage_internal.filename(); + temp_storage_internal = std::filesystem::absolute(filename); + } + + // auto temp_storage_internal = std::filesystem::absolute(storage_internal); + storage_internal = temp_storage_internal.string(); + br::Utils::RemoveAllDir(storage_internal, true); + br::Utils::CreateDir(storage_internal); + + butil::Status status = br::Utils::ConvertBackupTsToTso(backupts, backuptso_internal); + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_sql_meta = std::make_shared(coor_interaction, store_interaction, backupts, + backuptso_internal, storage, storage_internal); +} + +TEST_F(BrBackupSqlMetaTest, SetRegionMap) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneCoordinatorInteraction(); + + dingodb::pb::coordinator::GetRegionMapRequest request; + dingodb::pb::coordinator::GetRegionMapResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_tenant_id(-1); // get all tenants region map + + status = interaction->SendRequest("CoordinatorService", "GetRegionMap", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << response.error().errmsg(); + } + + region_maps = std::make_shared(response.regionmap()); + + DINGO_LOG(INFO) << fmt::format("region_maps size : {}", region_maps->regions_size()); + + for (const auto& region : region_maps->regions()) { + DINGO_LOG(INFO) << fmt::format("region_id: {}, region: {}", region.id(), region.DebugString()); + } + + backup_sql_meta->SetRegionMap(region_maps); +} + +TEST_F(BrBackupSqlMetaTest, ReserveSqlMeta) { + std::vector meta_region_list; + butil::Status status = backup_sql_meta->GetSqlMetaRegionFromCoordinator(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + + backup_sql_meta->GetSqlMetaRegionList(meta_region_list); + + status = backup_sql_meta->ReserveSqlMeta(meta_region_list); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlMetaTest, Filter) { + butil::Status status = backup_sql_meta->Filter(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlMetaTest, Run) { + butil::Status status = backup_sql_meta->Run(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlMetaTest, Backup) { + butil::Status status = backup_sql_meta->Backup(); + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); +} + +TEST_F(BrBackupSqlMetaTest, GetBackupMeta) { + std::shared_ptr> backup_meta; + backup_meta = backup_sql_meta->GetBackupMeta(); + EXPECT_NE(backup_meta.get(), nullptr); +} diff --git a/test/unit_test/br/test_helper.cc b/test/unit_test/br/test_helper.cc new file mode 100644 index 000000000..1d22d0d61 --- /dev/null +++ b/test/unit_test/br/test_helper.cc @@ -0,0 +1,104 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/helper.h" +#include "br/utils.h" +#include "fmt/core.h" + +class BrHelperTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(BrHelperTest, Ltrim) { + std::string s = " abc"; + std::string delete_str = " "; + std::string result = br::Helper::Ltrim(s, delete_str); + EXPECT_EQ(result, "abc"); +} + +TEST_F(BrHelperTest, Rtrim) { + std::string s = "abc "; + std::string delete_str = " "; + std::string result = br::Helper::Rtrim(s, delete_str); + EXPECT_EQ(result, "abc"); +} + +TEST_F(BrHelperTest, Trim) { + std::string s = " abc "; + std::string delete_str = " "; + std::string result = br::Helper::Trim(s, delete_str); + EXPECT_EQ(result, "abc"); +} + +TEST_F(BrHelperTest, GetRandInt) { + int result = br::Helper::GetRandInt(); + result = br::Helper::GetRandInt(); + result = br::Helper::GetRandInt(); +} + +TEST_F(BrHelperTest, StringToEndpoints) { + std::string str = "127.0.0.1:1025, 127.0.0.1:1026, 127.0.0.1:1027"; + std::vector end_points = br::Helper::StringToEndpoints(str); + EXPECT_EQ(end_points.size(), 3); + + str = "127.0.0.1:1025,127.0.0.1:1026,127.0.0.1:1027"; + end_points = br::Helper::StringToEndpoints(str); + EXPECT_EQ(end_points.size(), 3); +} + +TEST_F(BrHelperTest, VectorToEndpoints) { + std::vector addrs; + addrs.push_back("127.0.0.1:1025"); + addrs.push_back("127.0.0.1:1026"); + addrs.push_back("127.0.0.1:1027"); + + std::vector end_points = br::Helper::VectorToEndpoints(addrs); + EXPECT_EQ(end_points.size(), 3); + + addrs.clear(); + addrs.push_back(" 127.0.0.1:1025 "); + addrs.push_back(" 127.0.0.1:1026 "); + addrs.push_back(" 127.0.0.1:1027 "); + + end_points = br::Helper::VectorToEndpoints(addrs); + EXPECT_EQ(end_points.size(), 3); +} + +TEST_F(BrHelperTest, GetAddrsFromFile) { + std::string path = "./coor_list"; + std::ofstream writer; + auto status = br::Utils::CreateFile(writer, path); + writer << "# dingo-store coordinators" << std::endl; + writer << "172.30.14.11:32001" << std::endl; + writer << "172.30.14.11:32002" << std::endl; + writer << "172.30.14.11:32003" << std::endl; + + std::vector addrs = br::Helper::GetAddrsFromFile(path); + EXPECT_EQ(addrs.size(), 3); + + std::filesystem::remove(path); +} diff --git a/test/unit_test/br/test_interaction_manager.cc b/test/unit_test/br/test_interaction_manager.cc new file mode 100644 index 000000000..30eb1b381 --- /dev/null +++ b/test/unit_test/br/test_interaction_manager.cc @@ -0,0 +1,289 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include "br/helper.h" +#include "br/interaction_manager.h" +#include "br/interation.h" +#include "common/logging.h" +#include "fmt/core.h" +#include "proto/coordinator.pb.h" + +class BrInterationManagerTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(BrInterationManagerTest, SetCoordinatorInteraction) { + std::string coor_addrs = "127.0.0.1:32001, 127.0.0.1:32002, 127.0.0.1:32003"; + + std::shared_ptr interaction = std::make_shared(); + bool is_true = interaction->Init(coor_addrs); + EXPECT_TRUE(is_true); + + br::InteractionManager::GetInstance().SetCoordinatorInteraction(interaction); +} + +TEST_F(BrInterationManagerTest, SetStoreInteraction) { + std::string store_addrs = "127.0.0.1:30001, 127.0.0.1:30002, 127.0.0.1:30003"; + + std::shared_ptr interaction = std::make_shared(); + bool is_true = interaction->Init(store_addrs); + EXPECT_TRUE(is_true); + + br::InteractionManager::GetInstance().SetStoreInteraction(interaction); +} + +TEST_F(BrInterationManagerTest, SetIndexInteraction) { + std::string index_addrs = "127.0.0.1:31001, 127.0.0.1:31002, 127.0.0.1:31003"; + + std::shared_ptr interaction = std::make_shared(); + bool is_true = interaction->Init(index_addrs); + EXPECT_TRUE(is_true); + + br::InteractionManager::GetInstance().SetIndexInteraction(interaction); +} + +TEST_F(BrInterationManagerTest, SetDocumentInteraction) { + std::string document_addrs = "127.0.0.1:33001, 127.0.0.1:33002, 127.0.0.1:33003"; + + std::shared_ptr interaction = std::make_shared(); + bool is_true = interaction->Init(document_addrs); + EXPECT_TRUE(is_true); + + br::InteractionManager::GetInstance().SetDocumentInteraction(interaction); +} + +TEST_F(BrInterationManagerTest, GetCoordinatorInteraction) { + std::shared_ptr interaction = + br::InteractionManager::GetInstance().GetCoordinatorInteraction(); + EXPECT_NE(interaction, nullptr); + // coordinator + { + dingodb::pb::coordinator::HelloRequest request; + dingodb::pb::coordinator::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("CoordinatorService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, GetStoreInteraction) { + std::shared_ptr interaction = br::InteractionManager::GetInstance().GetStoreInteraction(); + EXPECT_NE(interaction, nullptr); + // store + { + dingodb::pb::store::HelloRequest request; + dingodb::pb::store::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("StoreService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, GetIndexInteraction) { + std::shared_ptr interaction = br::InteractionManager::GetInstance().GetIndexInteraction(); + EXPECT_NE(interaction, nullptr); + // index + { + dingodb::pb::index::HelloRequest request; + dingodb::pb::index::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("IndexService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, GetDocumentInteraction) { + std::shared_ptr interaction = br::InteractionManager::GetInstance().GetDocumentInteraction(); + EXPECT_NE(interaction, nullptr); + // document + { + dingodb::pb::document::HelloRequest request; + dingodb::pb::document::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("DocumentService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, CloneCoordinatorInteraction) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneCoordinatorInteraction(); + EXPECT_TRUE(status.ok()); + EXPECT_NE(interaction, nullptr); + // coordinator + { + dingodb::pb::coordinator::HelloRequest request; + dingodb::pb::coordinator::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("CoordinatorService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, CloneStoreInteraction) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneStoreInteraction(); + EXPECT_TRUE(status.ok()); + EXPECT_NE(interaction, nullptr); + // store + { + dingodb::pb::store::HelloRequest request; + dingodb::pb::store::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("StoreService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, CloneIndexInteraction) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneIndexInteraction(); + EXPECT_TRUE(status.ok()); + EXPECT_NE(interaction, nullptr); + + // index + { + dingodb::pb::index::HelloRequest request; + dingodb::pb::index::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("IndexService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationManagerTest, CloneDocumentInteraction) { + auto [status, interaction] = br::InteractionManager::GetInstance().CloneDocumentInteraction(); + EXPECT_TRUE(status.ok()); + EXPECT_NE(interaction, nullptr); + + // document + { + dingodb::pb::document::HelloRequest request; + dingodb::pb::document::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->SendRequest("DocumentService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} \ No newline at end of file diff --git a/test/unit_test/br/test_interation.cc b/test/unit_test/br/test_interation.cc new file mode 100644 index 000000000..188e0d81e --- /dev/null +++ b/test/unit_test/br/test_interation.cc @@ -0,0 +1,287 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/helper.h" +#include "br/interation.h" +#include "br/utils.h" +#include "common/helper.h" +#include "common/logging.h" +#include "fmt/core.h" +#include "proto/coordinator.pb.h" + +class BrInterationTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static br::ServerInteractionPtr coor_interaction; + inline static br::ServerInteractionPtr store_interaction; + inline static br::ServerInteractionPtr index_interaction; + inline static br::ServerInteractionPtr document_interaction; +}; + +TEST_F(BrInterationTest, InitSingle) { + std::string coor_addrs = "127.0.0.1:32001, 127.0.0.1:32002, 127.0.0.1:32003"; + std::string store_addrs = "127.0.0.1:30001, 127.0.0.1:30002, 127.0.0.1:30003"; + std::string index_addrs = "127.0.0.1:31001, 127.0.0.1:31002, 127.0.0.1:31003"; + std::string document_addrs = "127.0.0.1:33001, 127.0.0.1:33002, 127.0.0.1:33003"; + + coor_interaction = std::make_shared(); + store_interaction = std::make_shared(); + index_interaction = std::make_shared(); + document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); + + coor_interaction.reset(); + store_interaction.reset(); + index_interaction.reset(); + document_interaction.reset(); +} + +TEST_F(BrInterationTest, Init) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + coor_interaction = std::make_shared(); + store_interaction = std::make_shared(); + index_interaction = std::make_shared(); + document_interaction = std::make_shared(); + + EXPECT_TRUE(coor_interaction->Init(coor_addrs)); + EXPECT_TRUE(store_interaction->Init(store_addrs)); + EXPECT_TRUE(index_interaction->Init(index_addrs)); + EXPECT_TRUE(document_interaction->Init(document_addrs)); +} + +TEST_F(BrInterationTest, GetAddrs) { + std::vector coor_addrs = {"127.0.0.1:32001", "127.0.0.1:32002", "127.0.0.1:32003"}; + std::vector store_addrs = {"127.0.0.1:30001", "127.0.0.1:30002", "127.0.0.1:30003"}; + std::vector index_addrs = {"127.0.0.1:31001", "127.0.0.1:31002", "127.0.0.1:31003"}; + std::vector document_addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + + auto addrs = coor_interaction->GetAddrs(); + for (int i = 0; i < addrs.size(); ++i) { + EXPECT_EQ(addrs[i], coor_addrs[i]); + } + + addrs = store_interaction->GetAddrs(); + EXPECT_EQ(store_addrs, addrs); + for (int i = 0; i < addrs.size(); ++i) { + EXPECT_EQ(addrs[i], store_addrs[i]); + } + + addrs = index_interaction->GetAddrs(); + EXPECT_EQ(index_addrs, addrs); + for (int i = 0; i < addrs.size(); ++i) { + EXPECT_EQ(addrs[i], index_addrs[i]); + } + + addrs = document_interaction->GetAddrs(); + EXPECT_EQ(document_addrs, addrs); + for (int i = 0; i < addrs.size(); ++i) { + EXPECT_EQ(addrs[i], document_addrs[i]); + } +} + +TEST_F(BrInterationTest, SendRequest) { + // coordinator + { + dingodb::pb::coordinator::HelloRequest request; + dingodb::pb::coordinator::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = coor_interaction->SendRequest("CoordinatorService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } + + // store + { + dingodb::pb::store::HelloRequest request; + dingodb::pb::store::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = store_interaction->SendRequest("StoreService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } + + // document + { + dingodb::pb::document::HelloRequest request; + dingodb::pb::document::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = document_interaction->SendRequest("DocumentService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationTest, AllSendRequest) { + // coordinator + { + dingodb::pb::coordinator::RegisterBackupRequest request; + dingodb::pb::coordinator::RegisterBackupResponse response; + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + request.set_backup_name("RegisterBackupTaskBackName"); + request.set_backup_path("/home/dingodb/backup"); + int64_t current_now_s = dingodb::Helper::Timestamp(); + request.set_backup_current_timestamp(current_now_s); + request.set_backup_timeout_s(60); + + butil::Status status = coor_interaction->AllSendRequest("CoordinatorService", "RegisterBackup", request, response); + if (!status.ok()) { + std::string s = fmt::format("Fail to set RegisterBackup, status={}", status.error_cstr()); + DINGO_LOG(ERROR) << s; + } + + if (response.error().errcode() != dingodb::pb::error::OK) { + std::string s = fmt::format("Fail to set RegisterBackup, error={}", response.error().errmsg()); + DINGO_LOG(ERROR) << s; + } + + EXPECT_EQ(status.error_code(), dingodb::pb::error::OK); + } + + // store + { + dingodb::pb::store::ControlConfigRequest request; + dingodb::pb::store::ControlConfigResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + + dingodb::pb::common::ControlConfigVariable config_auto_split; + config_auto_split.set_name("FLAGS_region_enable_auto_split"); + config_auto_split.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_split)); + + dingodb::pb::common::ControlConfigVariable config_auto_merge; + config_auto_merge.set_name("FLAGS_region_enable_auto_merge"); + config_auto_merge.set_value("false"); + request.mutable_control_config_variable()->Add(std::move(config_auto_merge)); + + butil::Status status = store_interaction->AllSendRequest("StoreService", "ControlConfig", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + } + + DINGO_LOG(INFO) << response.DebugString(); + } + + // document + { + dingodb::pb::document::HelloRequest request; + dingodb::pb::document::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = document_interaction->AllSendRequest("DocumentService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} + +TEST_F(BrInterationTest, CreateInteraction) { + std::vector addrs = {"127.0.0.1:33001", "127.0.0.1:33002", "127.0.0.1:33003"}; + std::shared_ptr interaction; + butil::Status status = br::ServerInteraction::CreateInteraction(addrs, interaction); + + // document + { + dingodb::pb::document::HelloRequest request; + dingodb::pb::document::HelloResponse response; + + request.mutable_request_info()->set_request_id(br::Helper::GetRandInt()); + request.set_is_just_version_info(true); + + butil::Status status = interaction->AllSendRequest("DocumentService", "Hello", request, response); + if (!status.ok()) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + if (status.error_code() != dingodb::pb::error::OK) { + DINGO_LOG(ERROR) << status.error_cstr(); + return; + } + + DINGO_LOG(INFO) << "local version info : " << response.version_info().DebugString(); + } +} diff --git a/test/unit_test/br/test_sst_file_writer.cc b/test/unit_test/br/test_sst_file_writer.cc new file mode 100644 index 000000000..e4ab1aa90 --- /dev/null +++ b/test/unit_test/br/test_sst_file_writer.cc @@ -0,0 +1,104 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include + +#include "br/sst_file_writer.h" +#include "fmt/core.h" +#include "proto/error.pb.h" +#include "rocksdb/sst_file_reader.h" + +class BrSstFileWriterTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string file_path = "./br_sst_file_writer.sst"; + std::shared_ptr sst; +}; + +TEST_F(BrSstFileWriterTest, SaveFile1) { + rocksdb::Options options; + sst = std::make_shared(options); + + std::map kvs; + + butil::Status status; + status = sst->SaveFile(kvs, file_path); + + EXPECT_EQ(status.error_code(), 4); + + sst.reset(); + + sst = std::make_shared(options); + + status = sst->SaveFile(kvs, file_path); + + EXPECT_EQ(status.error_code(), 4); + sst.reset(); +} + +TEST_F(BrSstFileWriterTest, SaveFile2) { + rocksdb::Options options; + sst = std::make_shared(options); + + std::map kvs; + + for (int i = 0; i < 10; i++) { + kvs.insert({fmt::format("key_{}", i), fmt::format("value_{}", i)}); + } + butil::Status status; + status = sst->SaveFile(kvs, file_path); + EXPECT_EQ(status.error_code(), dingodb::pb::error::Errno::OK); + int64_t size = sst->GetSize(); + EXPECT_EQ(size, 1131); + sst.reset(); + + sst = std::make_shared(options); + status = sst->SaveFile(kvs, file_path); + EXPECT_EQ(status.error_code(), dingodb::pb::error::Errno::OK); + size = sst->GetSize(); + EXPECT_EQ(size, 1131); + sst.reset(); +} + +TEST_F(BrSstFileWriterTest, Read) { + rocksdb::Options options; + rocksdb::SstFileReader reader(options); + auto status = reader.Open(file_path); + EXPECT_EQ(status.code(), status.OK().code()); + + int i = 0; + auto *iter = reader.NewIterator(rocksdb::ReadOptions()); + iter->SeekToFirst(); + while (iter->Valid()) { + EXPECT_EQ(iter->key(), fmt::format("key_{}", i)); + EXPECT_EQ(iter->value(), fmt::format("value_{}", i)); + iter->Next(); + i++; + } + + EXPECT_EQ(i, 10); + + std::filesystem::remove(file_path); +} \ No newline at end of file diff --git a/test/unit_test/br/test_utils.cc b/test/unit_test/br/test_utils.cc new file mode 100644 index 000000000..ee89114e4 --- /dev/null +++ b/test/unit_test/br/test_utils.cc @@ -0,0 +1,69 @@ +// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "br/utils.h" +#include "fmt/core.h" + +class BrUtilsTest : public testing::Test { + protected: + static void SetUpTestSuite() {} + + static void TearDownTestSuite() {} + + void SetUp() override {} + void TearDown() override {} + + inline static std::string file_path = "./br_utils_test_file"; + inline static std::string dir_path = "./br_utils_test_dir"; + inline static std::string dir_recursion_path = "./br_utils_test_dir/a/b"; +}; + +TEST_F(BrUtilsTest, CreateFile) { + std::ofstream writer; + auto status = br::Utils::CreateFile(writer, file_path); + ASSERT_TRUE(status.ok()); + writer.close(); +} + +TEST_F(BrUtilsTest, FileExistsAndRegular) { + auto status = br::Utils::FileExistsAndRegular(file_path); + std::filesystem::remove(file_path); +} + +TEST_F(BrUtilsTest, CreateDir) { + auto status = br::Utils::CreateDir(dir_path); + ASSERT_TRUE(status.ok()); +} + +TEST_F(BrUtilsTest, DirExists) { + auto status = br::Utils::DirExists(dir_path); + ASSERT_TRUE(status.ok()); +} + +TEST_F(BrUtilsTest, CreateDirRecursion) { + auto status = br::Utils::CreateDirRecursion(dir_recursion_path); + ASSERT_TRUE(status.ok()); +} + +TEST_F(BrUtilsTest, ClearDir) { + auto status = br::Utils::ClearDir(dir_path); + ASSERT_TRUE(status.ok()); + + br::Utils::RemoveAllDir(dir_path, true); +}