implement gc data copy

Author: JacksonYao287

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.3.23"
+    version = "2.3.24"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"

src/include/homeobject/shard_manager.hpp

@@ -27,7 +27,6 @@ struct ShardInfo {
     uint64_t last_modified_time;
     uint64_t available_capacity_bytes;
     uint64_t total_capacity_bytes;
-    uint64_t deleted_capacity_bytes;
     std::optional< peer_id_t > current_leader{std::nullopt};
 
     auto operator<=>(ShardInfo const& rhs) const { return id <=> rhs.id; }

src/lib/homeobject_impl.hpp

@@ -161,8 +161,8 @@ class HomeObjectImpl : public HomeObject,
 
     /// BlobManager
     BlobManager::AsyncResult< blob_id_t > put(shard_id_t shard, Blob&&, trace_id_t tid) final;
-    BlobManager::AsyncResult< Blob > get(shard_id_t shard, blob_id_t const& blob, uint64_t off,
-                                         uint64_t len, trace_id_t tid) const final;
+    BlobManager::AsyncResult< Blob > get(shard_id_t shard, blob_id_t const& blob, uint64_t off, uint64_t len,
+                                         trace_id_t tid) const final;
     BlobManager::NullAsyncResult del(shard_id_t shard, blob_id_t const& blob, trace_id_t tid) final;
 };
 

src/lib/homestore_backend/CMakeLists.txt

@@ -119,7 +119,8 @@ add_test(NAME HomestoreResyncTestWithLeaderRestart
 add_executable(homestore_test_gc)
 target_sources(homestore_test_gc PRIVATE $<TARGET_OBJECTS:homestore_tests_gc>)
 target_link_libraries(homestore_test_gc PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
-add_test(NAME HomestoreTestGC COMMAND homestore_test_pg -csv error --executor immediate --config_path ./
-        --override_config homestore_config.consensus.snapshot_freq_distance:0
-        --override_config homestore_config.consensus.max_grpc_message_size:138412032)
+add_test(NAME HomestoreTestGC COMMAND homestore_test_gc -csv error --executor immediate --config_path ./
+        --override_config hs_backend_config.enable_gc=true 
+        --override_config hs_backend_config.gc_garbage_rate_threshold=1 
+        --override_config hs_backend_config.gc_scan_interval_sec=5)
 

src/lib/homestore_backend/gc_manager.cpp

@@ -16,24 +16,11 @@
 
 #include "heap_chunk_selector.h"
 #include "index_kv.hpp"
+#include "hs_backend_config.hpp"
 
 namespace homeobject {
 
 class HSHomeObject;
-// TODO:: make those #define below configurable
-
-// Default number of chunks reserved for GC per pdev
-#define RESERVED_CHUNK_NUM_PER_PDEV 6
-// reserved chunk number dedicated for emergent GC
-#define RESERVED_CHUNK_NUM_DEDICATED_FOR_EGC 2
-// GC interval in seconds, this is used to schedule the GC timer
-#define GC_SCAN_INTERVAL_SEC 10llu
-// Garbage rate threshold, bigger than which, a gc task will be scheduled for a chunk
-#define GC_GARBAGE_RATE_THRESHOLD 80
-// limit the io resource that gc thread can take, so that it will not impact the client io.
-// assuming the throughput of a HDD is 300M/s(including read and write) and gc can take 10% of the io resource, which is
-// 30M/s. A block is 4K, so gc can read/write 30M/s / 4K = 7680 blocks per second.
-#define MAX_READ_WRITE_BLOCK_COUNT_PER_SECOND 7680
 
 ENUM(task_priority, uint8_t, emergent = 0, normal, priority_count);
 
@@ -72,7 +59,7 @@ class GCManager {
 
     struct gc_reserved_chunk_superblk {
         chunk_id_t chunk_id;
-        static std::string name() { return _gc_actor_meta_name; }
+        static std::string name() { return _gc_reserved_chunk_meta_name; }
     };
 #pragma pack()
 
@@ -114,7 +101,7 @@ class GCManager {
         pdev_gc_actor& operator=(pdev_gc_actor&&) = delete;
 
     public:
-        void add_reserved_chunk(chunk_id_t chunk_id);
+        void add_reserved_chunk(homestore::superblk< GCManager::gc_reserved_chunk_superblk > reserved_chunk_sb);
         folly::SemiFuture< bool > add_gc_task(uint8_t priority, chunk_id_t move_from_chunk);
         void handle_recovered_gc_task(const GCManager::gc_task_superblk* gc_task);
         void start();
@@ -136,7 +123,7 @@ class GCManager {
         // before we select a reserved chunk and start gc, we need:
         //  1 clear all the entries of this chunk in the gc index table
         //  2 reset this chunk to make sure it is empty.
-        void purge_reserved_chunk(chunk_id_t move_to_chunk);
+        bool purge_reserved_chunk(chunk_id_t move_to_chunk);
 
     private:
         // utils
@@ -148,10 +135,18 @@ class GCManager {
         folly::MPMCQueue< chunk_id_t > m_reserved_chunk_queue;
         std::shared_ptr< GCBlobIndexTable > m_index_table;
         HSHomeObject* m_hs_home_object{nullptr};
-        RateLimiter m_rate_limiter{MAX_READ_WRITE_BLOCK_COUNT_PER_SECOND};
+
+        // limit the io resource that gc thread can take, so that it will not impact the client io.
+        // assuming the throughput of a HDD is 300M/s(including read and write) and gc can take 10% of the io resource,
+        // which is 30M/s. A block is 4K, so gc can read/write 30M/s / 4K = 7680 blocks per second.
+        RateLimiter m_rate_limiter{HS_BACKEND_DYNAMIC_CONFIG(max_read_write_block_count_per_second)};
+
         std::shared_ptr< folly::IOThreadPoolExecutor > m_gc_executor;
         std::shared_ptr< folly::IOThreadPoolExecutor > m_egc_executor;
         std::atomic_bool m_is_stopped{true};
+        // since we have a very small number of reserved chunks, a vector is enough
+        // TODO:: use a map if we have a large number of reserved chunks
+        std::vector< homestore::superblk< GCManager::gc_reserved_chunk_superblk > > m_reserved_chunks;
     };
 
 public:
@@ -179,9 +174,9 @@ class GCManager {
     void start();
     void stop();
 
-private:
     void scan_chunks_for_gc();
 
+private:
     void on_gc_task_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);
     void on_gc_actor_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);
     void on_reserved_chunk_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);

src/lib/homestore_backend/gc_manager.hpp

@@ -263,6 +263,31 @@ std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id,
     return num_chunk;
 }
 
+void HeapChunkSelector::switch_chunks_for_pg(const pg_id_t pg_id, const chunk_num_t old_chunk_id,
+                                             const chunk_num_t new_chunk_id) {
+    auto EXVchunk_old = get_extend_vchunk(old_chunk_id);
+    RELEASE_ASSERT(EXVchunk_old->m_v_chunk_id.has_value(), "old_chunk_id={} should has a vchunk_id", old_chunk_id);
+    RELEASE_ASSERT(EXVchunk_old->m_pg_id.has_value(), "old_chunk_id={} should belongs to a pg", old_chunk_id);
+    RELEASE_ASSERT(EXVchunk_old->m_pg_id.value() == pg_id, "old_chunk_id={} should belongs to pg={}", old_chunk_id,
+                   pg_id);
+
+    auto v_chunk_id = EXVchunk_old->m_v_chunk_id.value();
+
+    std::unique_lock lock_guard(m_chunk_selector_mtx);
+    auto pg_it = m_per_pg_chunks.find(pg_id);
+    RELEASE_ASSERT(pg_it != m_per_pg_chunks.end(), "No pg_chunk_collection found for pg={}", pg_id);
+    auto& pg_chunk_collection = pg_it->second;
+    auto& pg_chunks = pg_chunk_collection->m_pg_chunks;
+    std::scoped_lock lock(pg_chunk_collection->mtx);
+    auto old_available_blks = pg_chunks[v_chunk_id]->available_blks();
+    pg_chunks[v_chunk_id] = get_extend_vchunk(new_chunk_id);
+    pg_chunks[v_chunk_id]->m_pg_id = pg_id;
+    pg_chunks[v_chunk_id]->m_v_chunk_id = v_chunk_id;
+    auto new_available_blks = pg_chunks[v_chunk_id]->available_blks();
+
+    pg_chunk_collection->available_blk_count += new_available_blks - old_available_blks;
+}
+
 bool HeapChunkSelector::recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids) {
     std::unique_lock lock_guard(m_chunk_selector_mtx);
     // check pg exist

src/lib/homestore_backend/heap_chunk_selector.cpp

@@ -128,7 +128,7 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     // this should be called on each pg meta blk found
     bool recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);
 
-    // this should be called after all pg meta blk recovered
+    // this should be called after all pg meta blk and gc reserved chunk recovered
     void build_pdev_available_chunk_heap();
 
     // this should be called after ShardManager is initialized and get all the open shards
@@ -198,6 +198,8 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     homestore::cshared< ExtendedVChunk > get_extend_vchunk(const chunk_num_t chunk_id) const;
 
+    void switch_chunks_for_pg(const pg_id_t pg_id, const chunk_num_t old_chunk_id, const chunk_num_t new_chunk_id);
+
 private:
     void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);
 

src/lib/homestore_backend/heap_chunk_selector.h

@@ -18,6 +18,26 @@ table HSBackendSettings {
 
     //Reserved space in a chunk
     reserved_bytes_in_chunk: uint64 = 16777216 (hotswap);
+
+    //Enable GC
+    //TODO: make this hotswap after gc is well tested
+    enable_gc: bool = false;
+
+    //Total reserved chunk num (dedicated for gc/egc) per pdev 
+    reserved_chunk_num_per_pdev: uint8 = 6;
+
+    //Reserved chunk number (dedicated for egc) per pdev
+    reserved_chunk_num_per_pdev_for_egc: uint8 = 2;
+
+    //GC scan interval(second)
+    gc_scan_interval_sec: uint64 = 60;
+
+    //GC garbage rate threshold, upon which a chunk will be selected for gc
+    gc_garbage_rate_threshold: uint8 = 80;
+
+    //max read/write block count per second, which is used by ratelimiter to limit the io resource taken by gc
+    max_read_write_block_count_per_second: uint16 = 7680;
+
 }
 
 root_type HSBackendSettings;

src/lib/homestore_backend/hs_backend_config.fbs

@@ -223,10 +223,11 @@ void HSHomeObject::init_homestore() {
         repl_app->on_repl_devs_init_completed();
 
         // we need to regitster the meta blk handlers after metaservice is ready.
-        gc_mgr_ = std::make_unique< GCManager >(chunk_selector_, this);
+        gc_mgr_ = std::make_shared< GCManager >(chunk_selector_, this);
         // if this is the first time we are starting, we need to create gc metablk for each pdev， which record the
         // reserved chunks and indextable.
         auto pdev_chunks = chunk_selector_->get_pdev_chunks();
+        const auto reserved_chunk_num_per_pdev = HS_BACKEND_DYNAMIC_CONFIG(reserved_chunk_num_per_pdev);
         for (auto const& [pdev_id, chunks] : pdev_chunks) {
             // 1 create gc index table for each pdev
             auto gc_index_table = create_gc_index_table();
@@ -241,12 +242,12 @@ void HSHomeObject::init_homestore() {
             gc_actor_sb->index_table_uuid = uuid;
             gc_actor_sb.write();
 
-            RELEASE_ASSERT(chunks.size() > RESERVED_CHUNK_NUM_PER_PDEV,
+            RELEASE_ASSERT(chunks.size() > reserved_chunk_num_per_pdev,
                            "pdev {} has {} chunks, but we need at least {} chunks for reserved chunk", pdev_id,
-                           chunks.size(), RESERVED_CHUNK_NUM_PER_PDEV);
+                           chunks.size(), reserved_chunk_num_per_pdev);
 
             // 3 create reserved chunk meta blk for each pdev, which contains the reserved chunks.
-            for (size_t i = 0; i < RESERVED_CHUNK_NUM_PER_PDEV; ++i) {
+            for (size_t i = 0; i < reserved_chunk_num_per_pdev; ++i) {
                 auto chunk = chunks[i];
                 homestore::superblk< GCManager::gc_reserved_chunk_superblk > reserved_chunk_sb{
                     GCManager::_gc_reserved_chunk_meta_name};
@@ -352,15 +353,21 @@ void HSHomeObject::init_cp() {
 
 void HSHomeObject::init_gc() {
     using namespace homestore;
-    if (!gc_mgr_) gc_mgr_ = std::make_unique< GCManager >(chunk_selector_, this);
+    if (!gc_mgr_) gc_mgr_ = std::make_shared< GCManager >(chunk_selector_, this);
     // when initializing, there is not gc task. we need to recover reserved chunks here, so that the reserved chunks
     // will not be put into pdev heap when built
     HomeStore::instance()->meta_service().read_sub_sb(GCManager::_gc_actor_meta_name);
     HomeStore::instance()->meta_service().read_sub_sb(GCManager::_gc_reserved_chunk_meta_name);
     HomeStore::instance()->meta_service().read_sub_sb(GCManager::_gc_task_meta_name);
 
-    // TODO::enable gc after we have data copy
-    // gc_mgr_->start();
+    const auto enable_gc = HS_BACKEND_DYNAMIC_CONFIG(enable_gc);
+
+    if (enable_gc) {
+        LOGI("Starting GC");
+        gc_mgr_->start();
+    } else {
+        LOGI("GC is disabled");
+    }
 }
 
 // void HSHomeObject::trigger_timed_events() { persist_pg_sb(); }
@@ -452,4 +459,13 @@ std::shared_ptr< GCBlobIndexTable > HSHomeObject::get_gc_index_table(std::string
     return it->second;
 }
 
+void HSHomeObject::trigger_immediate_gc() {
+    if (!gc_mgr_) {
+        LOGI("scan chunks for gc immediately");
+        gc_mgr_->scan_chunks_for_gc();
+    } else {
+        LOGE("GC is not enabled");
+    }
+}
+
 } // namespace homeobject

src/lib/homestore_backend/hs_homeobject.cpp

@@ -82,7 +82,7 @@ class HSHomeObject : public HomeObjectImpl {
     std::once_flag replica_restart_flag_;
 
     // mapping from chunk to shard list.
-    folly::ConcurrentHashMap< homestore::chunk_num_t, std::set< shard_id_t > > chunk_to_shards_map_;
+    std::unordered_map< homestore::chunk_num_t, std::set< shard_id_t > > chunk_to_shards_map_;
 
 public:
 #pragma pack(1)
@@ -347,7 +347,7 @@ class HSHomeObject : public HomeObjectImpl {
         HS_Shard(homestore::superblk< shard_info_superblk >&& sb);
         ~HS_Shard() override = default;
 
-        void update_info(const ShardInfo& info);
+        void update_info(const ShardInfo& info, std::optional< homestore::chunk_num_t > p_chunk_id = std::nullopt);
         auto p_chunk_id() const { return sb_->p_chunk_id; }
     };
 
@@ -622,7 +622,7 @@ class HSHomeObject : public HomeObjectImpl {
     std::unordered_map< homestore::group_id_t, homestore::superblk< snapshot_ctx_superblk > > snp_ctx_sbs_;
     mutable std::shared_mutex snp_sbs_lock_;
     shared< HeapChunkSelector > chunk_selector_;
-    std::unique_ptr< GCManager > gc_mgr_;
+    shared< GCManager > gc_mgr_;
     unique< HttpManager > http_mgr_;
     bool recovery_done_{false};
 
@@ -766,6 +766,9 @@ class HSHomeObject : public HomeObjectImpl {
      */
     std::optional< homestore::chunk_num_t > get_shard_p_chunk_id(shard_id_t id) const;
 
+    void update_shard_meta_after_gc(const homestore::chunk_num_t move_from_chunk,
+                                    const homestore::chunk_num_t move_to_chunk);
+
     /**
      * @brief Retrieves the chunk number associated with the given shard ID.
      *
@@ -818,6 +821,7 @@ class HSHomeObject : public HomeObjectImpl {
                                        cintrusive< homestore::repl_req_ctx >& hs_ctx);
 
     cshared< HeapChunkSelector > chunk_selector() const { return chunk_selector_; }
+    cshared< GCManager > gc_manager() const { return gc_mgr_; }
 
     // Blob manager related.
     void on_blob_message_rollback(int64_t lsn, sisl::blob const& header, sisl::blob const& key,
@@ -837,14 +841,20 @@ class HSHomeObject : public HomeObjectImpl {
     recover_index_table(homestore::superblk< homestore::index_table_sb >&& sb);
     std::optional< pg_id_t > get_pg_id_with_group_id(homestore::group_id_t group_id) const;
 
-    const auto get_shards_in_chunk(homestore::chunk_num_t chunk_id) const { return chunk_to_shards_map_.at(chunk_id); }
+    const std::set< shard_id_t > get_shards_in_chunk(homestore::chunk_num_t chunk_id) const;
+
+    void update_pg_meta_after_gc(const pg_id_t pg_id, const homestore::chunk_num_t move_from_chunk,
+                                 const homestore::chunk_num_t move_to_chunk);
+
+    uint32_t get_pg_tombstone_blob_count(pg_id_t pg_id) const;
 
     // Snapshot persistence related
     sisl::io_blob_safe get_snapshot_sb_data(homestore::group_id_t group_id);
     void update_snapshot_sb(homestore::group_id_t group_id, std::shared_ptr< homestore::snapshot_context > ctx);
     void destroy_snapshot_sb(homestore::group_id_t group_id);
     const Shard* _get_hs_shard(const shard_id_t shard_id) const;
     std::shared_ptr< GCBlobIndexTable > get_gc_index_table(std::string uuid) const;
+    void trigger_immediate_gc();
 
 private:
     std::shared_ptr< BlobIndexTable > create_pg_index_table();