rpcdaemon: move transaction/cursor hierarchy to db (#2136)

Author: canepat

cmd/dev/grpc_toolbox.cpp

@@ -31,15 +31,16 @@
 
 #include <silkworm/core/common/util.hpp>
 #include <silkworm/core/types/address.hpp>
+#include <silkworm/db/kv/api/util.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/grpc/client/client_context_pool.hpp>
+#include <silkworm/infra/grpc/client/util.hpp>
 #include <silkworm/interfaces/remote/ethbackend.grpc.pb.h>
 #include <silkworm/interfaces/remote/kv.grpc.pb.h>
 #include <silkworm/interfaces/types/types.pb.h>
 #include <silkworm/rpc/common/constants.hpp>
 #include <silkworm/rpc/common/util.hpp>
 #include <silkworm/rpc/ethbackend/remote_backend.hpp>
-#include <silkworm/rpc/grpc/util.hpp>
 
 using namespace silkworm;
 using namespace silkworm::rpc;

examples/get_latest_block.cpp

@@ -28,6 +28,7 @@
 #include <boost/asio/use_future.hpp>
 #include <grpcpp/grpcpp.h>
 
+#include <silkworm/db/kv/api/state_cache.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/grpc/client/client_context_pool.hpp>
 #include <silkworm/rpc/common/constants.hpp>
@@ -36,6 +37,7 @@
 #include <silkworm/rpc/ethdb/kv/remote_database.hpp>
 
 using namespace silkworm;
+using namespace silkworm::db;
 using namespace silkworm::rpc;
 
 ABSL_FLAG(std::string, target, kDefaultPrivateApiAddr, "server location as string <address>:<port>");
@@ -93,7 +95,7 @@ int main(int argc, char* argv[]) {
         auto* io_context = context.io_context();
         auto* grpc_context = context.grpc_context();
 
-        ethdb::kv::CoherentStateCache state_cache;
+        kv::api::CoherentStateCache state_cache;
         auto channel{::grpc::CreateChannel(target, ::grpc::InsecureChannelCredentials())};
         auto backend{std::make_unique<rpc::ethbackend::RemoteBackEnd>(*io_context, channel, *grpc_context)};
         auto database = std::make_unique<ethdb::kv::RemoteDatabase>(backend.get(), &state_cache, *grpc_context, channel);

silkworm/db/CMakeLists.txt

@@ -22,9 +22,13 @@ add_subdirectory(snapshots)
 add_subdirectory(test_util)
 
 find_package(absl REQUIRED)
+find_package(asio-grpc REQUIRED)
+find_package(Boost REQUIRED headers)
+find_package(gRPC REQUIRED)
 find_package(magic_enum REQUIRED)
 find_package(Microsoft.GSL REQUIRED)
 find_package(nlohmann_json REQUIRED)
+find_package(Protobuf REQUIRED)
 
 set(LIBS_PUBLIC
     absl::btree
@@ -41,6 +45,10 @@ set(LIBS_PRIVATE
     magic_enum::magic_enum
     Microsoft.GSL::GSL
     nlohmann_json::nlohmann_json
+    asio-grpc::asio-grpc
+    Boost::headers
+    gRPC::grpc++
+    protobuf::libprotobuf
     silkworm_db_etl
     silkworm_infra
     silkworm_snapshots
@@ -53,4 +61,4 @@ silkworm_library(
   PRIVATE ${LIBS_PRIVATE}
 )
 
-target_link_libraries(silkworm_db_test PRIVATE silkworm_db_test_util)
+target_link_libraries(silkworm_db_test PRIVATE silkworm_db_test_util silkworm_infra_test_util)

silkworm/db/access_layer_test.cpp

@@ -30,7 +30,7 @@
 #include <silkworm/db/test_util/temp_chain_data.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 
-#include "test_util/mock_cursor.hpp"
+#include "test_util/mock_ro_cursor.hpp"
 #include "test_util/mock_txn.hpp"
 
 namespace silkworm {

silkworm/db/chain/chain.cpp

@@ -0,0 +1,83 @@
+/*
+   Copyright 2023 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "chain.hpp"
+
+#include <string>
+#include <utility>
+
+#include <silkworm/core/common/endian.hpp>
+#include <silkworm/core/common/util.hpp>
+#include <silkworm/core/rlp/decode.hpp>
+#include <silkworm/core/types/address.hpp>
+#include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/kv/api/util.hpp>
+#include <silkworm/db/tables.hpp>
+#include <silkworm/db/util.hpp>
+#include <silkworm/infra/common/log.hpp>
+
+namespace silkworm::db::chain {
+
+Task<uint64_t> read_header_number(kv::api::Transaction& tx, const evmc::bytes32& block_hash) {
+    const ByteView block_hash_bytes{block_hash.bytes, kHashLength};
+    const auto value{co_await tx.get_one(table::kHeaderNumbersName, block_hash_bytes)};
+    if (value.empty()) {
+        throw std::invalid_argument{"empty block number value in read_header_number"};
+    }
+    co_return endian::load_big_u64(value.data());
+}
+
+Task<evmc::bytes32> read_canonical_block_hash(kv::api::Transaction& tx, uint64_t block_number) {
+    const auto block_key = db::block_key(block_number);
+    SILK_TRACE << "read_canonical_block_hash block_key: " << to_hex(block_key);
+    const auto value{co_await tx.get_one(table::kCanonicalHashesName, block_key)};
+    if (value.empty()) {
+        throw std::invalid_argument{"empty block hash value in read_canonical_block_hash"};
+    }
+    const auto canonical_block_hash{to_bytes32(value)};
+    SILK_DEBUG << "read_canonical_block_hash canonical block hash: " << to_hex(canonical_block_hash);
+    co_return canonical_block_hash;
+}
+
+Task<intx::uint256> read_total_difficulty(kv::api::Transaction& tx, const evmc::bytes32& block_hash, uint64_t block_number) {
+    const auto block_key = db::block_key(block_number, block_hash.bytes);
+    SILK_TRACE << "read_total_difficulty block_key: " << to_hex(block_key);
+    const auto result{co_await tx.get_one(table::kDifficultyName, block_key)};
+    if (result.empty()) {
+        throw std::invalid_argument{"empty total difficulty value in read_total_difficulty"};
+    }
+    ByteView value{result};
+    intx::uint256 total_difficulty{0};
+    auto decoding_result{rlp::decode(value, total_difficulty)};
+    if (!decoding_result) {
+        throw std::runtime_error{"cannot RLP-decode total difficulty value in read_total_difficulty"};
+    }
+    SILK_DEBUG << "read_total_difficulty canonical total difficulty: " << total_difficulty;
+    co_return total_difficulty;
+}
+
+Task<evmc::bytes32> read_head_header_hash(kv::api::Transaction& tx) {
+    const Bytes kHeadHeaderKey = bytes_of_string(table::kHeadHeaderName);
+    const auto value = co_await tx.get_one(table::kHeadHeaderName, kHeadHeaderKey);
+    if (value.empty()) {
+        throw std::runtime_error{"empty head header hash value in read_head_header_hash"};
+    }
+    const auto head_header_hash{to_bytes32(value)};
+    SILK_DEBUG << "head header hash: " << to_hex(head_header_hash);
+    co_return head_header_hash;
+}
+
+}  // namespace silkworm::db::chain

silkworm/rpc/core/rawdb/chain.hpp renamed to silkworm/db/chain/chain.hpp

@@ -24,20 +24,20 @@
 #include <intx/intx.hpp>
 #include <nlohmann/json.hpp>
 
-#include <silkworm/rpc/ethdb/transaction.hpp>
+#include <silkworm/db/kv/api/transaction.hpp>
 
-namespace silkworm::rpc::core::rawdb {
+namespace silkworm::db::chain {
 
 // TODO(canepat) BlockReader or migrate to ChainStorage?
 
 using Transactions = std::vector<silkworm::Transaction>;
 
-Task<uint64_t> read_header_number(ethdb::Transaction& tx, const evmc::bytes32& block_hash);
+Task<uint64_t> read_header_number(kv::api::Transaction& tx, const evmc::bytes32& block_hash);
 
-Task<evmc::bytes32> read_canonical_block_hash(ethdb::Transaction& tx, BlockNum block_number);
+Task<evmc::bytes32> read_canonical_block_hash(kv::api::Transaction& tx, BlockNum block_number);
 
-Task<intx::uint256> read_total_difficulty(ethdb::Transaction& tx, const evmc::bytes32& block_hash, BlockNum block_number);
+Task<intx::uint256> read_total_difficulty(kv::api::Transaction& tx, const evmc::bytes32& block_hash, BlockNum block_number);
 
-Task<evmc::bytes32> read_head_header_hash(ethdb::Transaction& tx);
+Task<evmc::bytes32> read_head_header_hash(kv::api::Transaction& tx);
 
-}  // namespace silkworm::rpc::core::rawdb
+}  // namespace silkworm::db::chain

silkworm/rpc/storage/chain_storage.hpp renamed to silkworm/db/chain/chain_storage.hpp

@@ -22,7 +22,7 @@
 #include <silkworm/core/common/bytes.hpp>
 #include <silkworm/core/types/block.hpp>
 
-namespace silkworm::rpc {
+namespace silkworm::db::chain {
 
 //! ChainStorage represents the storage for blockchain primary data, namely: chain configuration, block headers,
 //! bodies and transactions.
@@ -40,12 +40,12 @@ class ChainStorage {
     [[nodiscard]] virtual Task<std::optional<BlockNum>> read_block_number(const Hash& hash) const = 0;
 
     //! Read block returning true on success and false on missing block
-    virtual Task<bool> read_block(HashAsSpan hash, BlockNum number, bool read_senders, silkworm::Block& block) const = 0;
-    virtual Task<bool> read_block(const Hash& hash, BlockNum number, silkworm::Block& block) const = 0;
-    virtual Task<bool> read_block(const Hash& hash, silkworm::Block& block) const = 0;
+    virtual Task<bool> read_block(HashAsSpan hash, BlockNum number, bool read_senders, Block& block) const = 0;
+    virtual Task<bool> read_block(const Hash& hash, BlockNum number, Block& block) const = 0;
+    virtual Task<bool> read_block(const Hash& hash, Block& block) const = 0;
 
     //! Read canonical block by number returning true on success and false on missing block
-    virtual Task<bool> read_block(BlockNum number, bool read_senders, silkworm::Block& block) const = 0;
+    virtual Task<bool> read_block(BlockNum number, bool read_senders, Block& block) const = 0;
 
     //! Read block header with the specified key (block number, hash)
     [[nodiscard]] virtual Task<std::optional<BlockHeader>> read_header(BlockNum number, HashAsArray hash) const = 0;
@@ -60,9 +60,9 @@ class ChainStorage {
     [[nodiscard]] virtual Task<std::vector<BlockHeader>> read_sibling_headers(BlockNum number) const = 0;
 
     //! Read block body in output parameter returning true on success and false on missing block
-    virtual Task<bool> read_body(BlockNum number, HashAsArray hash, bool read_senders, silkworm::BlockBody& body) const = 0;
-    virtual Task<bool> read_body(const Hash& hash, BlockNum number, silkworm::BlockBody& body) const = 0;
-    virtual Task<bool> read_body(const Hash& hash, silkworm::BlockBody& body) const = 0;
+    virtual Task<bool> read_body(BlockNum number, HashAsArray hash, bool read_senders, BlockBody& body) const = 0;
+    virtual Task<bool> read_body(const Hash& hash, BlockNum number, BlockBody& body) const = 0;
+    virtual Task<bool> read_body(const Hash& hash, BlockBody& body) const = 0;
 
     //! Read the canonical block hash at specified height
     [[nodiscard]] virtual Task<std::optional<Hash>> read_canonical_hash(BlockNum number) const = 0;
@@ -74,7 +74,7 @@ class ChainStorage {
     virtual Task<bool> read_canonical_body(BlockNum height, BlockBody& body) const = 0;
 
     //! Read the canonical block at specified height
-    virtual Task<bool> read_canonical_block(BlockNum height, silkworm::Block& block) const = 0;
+    virtual Task<bool> read_canonical_block(BlockNum height, Block& block) const = 0;
 
     //! Check the presence of a block body using block number and hash
     [[nodiscard]] virtual Task<bool> has_body(BlockNum number, HashAsArray hash) const = 0;
@@ -89,26 +89,6 @@ class ChainStorage {
     [[nodiscard]] virtual Task<std::optional<intx::uint256>> read_total_difficulty(const Hash& block_hash, BlockNum block_number) const = 0;
 
     virtual Task<std::optional<BlockNum>> read_block_number_by_transaction_hash(const evmc::bytes32& transaction_hash) const = 0;
-
-    // Task<silkworm::BlockHeader> read_current_header();
-
-    // Task<evmc::bytes32> read_head_header_hash();
-
-    // Task<uint64_t> read_cumulative_transaction_count(BlockNum block_number);
-
-    // Task<silkworm::Bytes> read_header_rlp(const evmc::bytes32& block_hash, BlockNum block_number);
-
-    // Task<silkworm::Bytes> read_body_rlp(const evmc::bytes32& block_hash, BlockNum block_number);
-
-    // Task<Addresses> read_senders(const evmc::bytes32& block_hash, BlockNum block_number);
-
-    // Task<Receipts> read_raw_receipts(BlockNum block_number);
-
-    // Task<Receipts> read_receipts(const silkworm::BlockWithHash& block_with_hash);
-
-    // Task<Transactions> read_canonical_transactions(uint64_t base_txn_id, uint64_t txn_count);
-
-    // Task<Transactions> read_noncanonical_transactions(uint64_t base_txn_id, uint64_t txn_count);
 };
 
-}  // namespace silkworm::rpc
+}  // namespace silkworm::db::chain