Merge ac219dc into merged_master (Bitcoin PR bitcoin/bitcoin#19160)

Author: apoelstra

build_msvc/bitcoind/bitcoind.vcxproj

@@ -10,6 +10,9 @@
   </PropertyGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\bitcoind.cpp" />
+    <ClCompile Include="..\..\src\init\bitcoind.cpp">
+      <ObjectFileName>$(IntDir)init_bitcoind.obj</ObjectFileName>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">

depends/packages/native_libmultiprocess.mk

@@ -1,8 +1,8 @@
 package=native_libmultiprocess
-$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0
+$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698
 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4
+$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370
 $(package)_dependencies=native_capnp
 
 define $(package)_config_cmds

doc/multiprocess.md

@@ -15,7 +15,7 @@ Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102
 
 ## Debugging
 
-After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes.
+The `-debug=ipc` command line option can be used to see requests and responses between processes.
 
 ## Installation
 
@@ -33,3 +33,40 @@ BITCOIND=bitcoin-node test/functional/test_runner.py
 The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option).
 
 Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general.
+
+## IPC implementation details
+
+Cross process Node, Wallet, and Chain interfaces are defined in
+[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow
+[conventions](developer-notes.md#internal-interface-guidelines), like passing
+serializable arguments so they can be called from different processes, and
+making methods pure virtual so they can have proxy implementations that forward
+calls between processes.
+
+When Wallet, Node, and Chain code is running in the same process, calling any
+interface method invokes the implementation directly. When code is running in
+different processes, calling an interface method invokes a proxy interface
+implementation that communicates with a remote process and invokes the real
+implementation in the remote process. The
+[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) code
+generation tool internally generates proxy client classes and proxy server
+classes for this purpose that are thin wrappers around Cap'n Proto
+[client](https://capnproto.org/cxxrpc.html#clients) and
+[server](https://capnproto.org/cxxrpc.html#servers) classes, which handle the
+actual serialization and socket communication.
+
+As much as possible, calls between processes are meant to work the same as
+calls within a single process without adding limitations or requiring extra
+implementation effort. Processes communicate with each other by calling regular
+[C++ interface methods](../src/interfaces/README.md). Method arguments and
+return values are automatically serialized and sent between processes. Object
+references and `std::function` arguments are automatically tracked and mapped
+to allow invoked code to call back into invoking code at any time, and there is
+a 1:1 threading model where any thread invoking a method in another process has
+a corresponding thread in the invoked process responsible for executing all
+method calls from the source thread, without blocking I/O or holding up another
+call, and using the same thread local variables, locks, and callbacks between
+calls. The forwarding, tracking, and threading is implemented inside the
+[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library
+which has the design goal of making calls between processes look like calls in
+the same process to the extent possible.

src/Makefile.am

@@ -74,6 +74,7 @@ EXTRA_LIBRARIES += \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_SERVER) \
   $(LIBBITCOIN_CLI) \
+  $(LIBBITCOIN_IPC) \
   $(LIBBITCOIN_WALLET) \
   $(LIBBITCOIN_WALLET_TOOL) \
   $(LIBBITCOIN_ZMQ)
@@ -165,7 +166,10 @@ BITCOIN_CORE_H = \
   init.h \
   init/common.h \
   interfaces/chain.h \
+  interfaces/echo.h \
   interfaces/handler.h \
+  interfaces/init.h \
+  interfaces/ipc.h \
   interfaces/node.h \
   interfaces/wallet.h \
   issuance.h \
@@ -312,6 +316,8 @@ obj/build.h: FORCE
 	  "$(abs_top_srcdir)"
 libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h
 
+ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h)
+
 # server: shared between bitcoind and bitcoin-qt
 # Contains code accessing mempool and chain state that is meant to be separated
 # from wallet and gui code (see node/README.md). Shared code should go in
@@ -599,7 +605,9 @@ libbitcoin_util_a_SOURCES = \
   compat/glibcxx_sanity.cpp \
   compat/strnlen.cpp \
   fs.cpp \
+  interfaces/echo.cpp \
   interfaces/handler.cpp \
+  interfaces/init.cpp \
   logging.cpp \
   random.cpp \
   randomenv.cpp \
@@ -675,17 +683,17 @@ elements_bin_ldadd = \
 
 elements_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS)
 
-elementsd_SOURCES = $(elements_daemon_sources)
+elementsd_SOURCES = $(elements_daemon_sources) init/bitcoind.cpp
 elementsd_CPPFLAGS = $(elements_bin_cppflags)
 elementsd_CXXFLAGS = $(elements_bin_cxxflags)
 elementsd_LDFLAGS = $(elements_bin_ldflags)
 elementsd_LDADD = $(LIBBITCOIN_SERVER) $(elements_bin_ldadd)
 
-elements_node_SOURCES = $(elements_daemon_sources)
+elements_node_SOURCES = $(elements_daemon_sources) init/bitcoin-node.cpp
 elements_node_CPPFLAGS = $(elements_bin_cppflags)
 elements_node_CXXFLAGS = $(elements_bin_cxxflags)
 elements_node_LDFLAGS = $(elements_bin_ldflags)
-elements_node_LDADD = $(LIBBITCOIN_SERVER) $(elements_bin_ldadd)
+elements_node_LDADD = $(LIBBITCOIN_SERVER) $(elements_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS)
 
 # bitcoin-cli binary #
 elements_cli_SOURCES = bitcoin-cli.cpp
@@ -849,6 +857,39 @@ if HARDEN
 	$(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS)
 endif
 
+libbitcoin_ipc_mpgen_input = \
+  ipc/capnp/echo.capnp \
+  ipc/capnp/init.capnp
+EXTRA_DIST += $(libbitcoin_ipc_mpgen_input)
+%.capnp:
+
+if BUILD_MULTIPROCESS
+LIBBITCOIN_IPC=libbitcoin_ipc.a
+libbitcoin_ipc_a_SOURCES = \
+  ipc/capnp/init-types.h \
+  ipc/capnp/protocol.cpp \
+  ipc/capnp/protocol.h \
+  ipc/exception.h \
+  ipc/interfaces.cpp \
+  ipc/process.cpp \
+  ipc/process.h \
+  ipc/protocol.h
+libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
+libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS)
+
+include $(MPGEN_PREFIX)/include/mpgen.mk
+libbitcoin_ipc_mpgen_output = \
+  $(libbitcoin_ipc_mpgen_input:=.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.h) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-client.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-server.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-types.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-types.h) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy.h)
+nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
+CLEANFILES += $(libbitcoin_ipc_mpgen_output)
+endif
+
 if EMBEDDED_LEVELDB
 include Makefile.crc32c.include
 include Makefile.leveldb.include

src/bitcoind.cpp

@@ -12,6 +12,7 @@
 #include <compat.h>
 #include <init.h>
 #include <interfaces/chain.h>
+#include <interfaces/init.h>
 #include <node/context.h>
 #include <node/ui_interface.h>
 #include <noui.h>
@@ -104,10 +105,8 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint)
 
 #endif
 
-static bool AppInit(int argc, char* argv[])
+static bool AppInit(NodeContext& node, int argc, char* argv[])
 {
-    NodeContext node;
-
     bool fRet = false;
 
     util::ThreadSetInternalName("init");
@@ -254,10 +253,18 @@ int main(int argc, char* argv[])
     util::WinCmdLineArgs winArgs;
     std::tie(argc, argv) = winArgs.get();
 #endif
+
+    NodeContext node;
+    int exit_status;
+    std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node, argc, argv, exit_status);
+    if (!init) {
+        return exit_status;
+    }
+
     SetupEnvironment();
 
     // Connect bitcoind signal handlers
     noui_connect();
 
-    return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE);
+    return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE);
 }

src/init/bitcoin-node.cpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/echo.h>
+#include <interfaces/init.h>
+#include <interfaces/ipc.h>
+#include <node/context.h>
+
+#include <memory>
+
+namespace init {
+namespace {
+const char* EXE_NAME = "bitcoin-node";
+
+class BitcoinNodeInit : public interfaces::Init
+{
+public:
+    BitcoinNodeInit(NodeContext& node, const char* arg0)
+        : m_node(node),
+          m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this))
+    {
+        m_node.init = this;
+    }
+    std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); }
+    interfaces::Ipc* ipc() override { return m_ipc.get(); }
+    NodeContext& m_node;
+    std::unique_ptr<interfaces::Ipc> m_ipc;
+};
+} // namespace
+} // namespace init
+
+namespace interfaces {
+std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status)
+{
+    auto init = std::make_unique<init::BitcoinNodeInit>(node, argc > 0 ? argv[0] : "");
+    // Check if bitcoin-node is being invoked as an IPC server. If so, then
+    // bypass normal execution and just respond to requests over the IPC
+    // channel and return null.
+    if (init->m_ipc->startSpawnedProcess(argc, argv, exit_status)) {
+        return nullptr;
+    }
+    return init;
+}
+} // namespace interfaces

src/init/bitcoind.cpp

@@ -0,0 +1,29 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/init.h>
+#include <node/context.h>
+
+#include <memory>
+
+namespace init {
+namespace {
+class BitcoindInit : public interfaces::Init
+{
+public:
+    BitcoindInit(NodeContext& node) : m_node(node)
+    {
+        m_node.init = this;
+    }
+    NodeContext& m_node;
+};
+} // namespace
+} // namespace init
+
+namespace interfaces {
+std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status)
+{
+    return std::make_unique<init::BitcoindInit>(node);
+}
+} // namespace interfaces

src/interfaces/README.md

@@ -12,6 +12,8 @@ The following interfaces are defined here:
 
 * [`Handler`](handler.h) — returned by `handleEvent` methods on interfaces above and used to manage lifetimes of event handlers.
 
-* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#10102](https://github.com/bitcoin/bitcoin/pull/10102).
+* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160).
 
-The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in different processes, and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally.
+* [`Ipc`](ipc.h) — used by multiprocess code to access `Init` interface across processes. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160).
+
+The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in [different processes](../../doc/multiprocess.md), and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally.

src/interfaces/echo.cpp

@@ -0,0 +1,18 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/echo.h>
+
+#include <memory>
+
+namespace interfaces {
+namespace {
+class EchoImpl : public Echo
+{
+public:
+    std::string echo(const std::string& echo) override { return echo; }
+};
+} // namespace
+std::unique_ptr<Echo> MakeEcho() { return std::make_unique<EchoImpl>(); }
+} // namespace interfaces

src/interfaces/echo.h

@@ -0,0 +1,26 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_INTERFACES_ECHO_H
+#define BITCOIN_INTERFACES_ECHO_H
+
+#include <memory>
+#include <string>
+
+namespace interfaces {
+//! Simple string echoing interface for testing.
+class Echo
+{
+public:
+    virtual ~Echo() {}
+
+    //! Echo provided string.
+    virtual std::string echo(const std::string& echo) = 0;
+};
+
+//! Return implementation of Echo interface.
+std::unique_ptr<Echo> MakeEcho();
+} // namespace interfaces
+
+#endif // BITCOIN_INTERFACES_ECHO_H