[GH-846] Added bootstrap-url parameter for simple initialization of archive node.

Author: Un1oR

nil/cmd/nild/main.go

@@ -2,9 +2,12 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"os"
+	"time"
 
+	rpc_client "github.com/NilFoundation/nil/nil/client/rpc"
 	"github.com/NilFoundation/nil/nil/cmd/nild/nildconfig"
 	"github.com/NilFoundation/nil/nil/common/check"
 	"github.com/NilFoundation/nil/nil/common/concurrent"
@@ -19,6 +22,7 @@ import (
 	"github.com/NilFoundation/nil/nil/services/indexer"
 	"github.com/NilFoundation/nil/nil/services/nilservice"
 	"github.com/NilFoundation/nil/nil/services/rpc/transport"
+	rpctypes "github.com/NilFoundation/nil/nil/services/rpc/types"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -32,7 +36,7 @@ var logFilter string
 func main() {
 	logger := logging.NewLogger("nild")
 
-	cfg := parseArgs()
+	cfg := parseArgs(logger)
 
 	logging.ApplyComponentsFilter(logFilter)
 
@@ -100,7 +104,49 @@ func addBasicFlags(fset *pflag.FlagSet, cfg *nildconfig.Config) {
 		&cfg.CollatorTickPeriodMs, "collator-tick-ms", cfg.CollatorTickPeriodMs, "collator tick period in milliseconds")
 }
 
-func parseArgs() *nildconfig.Config {
+func doBootstrapRequestAndPatchConfig(bootstrapUrl string, cfg *nildconfig.Config, logger logging.Logger) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	client := rpc_client.NewRawClient(bootstrapUrl, logger)
+	response, err := client.RawCall(ctx, "debug_getBootstrapConfig")
+	if err != nil {
+		logger.Error().Err(err).Msg("failed to fetch bootstrap config")
+		return err
+	}
+
+	var bootstrapConfig rpctypes.BootstrapConfig
+	err = json.Unmarshal(response, &bootstrapConfig)
+	if err != nil {
+		return err
+	}
+
+	// Patch config
+	//
+	// Since we override a limited set of settings rather than arbitrary ones,
+	// we can choose a reasonable merging strategy that mostly avoids unexpected results.
+	cfg.NShards = bootstrapConfig.NShards
+
+	if cfg.ZeroState != nil {
+		logger.Warn().Msg("overriding zero state config")
+	}
+	cfg.ZeroState = bootstrapConfig.ZeroStateConfig
+
+	if cfg.Network.TcpPort == 0 {
+		tcpPort := 3000 // Some port to work through libp2p
+		logger.Info().Msgf("setting TCP port to %d", tcpPort)
+		cfg.Network.TcpPort = tcpPort
+	}
+
+	cfg.BootstrapPeers = append(cfg.BootstrapPeers, bootstrapConfig.BootstrapPeers...)
+
+	cfg.Network.DHTBootstrapPeers = append(cfg.Network.DHTBootstrapPeers, bootstrapConfig.DhtBootstrapPeers...)
+	cfg.Network.DHTEnabled = true
+
+	return nil
+}
+
+func parseArgs(logger logging.Logger) *nildconfig.Config {
 	cfg, err := loadConfig()
 	check.PanicIfErr(err)
 
@@ -179,17 +225,30 @@ func parseArgs() *nildconfig.Config {
 	replayCmd.Flags().Var(&cfg.Replay.BlockIdLast, "last-block", "last block id to replay")
 	replayCmd.Flags().Var(&cfg.Replay.ShardId, "shard-id", "shard id to replay block from")
 
+	var bootstrapUrl string
 	archiveCmd := &cobra.Command{
 		Use:   "archive",
 		Short: "Run nil archive node",
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg.RunMode = nilservice.ArchiveRunMode
+
+			if bootstrapUrl != "" {
+				return doBootstrapRequestAndPatchConfig(bootstrapUrl, cfg, logger)
+			}
+
+			return nil
 		},
 	}
 
 	addBasicFlags(archiveCmd.Flags(), cfg)
 	cmdflags.AddNetwork(archiveCmd.Flags(), cfg.Network)
 	cmdflags.AddTelemetry(archiveCmd.Flags(), cfg.Telemetry)
+	// N.B. Despite the fact that we override the config with the loaded configuration, we handle this flag as usual,
+	// and not through a hack like GetConfigNameFromArgs. There are two reasons for this:
+	// - We patch a limited number of settings and are confident that other processing should not depend on them
+	// - We believe that the loaded settings should take precedence over all others, so we apply them last.
+	archiveCmd.Flags().StringVar(
+		&bootstrapUrl, "bootstrap-url", "", "url to fetch initial configuration from (genesis config etc.")
 
 	rpcCmd := &cobra.Command{
 		Use:   "rpc",

nil/internal/cobrax/cmdflags/network.go

@@ -15,6 +15,7 @@ func AddNetwork(fset *pflag.FlagSet, cfg *network.Config) {
 
 	fset.BoolVar(&cfg.ServeRelay, "serve-relay", cfg.ServeRelay, "enable relay")
 	fset.Var(&cfg.Relays, "relays", "relay peers")
+	fset.Var(&cfg.RelayPublicAddress, "relay-public-address", "public address of relay")
 
 	fset.BoolVar(&cfg.DHTEnabled, "with-discovery", cfg.DHTEnabled, "enable discovery (with Kademlia DHT)")
 	fset.Var(&cfg.DHTBootstrapPeers, "discovery-bootstrap-peers", "bootstrap peers for discovery")

nil/internal/execution/zerostate.go

@@ -16,12 +16,12 @@ import (
 )
 
 type ContractDescr struct {
-	Name     string        `yaml:"name"`
-	Address  types.Address `yaml:"address,omitempty"`
-	Value    types.Value   `yaml:"value"`
-	Shard    types.ShardId `yaml:"shard,omitempty"`
-	Contract string        `yaml:"contract"`
-	CtorArgs []any         `yaml:"ctorArgs,omitempty"`
+	Name     string        `yaml:"name" json:"name"`
+	Address  types.Address `yaml:"address,omitempty" json:"address,omitempty"`
+	Value    types.Value   `yaml:"value" json:"value"`
+	Shard    types.ShardId `yaml:"shard,omitempty" json:"shard,omitempty"`
+	Contract string        `yaml:"contract" json:"contract"`
+	CtorArgs []any         `yaml:"ctorArgs,omitempty" json:"ctorArgs,omitempty"`
 }
 
 type MainKeys struct {
@@ -30,13 +30,13 @@ type MainKeys struct {
 }
 
 type ConfigParams struct {
-	Validators config.ParamValidators `yaml:"validators,omitempty"`
-	GasPrice   config.ParamGasPrice   `yaml:"gasPrice"`
+	Validators config.ParamValidators `yaml:"validators,omitempty" json:"validators,omitempty"`
+	GasPrice   config.ParamGasPrice   `yaml:"gasPrice" json:"gasPrice"`
 }
 
 type ZeroStateConfig struct {
-	ConfigParams ConfigParams     `yaml:"config,omitempty"`
-	Contracts    []*ContractDescr `yaml:"contracts"`
+	ConfigParams ConfigParams     `yaml:"config,omitempty" json:"config,omitempty"`
+	Contracts    []*ContractDescr `yaml:"contracts" json:"contracts"`
 }
 
 func CreateDefaultZeroStateConfig(mainPublicKey []byte) (*ZeroStateConfig, error) {

nil/internal/network/addr_info.go

@@ -1,12 +1,19 @@
 package network
 
 import (
+	"errors"
+
 	"github.com/NilFoundation/nil/nil/common"
 	"github.com/libp2p/go-libp2p/core/peer"
+	"gopkg.in/yaml.v3"
 )
 
 type AddrInfo peer.AddrInfo
 
+func (a AddrInfo) Empty() bool {
+	return errors.Is(a.ID.Validate(), peer.ErrEmptyPeerID)
+}
+
 func (a *AddrInfo) Set(value string) error {
 	addr, err := peer.AddrInfoFromString(value)
 	if err != nil {
@@ -44,6 +51,18 @@ func (a *AddrInfo) UnmarshalText(text []byte) error {
 	return a.Set(string(text))
 }
 
+func (a AddrInfo) MarshalYAML() (any, error) {
+	return a.String(), nil
+}
+
+func (a *AddrInfo) UnmarshalYAML(value *yaml.Node) error {
+	var str string
+	if err := value.Decode(&str); err != nil {
+		return err
+	}
+	return a.Set(str)
+}
+
 type AddrInfoSlice = common.PValueSlice[*AddrInfo, AddrInfo]
 
 func ToLibP2pAddrInfoSlice(s AddrInfoSlice) []peer.AddrInfo {

nil/internal/network/addr_info_test.go

@@ -25,14 +25,17 @@ func TestConfigYamlSerialization(t *testing.T) {
 			addrInfo1,
 			addrInfo2,
 		},
+		RelayPublicAddress: addrInfo1,
 	}
 
 	data, err := yaml.Marshal(config)
 	require.NoError(t, err)
 
-	expectedYaml := `dhtBootstrapPeers:
-- /ip4/127.0.0.1/tcp/1500/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf
-- /ip4/192.168.1.1/tcp/4002/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yg
+	expectedYaml := `---
+dhtBootstrapPeers:
+  - /ip4/127.0.0.1/tcp/1500/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf
+  - /ip4/192.168.1.1/tcp/4002/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yg
+relayPublicAddress: /ip4/127.0.0.1/tcp/1500/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf
 `
 	require.YAMLEq(t, expectedYaml, string(data))
 
@@ -53,15 +56,14 @@ func TestAddrInfoStringRepresentation(t *testing.T) {
 		"/ip4/192.168.0.10/tcp/4002/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf")
 	require.NoError(t, err)
 
-	_, id := peer.SplitAddr(addr1)
+	transport1, id := peer.SplitAddr(addr1)
+	transport2, _ := peer.SplitAddr(addr2)
 	addrInfo := AddrInfo{
 		ID:    id,
-		Addrs: []ma.Multiaddr{addr1, addr2},
+		Addrs: []ma.Multiaddr{transport1, transport2},
 	}
 
-	expectedString := "/ip4/127.0.0.1/tcp/1500/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf" +
-		"/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf," +
-		"/ip4/192.168.0.10/tcp/4002/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf" +
-		"/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf"
+	expectedString := "/ip4/127.0.0.1/tcp/1500/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf," +
+		"/ip4/192.168.0.10/tcp/4002/p2p/16Uiu2HAmQctkUi9y7WfUtYa9rPon1m5TRBtXSvUwi2VtpbWZj4yf"
 	require.Equal(t, expectedString, addrInfo.String())
 }

nil/internal/network/config.go

@@ -23,6 +23,8 @@ type Config struct {
 
 	ServeRelay bool          `yaml:"serveRelay,omitempty"`
 	Relays     AddrInfoSlice `yaml:"relays,omitempty"`
+	// RelayPublicAddress is the public relay address used to wrap the node's own address in BootstrapConfig if set.
+	RelayPublicAddress AddrInfo `yaml:"relayPublicAddress,omitempty"`
 
 	DHTEnabled        bool          `yaml:"dhtEnabled,omitempty"`
 	DHTBootstrapPeers AddrInfoSlice `yaml:"dhtBootstrapPeers,omitempty"`

nil/internal/network/host.go

@@ -14,6 +14,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
+	"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
 	"github.com/libp2p/go-libp2p/p2p/security/noise"
 	quic "github.com/libp2p/go-libp2p/p2p/transport/quic"
 	"github.com/libp2p/go-libp2p/p2p/transport/tcp"
@@ -88,7 +89,7 @@ func newHost(ctx context.Context, conf *Config) (Host, logging.Logger, error) {
 	}
 
 	if conf.ServeRelay {
-		options = append(options, libp2p.EnableRelayService())
+		options = append(options, libp2p.EnableRelayService(relay.WithInfiniteLimits()))
 
 		// todo: remove it after relay is tested
 		// this is to make sure that the relay is not disabled

nil/internal/network/relay_test.go

@@ -1,10 +1,10 @@
 package network
 
 import (
+	"context"
 	"testing"
 
 	"github.com/libp2p/go-libp2p/core/network"
-	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -29,6 +29,9 @@ func (s *RelayTestSuite) TestRelay() {
 		Relays:       []AddrInfo{CalcAddress(relay)},
 		Reachability: network.ReachabilityPrivate,
 	})
+	private.SetRequestHandler(s.context, "/hello", func(context.Context, []byte) ([]byte, error) {
+		return []byte("world"), nil
+	})
 	defer private.Close()
 
 	// Connect the private node to the relay (avoiding discovery)
@@ -38,12 +41,15 @@ func (s *RelayTestSuite) TestRelay() {
 	client := s.newManager()
 	defer client.Close()
 
-	relayedAddr, err := peer.AddrInfoFromString(hostAddress(relay) + "/p2p-circuit/p2p/" + private.host.ID().String())
+	relayedAddr := RelayedAddress(private.host.ID(), CalcAddress(relay))
+	id, err := client.Connect(s.context, relayedAddr)
 	s.Require().NoError(err)
+	s.Require().Equal(private.host.ID(), id)
 
-	id, err := client.Connect(s.context, AddrInfo(*relayedAddr))
+	resp, err := client.SendRequestAndGetResponse(
+		network.WithAllowLimitedConn(s.context, "relay"), id, "/hello", []byte("hello"))
 	s.Require().NoError(err)
-	s.Require().Equal(private.host.ID(), id)
+	s.Require().Equal("world", string(resp))
 }
 
 func TestRelay(t *testing.T) {

nil/internal/network/testaide.go

@@ -11,28 +11,14 @@ import (
 	"time"
 
 	"github.com/NilFoundation/nil/nil/common"
-	"github.com/NilFoundation/nil/nil/common/check"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/stretchr/testify/require"
 )
 
-func CalcAddress(m Manager) AddrInfo {
-	addr, err := peer.AddrInfoFromString(hostAddress(m))
-	check.PanicIfErr(err)
-	return AddrInfo(*addr)
-}
-
-func hostAddress(m Manager) string {
-	return m.getHost().Addrs()[0].String() + "/p2p/" + m.getHost().ID().String()
-}
-
 func NewTestManagerWithBaseConfig(ctx context.Context, t *testing.T, conf *Config) *BasicManager {
 	t.Helper()
 
 	conf = common.CopyPtr(conf)
-	if conf.IPV4Address == "" {
-		conf.IPV4Address = "127.0.0.1"
-	}
 	if conf.PrivateKey == nil {
 		privateKey, err := GeneratePrivateKey()
 		require.NoError(t, err)

nil/internal/network/utils.go

@@ -0,0 +1,28 @@
+package network
+
+import (
+	"github.com/NilFoundation/nil/nil/common/check"
+	ma "github.com/multiformats/go-multiaddr"
+)
+
+func CalcAddress(m Manager) AddrInfo {
+	return AddrInfo{m.getHost().ID(), m.getHost().Addrs()}
+}
+
+func RelayedAddress(p PeerID, relayAddrInfo AddrInfo) AddrInfo {
+	addrInfo := AddrInfo{
+		ID:    p,
+		Addrs: make([]ma.Multiaddr, len(relayAddrInfo.Addrs)),
+	}
+	for i, relayAddr := range relayAddrInfo.Addrs {
+		var addr ma.Multiaddr
+		relay, err := ma.NewComponent("p2p", relayAddrInfo.ID.String())
+		check.PanicIfErr(err)
+		addr = addr.AppendComponent(relay)
+		circuit, err := ma.NewComponent("p2p-circuit", "")
+		check.PanicIfErr(err)
+		addr = addr.AppendComponent(circuit)
+		addrInfo.Addrs[i] = relayAddr.Encapsulate(addr)
+	}
+	return addrInfo
+}