Merge pull request #389 from uprendis/feature/incremental-genesis

Incremental genesis

Author: uprendis

cmd/opera/launcher/chaincmd.go

@@ -8,9 +8,18 @@ import (
 var (
 	EvmExportMode = cli.StringFlag{
 		Name:  "export.evm.mode",
-		Usage: `EVM export mode ("full" or "ext-mpt" or "mpt" or "none")`,
+		Usage: `EVM export mode ("full" or "ext-mpt" or "mpt")`,
 		Value: "mpt",
 	}
+	EvmExportExclude = cli.StringFlag{
+		Name:  "export.evm.exclude",
+		Usage: `DB of EVM keys to exclude from genesis`,
+	}
+	GenesisExportSections = cli.StringFlag{
+		Name:  "export.sections",
+		Usage: `Genesis sections to export separated by comma (e.g. "brs-1" or "ers" or "evm-2")`,
+		Value: "brs,ers,evm",
+	}
 	importCommand = cli.Command{
 		Name:      "import",
 		Usage:     "Import a blockchain file",
@@ -76,11 +85,13 @@ be gzipped
 			{
 				Name:      "genesis",
 				Usage:     "Export current state into a genesis file",
-				ArgsUsage: "<filename or dry-run> [<epochFrom> <epochTo>] [--export.evm.mode=none]",
+				ArgsUsage: "<filename or dry-run> [<epochFrom> <epochTo>] [--export.evm.mode=MODE --export.evm.exclude=DB_PATH --export.sections=A,B,C]",
 				Action:    utils.MigrateFlags(exportGenesis),
 				Flags: []cli.Flag{
 					DataDirFlag,
 					EvmExportMode,
+					EvmExportExclude,
+					GenesisExportSections,
 				},
 				Description: `
     opera export genesis
@@ -91,6 +102,20 @@ Optional second and third arguments control the first and
 last epoch to write.
 Pass dry-run instead of filename for calculation of hashes without exporting data.
 EVM export mode is configured with --export.evm.mode.
+`,
+			},
+			{
+				Name:      "evm-keys",
+				Usage:     "Export EVM node keys",
+				ArgsUsage: "<directory>",
+				Action:    utils.MigrateFlags(exportEvmKeys),
+				Flags: []cli.Flag{
+					DataDirFlag,
+				},
+				Description: `
+    opera export evm-keys
+
+Requires a first argument of the DB directory to write to.
 `,
 			},
 		},

cmd/opera/launcher/check.go

@@ -27,12 +27,14 @@ func checkEvm(ctx *cli.Context) error {
 	start, reported := time.Now(), time.Now()
 
 	var prevPoint idx.Block
+	var prevIndex idx.Block
 	checkBlocks := func(stateOK func(root common.Hash) (bool, error)) {
 		var (
 			lastIdx            = gdb.GetLatestBlockIndex()
 			prevPointRootExist bool
 		)
 		gdb.ForEachBlock(func(index idx.Block, block *inter.Block) {
+			prevIndex = index
 			found, err := stateOK(common.Hash(block.Root))
 			if found != prevPointRootExist {
 				if index > 0 && found {
@@ -57,9 +59,9 @@ func checkEvm(ctx *cli.Context) error {
 		})
 	}
 
-	if err := evms.CheckEvm(checkBlocks); err != nil {
+	if err := evms.CheckEvm(checkBlocks, true); err != nil {
 		return err
 	}
-	log.Info("EVM storage is verified", "last", prevPoint, "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Info("EVM storage is verified", "last", prevIndex, "elapsed", common.PrettyDuration(time.Since(start)))
 	return nil
 }

cmd/opera/launcher/export.go

@@ -10,11 +10,14 @@ import (
 
 	"github.com/Fantom-foundation/lachesis-base/hash"
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
+	"github.com/Fantom-foundation/lachesis-base/kvdb/batched"
+	"github.com/Fantom-foundation/lachesis-base/kvdb/pebble"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/status-im/keycard-go/hexutils"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"gopkg.in/urfave/cli.v1"
 
 	"github.com/Fantom-foundation/go-opera/gossip"
@@ -114,3 +117,38 @@ func exportTo(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) {
 
 	return
 }
+
+func exportEvmKeys(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+
+	cfg := makeAllConfigs(ctx)
+
+	rawDbs := makeDirectDBsProducer(cfg)
+	gdb := makeGossipStore(rawDbs, cfg)
+	defer gdb.Close()
+
+	fn := ctx.Args().First()
+
+	keysDB_, err := pebble.New(fn, 1024*opt.MiB, utils.MakeDatabaseHandles()/2, nil, nil)
+	if err != nil {
+		return err
+	}
+	keysDB := batched.Wrap(keysDB_)
+	defer keysDB.Close()
+
+	it := gdb.EvmStore().EvmDb.NewIterator(nil, nil)
+	// iterate only over MPT data
+	it = mptAndPreimageIterator{it}
+	defer it.Release()
+
+	log.Info("Exporting EVM keys", "dir", fn)
+	for it.Next() {
+		if err := keysDB.Put(it.Key(), []byte{0}); err != nil {
+			return err
+		}
+	}
+	log.Info("Exported EVM keys", "dir", fn)
+	return nil
+}

cmd/opera/launcher/genesiscmd.go

@@ -9,16 +9,20 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"strings"
 
 	"github.com/Fantom-foundation/lachesis-base/common/bigendian"
 	"github.com/Fantom-foundation/lachesis-base/hash"
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
 	"github.com/Fantom-foundation/lachesis-base/kvdb"
+	"github.com/Fantom-foundation/lachesis-base/kvdb/pebble"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"gopkg.in/urfave/cli.v1"
 
+	"github.com/Fantom-foundation/go-opera/gossip"
 	"github.com/Fantom-foundation/go-opera/gossip/evmstore"
 	"github.com/Fantom-foundation/go-opera/inter/ibr"
 	"github.com/Fantom-foundation/go-opera/inter/ier"
@@ -65,6 +69,20 @@ func (it mptAndPreimageIterator) Next() bool {
 	return false
 }
 
+type excludingIterator struct {
+	kvdb.Iterator
+	exclude kvdb.Reader
+}
+
+func (it excludingIterator) Next() bool {
+	for it.Iterator.Next() {
+		if ok, _ := it.exclude.Has(it.Key()); !ok {
+			return true
+		}
+	}
+	return false
+}
+
 type unitWriter struct {
 	plain            io.WriteSeeker
 	gziper           *gzip.Writer
@@ -176,6 +194,14 @@ func (w *unitWriter) Write(b []byte) (n int, err error) {
 	return
 }
 
+func getEpochBlock(epoch idx.Epoch, store *gossip.Store) idx.Block {
+	bs, _ := store.GetHistoryBlockEpochState(epoch)
+	if bs == nil {
+		return 0
+	}
+	return bs.LastBlock.Idx
+}
+
 func exportGenesis(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
 		utils.Fatalf("This command requires an argument.")
@@ -198,8 +224,35 @@ func exportGenesis(ctx *cli.Context) error {
 		to = idx.Epoch(n)
 	}
 	mode := ctx.String(EvmExportMode.Name)
-	if mode != "full" && mode != "ext-mpt" && mode != "mpt" && mode != "none" {
-		return errors.New("--export.evm.mode must be one of {full, ext-mpt, mpt, none}")
+	if mode != "full" && mode != "ext-mpt" && mode != "mpt" {
+		return errors.New("--export.evm.mode must be one of {full, ext-mpt, mpt}")
+	}
+
+	var excludeEvmDB kvdb.Store
+	if excludeEvmDBPath := ctx.String(EvmExportExclude.Name); len(excludeEvmDBPath) > 0 {
+		db, err := pebble.New(excludeEvmDBPath, 1024*opt.MiB, utils.MakeDatabaseHandles()/2, nil, nil)
+		if err != nil {
+			return err
+		}
+		excludeEvmDB = db
+	}
+
+	sectionsStr := ctx.String(GenesisExportSections.Name)
+	sections := map[string]string{}
+	for _, str := range strings.Split(sectionsStr, ",") {
+		before := len(sections)
+		if strings.HasPrefix(str, "brs") {
+			sections["brs"] = str
+		} else if strings.HasPrefix(str, "ers") {
+			sections["ers"] = str
+		} else if strings.HasPrefix(str, "evm") {
+			sections["evm"] = str
+		} else {
+			return fmt.Errorf("unknown section '%s': has to start with either 'brs' or 'ers' or 'evm'", str)
+		}
+		if len(sections) == before {
+			return fmt.Errorf("duplicate section: '%s'", str)
+		}
 	}
 
 	cfg := makeAllConfigs(ctx)
@@ -243,12 +296,10 @@ func exportGenesis(ctx *cli.Context) error {
 	if to > gdb.GetEpoch() {
 		to = gdb.GetEpoch()
 	}
-	toBlock := idx.Block(0)
-	fromBlock := idx.Block(0)
-	{
+	if len(sections["ers"]) > 0 {
 		log.Info("Exporting epochs", "from", from, "to", to)
 		writer := newUnitWriter(plain)
-		err := writer.Start(header, genesisstore.EpochsSection, tmpPath)
+		err := writer.Start(header, sections["ers"], tmpPath)
 		if err != nil {
 			return err
 		}
@@ -266,28 +317,29 @@ func exportGenesis(ctx *cli.Context) error {
 			if err != nil {
 				return err
 			}
-			if i == from {
-				fromBlock = er.BlockState.LastBlock.Idx
-			}
-			if i == to {
-				toBlock = er.BlockState.LastBlock.Idx
-			}
 		}
 		epochsHash, err = writer.Flush()
 		if err != nil {
 			return err
 		}
-		log.Info("Exported epochs", "hash", epochsHash.String())
+		log.Info("Exported epochs")
+		fmt.Printf("- Epochs hash: %v \n", epochsHash.String())
 	}
 
-	if fromBlock < 1 {
-		// avoid underflow
-		fromBlock = 1
-	}
-	{
+	if len(sections["brs"]) > 0 {
+		toBlock := getEpochBlock(to, gdb)
+		fromBlock := getEpochBlock(from, gdb)
+		if sections["brs"] != "brs" {
+			// to continue prev section, include blocks of prev epochs too, excluding first blocks of prev epoch (which is last block if prev section)
+			fromBlock = getEpochBlock(from-1, gdb) + 1
+		}
+		if fromBlock < 1 {
+			// avoid underflow
+			fromBlock = 1
+		}
 		log.Info("Exporting blocks", "from", fromBlock, "to", toBlock)
 		writer := newUnitWriter(plain)
-		err := writer.Start(header, genesisstore.BlocksSection, tmpPath)
+		err := writer.Start(header, sections["brs"], tmpPath)
 		if err != nil {
 			return err
 		}
@@ -313,13 +365,14 @@ func exportGenesis(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		log.Info("Exported blocks", "hash", blocksHash.String())
+		log.Info("Exported blocks")
+		fmt.Printf("- Blocks hash: %v \n", blocksHash.String())
 	}
 
-	if mode != "none" {
-		log.Info("Exporting EVM data", "from", fromBlock, "to", toBlock)
+	if len(sections["evm"]) > 0 {
+		log.Info("Exporting EVM data")
 		writer := newUnitWriter(plain)
-		err := writer.Start(header, genesisstore.EvmSection, tmpPath)
+		err := writer.Start(header, sections["evm"], tmpPath)
 		if err != nil {
 			return err
 		}
@@ -331,6 +384,9 @@ func exportGenesis(ctx *cli.Context) error {
 			// iterate only over MPT data and preimages
 			it = mptAndPreimageIterator{it}
 		}
+		if excludeEvmDB != nil {
+			it = excludingIterator{it, excludeEvmDB}
+		}
 		defer it.Release()
 		err = iodb.Write(writer, it)
 		if err != nil {
@@ -340,12 +396,9 @@ func exportGenesis(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		log.Info("Exported EVM data", "hash", evmHash.String())
+		log.Info("Exported EVM data")
+		fmt.Printf("- EVM hash: %v \n", evmHash.String())
 	}
 
-	fmt.Printf("- Epochs hash: %v \n", epochsHash.String())
-	fmt.Printf("- Blocks hash: %v \n", blocksHash.String())
-	fmt.Printf("- EVM hash: %v \n", evmHash.String())
-
 	return nil
 }

gossip/evmstore/utils.go

@@ -28,7 +28,7 @@ var (
 	emptyHash     = common.Hash{}
 )
 
-func (s *Store) CheckEvm(forEachState func(func(root common.Hash) (found bool, err error))) error {
+func (s *Store) CheckEvm(forEachState func(func(root common.Hash) (found bool, err error)), onlyRoots bool) error {
 	log.Info("Checking every node hash")
 	nodeIt := s.table.Evm.NewIterator(nil, nil)
 	defer nodeIt.Release()
@@ -68,7 +68,11 @@ func (s *Store) CheckEvm(forEachState func(func(root common.Hash) (found bool, e
 		}
 	}
 
-	log.Info("Checking presence of every node")
+	if onlyRoots {
+		log.Info("Checking presence of every root")
+	} else {
+		log.Info("Checking presence of every node")
+	}
 	var (
 		visitedHashes   = make([]common.Hash, 0, 1000000)
 		visitedI        = 0
@@ -88,7 +92,7 @@ func (s *Store) CheckEvm(forEachState func(func(root common.Hash) (found bool, e
 	forEachState(func(root common.Hash) (found bool, err error) {
 		stateTrie, err := s.EvmState.OpenTrie(root)
 		found = stateTrie != nil && err == nil
-		if !found {
+		if !found || onlyRoots {
 			return
 		}
 

opera/genesisstore/disk.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"github.com/Fantom-foundation/lachesis-base/common/bigendian"
@@ -133,14 +134,19 @@ func OpenGenesisStore(rawReader ReadAtSeekerCloser) (*Store, genesis.Hashes, err
 		// wrap with a logger
 		// human-readable name
 		name := unit.UnitName
-		if unit.UnitName == BlocksSection {
-			name = "blocks"
+		scanfName := strings.ReplaceAll(name, "-", "")
+		if scanfName[len(scanfName)-1] < '0' || scanfName[len(scanfName)-1] > '9' {
+			scanfName += "0"
 		}
-		if unit.UnitName == EpochsSection {
-			name = "epochs"
+		var part int
+		if _, err := fmt.Sscanf(scanfName, "brs%d", &part); err == nil {
+			name = fmt.Sprintf("blocks unit %d", part)
 		}
-		if unit.UnitName == EvmSection {
-			name = "EVM data"
+		if _, err := fmt.Sscanf(scanfName, "ers%d", &part); err == nil {
+			name = fmt.Sprintf("epochs unit %d", part)
+		}
+		if _, err := fmt.Sscanf(scanfName, "evm%d", &part); err == nil {
+			name = fmt.Sprintf("EVM unit %d", part)
 		}
 		loggedReader := filelog.Wrap(gzipReader, name, uncompressedSize, time.Minute)