Introduce runner and override makeConfig

.gitignore

@@ -238,3 +238,6 @@ temp-test-unit-cover.txt
 
 # server dev env for Air
 .air.toml
+
+# e2e framework
+monitoring/

go.mod

@@ -17,6 +17,7 @@ require (
 	cosmossdk.io/log v1.6.0
 	cosmossdk.io/math v1.5.3
 	cosmossdk.io/store v1.10.0-rc.1.0.20241218084712-ca559989da43
+	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c
 	github.com/cenkalti/backoff/v5 v5.0.2
 	github.com/cometbft/cometbft v1.0.1-0.20241220100824-07c737de00ff
 	github.com/cometbft/cometbft/api v1.0.1-0.20241220100824-07c737de00ff
@@ -29,6 +30,7 @@ require (
 	github.com/go-faster/xor v1.0.0
 	github.com/go-playground/validator/v10 v10.26.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-metrics v0.5.4
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/holiman/uint256 v1.3.2
@@ -153,7 +155,6 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/orderedcode v0.0.1 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/handlers v1.5.2 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect

scripts/build/build.mk

@@ -126,3 +126,12 @@ build-docker-e2e: ## build a docker image containing `beacond` used in the e2e t
 	-f ${DOCKERFILE_E2E} \
 	-t cometbft/e2e-node:local-version \
 	./testing # work around .dockerignore restrictions in the root folder
+
+build-runner: ## build e2e runner
+		@echo "Build the e2e runner..."
+		@go build -mod=readonly -o $(OUT_DIR)/runner ./testing/runner
+
+build-e2e:
+	@$(MAKE) build-docker VERSION=local-version \
+		build-docker-e2e \
+		build-runner

scripts/build/testing.mk

@@ -407,3 +407,14 @@ test-e2e-deposits: ## run e2e tests
 
 test-e2e-deposits-no-build:
 	go test -timeout 0 -tags e2e,bls12381,test ./testing/e2e/. -v -testify.m TestDepositRobustness
+
+###############################################################################
+###                          E2E Framework Testing                          ###
+###############################################################################
+
+test-e2e-single: ## run e2e single node test
+	@$(MAKE) build-e2e test-e2e-single-no-build
+
+test-e2e-single-no-build:
+	mkdir -p monitoring
+	testing/files/run-multiple.sh testing/networks/single.toml

testing/files/run-multiple.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# This is a convenience script that takes a list of testnet manifests
+# as arguments and runs each one of them sequentially. If a testnet
+# fails, the container logs are dumped to stdout along with the testnet
+# manifest, but the remaining testnets are still run.
+#
+# This is mostly used to run generated networks in nightly CI jobs.
+#
+
+set -euo pipefail
+
+if [[ $# == 0 ]]; then
+	echo "Usage: $0 [MANIFEST...]" >&2
+	exit 1
+fi
+
+FAILED=()
+RUNNER="${RUNNER:-build/bin/runner}"
+
+for MANIFEST in "$@"; do
+	START=$SECONDS
+	echo "==> Running testnet: $MANIFEST"
+	echo "==> Manifest:"
+	cat "$MANIFEST"
+
+	if ! "$RUNNER" -f "$MANIFEST"; then
+		echo "==> Testnet failed"
+
+		echo "==> Dumping container logs for $MANIFEST..."
+		"$RUNNER" -f "$MANIFEST" logs
+
+		echo "==> Cleaning up failed testnet $MANIFEST..."
+		"$RUNNER" -f "$MANIFEST" cleanup
+
+		FAILED+=("$MANIFEST")
+	fi
+
+	echo "==> Completed testnet $MANIFEST in $(( SECONDS - START ))s"
+	echo ""
+done
+
+if [[ ${#FAILED[@]} -ne 0 ]]; then
+	echo "${#FAILED[@]} testnets failed:"
+	for MANIFEST in "${FAILED[@]}"; do
+		echo "- $MANIFEST"
+	done
+	exit 1
+else
+	echo "All testnets successful"
+fi

testing/networks/single.toml

@@ -0,0 +1 @@
+[node.validator]

testing/runner/benchmark.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"path/filepath"
+	"time"
+
+	e2e "github.com/cometbft/cometbft/test/e2e/pkg"
+	"github.com/cometbft/cometbft/types"
+)
+
+// Benchmark is a simple function for fetching, calculating and printing
+// the following metrics:
+// 1. Average block production time
+// 2. Block interval standard deviation
+// 3. Max block interval (slowest block)
+// 4. Min block interval (fastest block)
+//
+// Metrics are based of the `benchmarkLength`, the amount of consecutive blocks
+// sampled from in the testnet.
+func Benchmark(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) error {
+	block, _, err := waitForHeight(ctx, testnet, 0)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Beginning benchmark period...", "height", block.Height)
+	startAt := time.Now()
+
+	// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
+	// which should be sufficient.
+	waitingTime := time.Duration(benchmarkLength*5) * time.Second
+	endHeight, err := waitForAllNodes(ctx, testnet, block.Height+benchmarkLength, waitingTime)
+	if err != nil {
+		return err
+	}
+	dur := time.Since(startAt)
+
+	logger.Info("Ending benchmark period", "height", endHeight)
+
+	// fetch a sample of blocks
+	blocks, err := fetchBlockChainSample(ctx, testnet, benchmarkLength)
+	if err != nil {
+		return err
+	}
+
+	// slice into time intervals and collate data
+	timeIntervals := splitIntoBlockIntervals(blocks)
+	testnetStats := extractTestnetStats(timeIntervals)
+	testnetStats.populateTxns(blocks)
+	testnetStats.totalTime = dur
+	testnetStats.startHeight = blocks[0].Header.Height
+	testnetStats.endHeight = blocks[len(blocks)-1].Header.Height
+
+	// print and return
+	logger.Info(testnetStats.OutputJSON(testnet))
+	return nil
+}
+
+func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) {
+	t.numtxns = 0
+	for _, b := range blocks {
+		t.numtxns += int64(b.NumTxs)
+	}
+}
+
+type testnetStats struct {
+	startHeight int64
+	endHeight   int64
+
+	numtxns   int64
+	totalTime time.Duration
+	// average time to produce a block
+	mean time.Duration
+	// standard deviation of block production
+	std float64
+	// longest time to produce a block
+	max time.Duration
+	// shortest time to produce a block
+	min time.Duration
+}
+
+func (t *testnetStats) OutputJSON(net *e2e.Testnet) string {
+	jsn, err := json.Marshal(map[string]any{
+		"case":         filepath.Base(net.File),
+		"start_height": t.startHeight,
+		"end_height":   t.endHeight,
+		"blocks":       t.endHeight - t.startHeight,
+		"stddev":       t.std,
+		"mean":         t.mean.Seconds(),
+		"max":          t.max.Seconds(),
+		"min":          t.min.Seconds(),
+		"size":         len(net.Nodes),
+		"txns":         t.numtxns,
+		"dur":          t.totalTime.Seconds(),
+	})
+	if err != nil {
+		return ""
+	}
+
+	return string(jsn)
+}
+
+func (t *testnetStats) String() string {
+	return fmt.Sprintf(`Benchmarked from height %v to %v
+	Mean Block Interval: %v
+	Standard Deviation: %f
+	Max Block Interval: %v
+	Min Block Interval: %v
+	`,
+		t.startHeight,
+		t.endHeight,
+		t.mean,
+		t.std,
+		t.max,
+		t.min,
+	)
+}
+
+// fetchBlockChainSample waits for `benchmarkLength` amount of blocks to pass, fetching
+// all of the headers for these blocks from an archive node and returning it.
+func fetchBlockChainSample(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
+	var blocks []*types.BlockMeta
+
+	// Find the first archive node
+	archiveNode := testnet.ArchiveNodes()[0]
+	c, err := archiveNode.Client()
+	if err != nil {
+		return nil, err
+	}
+
+	// find the latest height
+	s, err := c.Status(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	to := s.SyncInfo.LatestBlockHeight
+	from := to - benchmarkLength + 1
+	if from <= testnet.InitialHeight {
+		return nil, fmt.Errorf("tesnet was unable to reach required height for benchmarking (latest height %d)", to)
+	}
+
+	// Fetch blocks
+	for from < to {
+		// fetch the blockchain metas. Currently we can only fetch 20 at a time
+		resp, err := c.BlockchainInfo(ctx, from, min(from+19, to))
+		if err != nil {
+			return nil, err
+		}
+
+		blockMetas := resp.BlockMetas
+		// we receive blocks in descending order so we have to add them in reverse
+		for i := len(blockMetas) - 1; i >= 0; i-- {
+			if blockMetas[i].Header.Height != from {
+				return nil, fmt.Errorf("node gave us another header. Wanted %d, got %d",
+					from,
+					blockMetas[i].Header.Height,
+				)
+			}
+			from++
+			blocks = append(blocks, blockMetas[i])
+		}
+	}
+
+	return blocks, nil
+}
+
+func splitIntoBlockIntervals(blocks []*types.BlockMeta) []time.Duration {
+	intervals := make([]time.Duration, len(blocks)-1)
+	lastTime := blocks[0].Header.Time
+	for i, block := range blocks {
+		// skip the first block
+		if i == 0 {
+			continue
+		}
+
+		intervals[i-1] = block.Header.Time.Sub(lastTime)
+		lastTime = block.Header.Time
+	}
+	return intervals
+}
+
+func extractTestnetStats(intervals []time.Duration) testnetStats {
+	var (
+		sum, mean time.Duration
+		std       float64
+		max       = intervals[0]
+		min       = intervals[0]
+	)
+
+	for _, interval := range intervals {
+		sum += interval
+
+		if interval > max {
+			max = interval
+		}
+
+		if interval < min {
+			min = interval
+		}
+	}
+	mean = sum / time.Duration(len(intervals))
+
+	for _, interval := range intervals {
+		diff := (interval - mean).Seconds()
+		std += math.Pow(diff, 2)
+	}
+	std = math.Sqrt(std / float64(len(intervals)))
+
+	return testnetStats{
+		mean: mean,
+		std:  std,
+		max:  max,
+		min:  min,
+	}
+}
+
+func min(a, b int64) int64 {
+	if a > b {
+		return b
+	}
+	return a
+}