Sharper gasprice estimation

evmcore/tx_pool.go

@@ -556,17 +556,6 @@ func (pool *TxPool) Pending(enforceTips bool) (map[common.Address]types.Transact
 	return pending, nil
 }
 
-func (pool *TxPool) PendingSlice() types.Transactions {
-	pool.mu.Lock()
-	defer pool.mu.Unlock()
-
-	pending := make(types.Transactions, 0, 1000)
-	for _, list := range pool.pending {
-		pending = append(pending, list.Flatten()...)
-	}
-	return pending
-}
-
 func (pool *TxPool) SampleHashes(max int) []common.Hash {
 	return pool.all.SampleHashes(max)
 }

gossip/dummy_tx_pool.go

@@ -76,13 +76,6 @@ func (p *dummyTxPool) Pending(enforceTips bool) (map[common.Address]types.Transa
 	return batches, nil
 }
 
-func (p *dummyTxPool) PendingSlice() types.Transactions {
-	p.lock.RLock()
-	defer p.lock.RUnlock()
-
-	return append(make(types.Transactions, 0, len(p.pool)), p.pool...)
-}
-
 func (p *dummyTxPool) SubscribeNewTxsNotify(ch chan<- evmcore.NewTxsNotify) notify.Subscription {
 	return p.txFeed.Subscribe(ch)
 }

gossip/gasprice/gasprice.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
 	"github.com/Fantom-foundation/lachesis-base/utils/piecefunc"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
 	lru "github.com/hashicorp/golang-lru"
@@ -56,7 +57,7 @@ type Reader interface {
 	TotalGasPowerLeft() uint64
 	GetRules() opera.Rules
 	GetPendingRules() opera.Rules
-	PendingTxs() types.Transactions
+	PendingTxs() map[common.Address]types.Transactions
 }
 
 type tipCache struct {

gossip/gasprice/gasprice_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/stretchr/testify/require"
@@ -42,14 +43,17 @@ func (t TestBackend) GetPendingRules() opera.Rules {
 	return t.pendingRules
 }
 
-func (t TestBackend) PendingTxs() types.Transactions {
-	txs := make(types.Transactions, 0, len(t.pendingTxs))
-	for _, tx := range t.pendingTxs {
-		txs = append(txs, types.NewTx(&types.DynamicFeeTx{
-			GasTipCap: tx.tip,
-			GasFeeCap: tx.cap,
-			Gas:       tx.gas,
-		}))
+func (t TestBackend) PendingTxs() map[common.Address]types.Transactions {
+	txs := make(map[common.Address]types.Transactions, len(t.pendingTxs))
+	for i, tx := range t.pendingTxs {
+		txs[common.BytesToAddress(big.NewInt(int64(i)).Bytes())] = types.Transactions{
+			types.NewTx(&types.DynamicFeeTx{
+				Nonce:     uint64(i),
+				GasTipCap: tx.tip,
+				GasFeeCap: tx.cap,
+				Gas:       tx.gas,
+			}),
+		}
 	}
 	return txs
 }
@@ -243,7 +247,7 @@ func TestOracle_reactiveGasPrice(t *testing.T) {
 	for i := 0; i < statsBuffer-5; i++ {
 		gpo.txpoolStatsTick()
 	}
-	require.Equal(t, "958333333", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "933333333", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "3500000000", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
 	require.Equal(t, "1000000000", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
@@ -255,16 +259,23 @@ func TestOracle_reactiveGasPrice(t *testing.T) {
 	// change minGasPrice
 	backend.rules.Economy.MinGasPrice = big.NewInt(100)
 	gpo.txpoolStatsTick()
-	require.Equal(t, "958333337", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
-	require.Equal(t, "3479166670", gpo.reactiveGasPrice(DecimalUnit).String())
+	require.Equal(t, "933333340", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "3466666673", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
-	require.Equal(t, "916666675", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
-	require.Equal(t, "3458333341", gpo.reactiveGasPrice(DecimalUnit).String())
-	for i := 0; i < statsBuffer-3; i++ {
+	require.Equal(t, "866666680", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "3433333346", gpo.reactiveGasPrice(DecimalUnit).String())
+	gpo.txpoolStatsTick()
+	require.Equal(t, "800000020", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "3400000020", gpo.reactiveGasPrice(DecimalUnit).String())
+	gpo.txpoolStatsTick()
+	// recent gas price plus 5%
+	require.Equal(t, "105", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "3150000105", gpo.reactiveGasPrice(DecimalUnit).String())
+	for i := 0; i < statsBuffer-5; i++ {
 		gpo.txpoolStatsTick()
 	}
-	require.Equal(t, "41666762", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
-	require.Equal(t, "3020833429", gpo.reactiveGasPrice(DecimalUnit).String())
+	require.Equal(t, "105", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "3033333426", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
 	require.Equal(t, "100", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "3000000100", gpo.reactiveGasPrice(DecimalUnit).String())
@@ -275,15 +286,15 @@ func TestOracle_reactiveGasPrice(t *testing.T) {
 	// half of txs are confirmed now
 	backend.pendingTxs = backend.pendingTxs[:2]
 	gpo.txpoolStatsTick()
-	require.Equal(t, "95", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "93", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "3000000100", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
-	require.Equal(t, "91", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "86", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "3000000100", gpo.reactiveGasPrice(DecimalUnit).String())
 	for i := 0; i < statsBuffer-3; i++ {
 		gpo.txpoolStatsTick()
 	}
-	require.Equal(t, "4", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
+	require.Equal(t, "0", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "3000000100", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
 	require.Equal(t, "0", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
@@ -304,7 +315,7 @@ func TestOracle_reactiveGasPrice(t *testing.T) {
 		gpo.txpoolStatsTick()
 	}
 	require.Equal(t, "0", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
-	require.Equal(t, "3000000100", gpo.reactiveGasPrice(DecimalUnit).String())
+	require.Equal(t, "0", gpo.reactiveGasPrice(DecimalUnit).String())
 	gpo.txpoolStatsTick()
 	require.Equal(t, "0", gpo.reactiveGasPrice(0.8*DecimalUnit).String())
 	require.Equal(t, "0", gpo.reactiveGasPrice(DecimalUnit).String())

gossip/gasprice/reactive.go

@@ -13,7 +13,7 @@ import (
 const (
 	percentilesPerStat = 20
 	statUpdatePeriod   = 1 * time.Second
-	statsBuffer        = int((24 * time.Second) / statUpdatePeriod)
+	statsBuffer        = int((15 * time.Second) / statUpdatePeriod)
 	maxGasToIndex      = 40000000
 )
 
@@ -79,6 +79,13 @@ func (gpo *Oracle) txpoolStatsLoop() {
 	}
 }
 
+func (c *circularTxpoolStats) dec(v int) int {
+	if v == 0 {
+		return len(c.stats) - 1
+	}
+	return v - 1
+}
+
 // calcAvg calculates average of statistics in the circular buffer
 func (c *circularTxpoolStats) calcAvg() txpoolStat {
 	avg := txpoolStat{}
@@ -93,9 +100,10 @@ func (c *circularTxpoolStats) calcAvg() txpoolStat {
 		nonZero++
 		avg.totalGas += s.totalGas
 		for p := range s.percentiles {
-			if s.percentiles[p] != nil {
-				avg.percentiles[p].Add(avg.percentiles[p], s.percentiles[p])
+			if s.percentiles[p] == nil {
+				continue
 			}
+			avg.percentiles[p].Add(avg.percentiles[p], s.percentiles[p])
 		}
 	}
 	if nonZero == 0 {
@@ -106,7 +114,44 @@ func (c *circularTxpoolStats) calcAvg() txpoolStat {
 	for p := range avg.percentiles {
 		avg.percentiles[p].Div(avg.percentiles[p], nonZeroBn)
 	}
-	return avg
+
+	// take maximum from previous 4 stats plus 5%
+	rec := txpoolStat{}
+	for p := range rec.percentiles {
+		rec.percentiles[p] = new(big.Int)
+	}
+	recI1 := c.dec(c.i)
+	recI2 := c.dec(recI1)
+	recI3 := c.dec(recI2)
+	recI4 := c.dec(recI3)
+	for _, s := range []txpoolStat{c.stats[recI1], c.stats[recI2], c.stats[recI3], c.stats[recI4]} {
+		for p := range s.percentiles {
+			if s.percentiles[p] == nil {
+				continue
+			}
+			if rec.percentiles[p].Cmp(s.percentiles[p]) < 0 {
+				rec.percentiles[p].Set(s.percentiles[p])
+			}
+		}
+	}
+	// increase by 5%
+	for p := range rec.percentiles {
+		rec.percentiles[p].Mul(rec.percentiles[p], big.NewInt(21))
+		rec.percentiles[p].Div(rec.percentiles[p], big.NewInt(20))
+	}
+
+	// return minimum from max(recent two stats * 1.05) and avg stats
+	res := txpoolStat{}
+	res.totalGas = avg.totalGas
+	for _, s := range []txpoolStat{avg, rec} {
+		for p := range s.percentiles {
+			if res.percentiles[p] == nil || res.percentiles[p].Cmp(s.percentiles[p]) > 0 {
+				res.percentiles[p] = s.percentiles[p]
+			}
+		}
+	}
+
+	return res
 }
 
 func (c *circularTxpoolStats) getGasPriceForGasAbove(gas uint64) *big.Int {
@@ -161,12 +206,18 @@ func (c *circularTxpoolStats) totalGas() uint64 {
 
 // calcTxpoolStat retrieves txpool transactions and calculates statistics
 func (gpo *Oracle) calcTxpoolStat() txpoolStat {
-	txs := gpo.backend.PendingTxs()
+	txsMap := gpo.backend.PendingTxs()
 	s := txpoolStat{}
-	if len(txs) == 0 {
+	if len(txsMap) == 0 {
 		// short circuit if empty txpool
 		return s
 	}
+	// take only one tx from each account
+	txs := make(types.Transactions, 0, 1000)
+	for _, aTxs := range txsMap {
+		txs = append(txs, aTxs[0])
+	}
+
 	// don't index more transactions than needed for GPO purposes
 	const maxTxsToIndex = 400
 
@@ -175,28 +226,24 @@ func (gpo *Oracle) calcTxpoolStat() txpoolStat {
 	sorted := txs
 	sort.Slice(sorted, func(i, j int) bool {
 		a, b := sorted[i], sorted[j]
-		return a.EffectiveGasTipCmp(b, minGasPrice) < 0
+		cmp := a.EffectiveGasTipCmp(b, minGasPrice)
+		if cmp == 0 {
+			return a.Gas() > b.Gas()
+		}
+		return cmp > 0
 	})
 
-	if len(txs) > maxTxsToIndex {
-		txs = txs[:maxTxsToIndex]
-	}
-	sortedDown := make(types.Transactions, len(sorted))
 	for i, tx := range sorted {
-		sortedDown[len(sorted)-1-i] = tx
-	}
-
-	for i, tx := range sortedDown {
 		s.totalGas += tx.Gas()
-		if s.totalGas > maxGasToIndex {
-			sortedDown = sortedDown[:i+1]
+		if s.totalGas > maxGasToIndex || i > maxTxsToIndex {
+			sorted = sorted[:i+1]
 			break
 		}
 	}
 
 	gasCounter := uint64(0)
 	p := uint64(0)
-	for _, tx := range sortedDown {
+	for _, tx := range sorted {
 		for p < uint64(len(s.percentiles)) && gasCounter >= p*maxGasToIndex/uint64(len(s.percentiles)) {
 			s.percentiles[p] = tx.EffectiveGasTipValue(minGasPrice)
 			if s.percentiles[p].Sign() < 0 {

gossip/gpo_backend.go

@@ -3,6 +3,7 @@ package gossip
 import (
 	"github.com/Fantom-foundation/lachesis-base/hash"
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
 	"github.com/Fantom-foundation/go-opera/eventcheck/gaspowercheck"
@@ -32,8 +33,12 @@ func (b *GPOBackend) GetPendingRules() opera.Rules {
 	return es.Rules
 }
 
-func (b *GPOBackend) PendingTxs() types.Transactions {
-	return b.txpool.PendingSlice()
+func (b *GPOBackend) PendingTxs() map[common.Address]types.Transactions {
+	txs, err := b.txpool.Pending(false)
+	if err != nil {
+		return map[common.Address]types.Transactions{}
+	}
+	return txs
 }
 
 // TotalGasPowerLeft returns a total amount of obtained gas power by the validators, according to the latest events from each validator

gossip/protocol.go

@@ -119,7 +119,6 @@ type TxPool interface {
 	Stats() (int, int)
 	Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
 	ContentFrom(addr common.Address) (types.Transactions, types.Transactions)
-	PendingSlice() types.Transactions
 }
 
 // handshakeData is the network packet for the initial handshake message