Add evm and flow fees metric

bootstrap/bootstrap.go

@@ -290,7 +290,7 @@ func startServer(
 	}
 
 	// create transaction pool
-	txPool := requester.NewTxPool(client, transactionsPublisher, logger)
+	txPool := requester.NewTxPool(client, transactionsPublisher, logger, collector)
 
 	evm, err := requester.NewEVM(
 		client,

go.mod

@@ -211,3 +211,5 @@ require (
 	lukechampine.com/blake3 v1.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
+
+replace github.com/onflow/flow-go v0.37.1 => github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc

go.sum

@@ -978,6 +978,8 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
 github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc h1:Cs/omBT4gRwW/L/GF4A8KQw0VPvCPQ6F8T9DVG6mGLI=
+github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE=
 github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
 github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
@@ -1861,8 +1863,6 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE=
-github.com/onflow/flow-go v0.37.1 h1:DHvadojDigTOjLBLrwwKyyWXVRawmlefAk/DNVbK8cQ=
-github.com/onflow/flow-go v0.37.1/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE=
 github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo=
 github.com/onflow/flow-go-sdk v1.0.0-preview.50 h1:j5HotrV/ieo5JckmMxR2dMxO3x1j7YO8SP2EuGMEwRQ=
 github.com/onflow/flow-go-sdk v1.0.0-preview.50/go.mod h1:Ykk4PS7fgWuc6BB073tdzHu/VtzOd0CVNIoDjaqFHLg=

metrics/collector.go

@@ -2,10 +2,17 @@ package metrics
 
 import (
 	"fmt"
+	"math/big"
+	"strings"
 	"time"
 
+	sdk "github.com/onflow/flow-go-sdk"
+	"github.com/onflow/flow-go/fvm/evm/events"
+	"github.com/onflow/flow-go/model/flow"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-evm-gateway/models"
 )
 
 type Collector interface {
@@ -15,18 +22,22 @@ type Collector interface {
 	EVMHeightIndexed(height uint64)
 	EVMAccountInteraction(address string)
 	MeasureRequestDuration(start time.Time, method string)
+	EVMFeesCollected(tx models.Transaction, receipt *models.StorageReceipt)
+	FlowFeesCollected(from sdk.Address, txEvents []sdk.Event)
 }
 
 var _ Collector = &DefaultCollector{}
 
 type DefaultCollector struct {
-	// TODO: for now we cannot differentiate which api request failed number of times
+	logger                    zerolog.Logger
 	apiErrorsCounter          prometheus.Counter
 	traceDownloadErrorCounter prometheus.Counter
 	serverPanicsCounters      *prometheus.CounterVec
 	evmBlockHeight            prometheus.Gauge
 	evmAccountCallCounters    *prometheus.CounterVec
 	requestDurations          *prometheus.HistogramVec
+	evmFees                   *prometheus.GaugeVec
+	flowFees                  *prometheus.GaugeVec
 }
 
 func NewCollector(logger zerolog.Logger) Collector {
@@ -62,26 +73,46 @@ func NewCollector(logger zerolog.Logger) Collector {
 		Buckets: prometheus.DefBuckets,
 	}, []string{"method"})
 
+	evmFees := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: prefixedName("evm_fees_total"),
+			Help: "The total amount of fees collected on EVM side in gas",
+		}, []string{"account"},
+	)
+
+	flowFees := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: prefixedName("flow_fees_total"),
+			Help: "The total amount of fees collected on Flow side in FLOW",
+		}, []string{"account"},
+	)
+
 	metrics := []prometheus.Collector{
 		apiErrors,
 		traceDownloadErrorCounter,
 		serverPanicsCounters,
 		evmBlockHeight,
 		evmAccountCallCounters,
 		requestDurations,
+		evmFees,
+		flowFees,
 	}
+
 	if err := registerMetrics(logger, metrics...); err != nil {
 		logger.Info().Msg("using noop collector as metric register failed")
 		return NopCollector
 	}
 
 	return &DefaultCollector{
+		logger:                    logger,
 		apiErrorsCounter:          apiErrors,
 		traceDownloadErrorCounter: traceDownloadErrorCounter,
 		serverPanicsCounters:      serverPanicsCounters,
 		evmBlockHeight:            evmBlockHeight,
 		evmAccountCallCounters:    evmAccountCallCounters,
 		requestDurations:          requestDurations,
+		evmFees:                   evmFees,
+		flowFees:                  flowFees,
 	}
 }
 
@@ -123,6 +154,62 @@ func (c *DefaultCollector) MeasureRequestDuration(start time.Time, method string
 		Observe(float64(time.Since(start)))
 }
 
+func (c *DefaultCollector) EVMFeesCollected(tx models.Transaction, receipt *models.StorageReceipt) {
+	if receipt == nil {
+		c.logger.Debug().Msg("storage receipt is nil")
+		return
+	}
+
+	from, err := tx.From()
+	if err != nil {
+		c.logger.Debug().Msg("could not retrieve transaction sender")
+		return
+	}
+
+	gasUsed := receipt.GasUsed
+	gasPrice := receipt.EffectiveGasPrice
+
+	gasUsedBigInt := new(big.Int).SetUint64(gasUsed)
+	gasBigInt := new(big.Int).Mul(gasUsedBigInt, gasPrice)
+
+	gasFloat64, accuracy := gasBigInt.Float64()
+	if accuracy != big.Exact {
+		c.logger.Debug().Msg("precision lost when converting gas price to float64 in metrics collector")
+		return
+	}
+
+	gas := float64(gasUsed) * gasFloat64
+	c.evmFees.With(prometheus.Labels{"account": from.String()}).Add(gas)
+}
+
+func (c *DefaultCollector) FlowFeesCollected(from sdk.Address, txEvents []sdk.Event) {
+	feesDeducted, err := findEvent(events.EventTypeFlowFeesDeducted, txEvents)
+	if err != nil {
+		c.logger.Debug().Err(err).Msg("fees metric will not be collected as appropriate event was not found")
+		return
+	}
+
+	feesDeductedPayload, err := events.DecodeFlowFeesDeductedEventPayload(feesDeducted.Value)
+	if err != nil {
+		c.logger.Debug().Err(err).Msg("failed to decode fees deducted payload")
+		return
+	}
+
+	c.flowFees.
+		With(prometheus.Labels{"account": from.String()}).
+		Add(float64(feesDeductedPayload.Amount))
+}
+
 func prefixedName(name string) string {
 	return fmt.Sprintf("evm_gateway_%s", name)
 }
+
+func findEvent(eventType flow.EventType, events []sdk.Event) (sdk.Event, error) {
+	for _, event := range events {
+		if strings.Contains(event.Type, string(eventType)) {
+			return event, nil
+		}
+	}
+
+	return sdk.Event{}, fmt.Errorf("no event with type %s found", eventType)
+}

metrics/nop.go

@@ -2,6 +2,10 @@ package metrics
 
 import (
 	"time"
+
+	sdk "github.com/onflow/flow-go-sdk"
+
+	"github.com/onflow/flow-evm-gateway/models"
 )
 
 type nopCollector struct{}
@@ -10,9 +14,11 @@ var _ Collector = &nopCollector{}
 
 var NopCollector = &nopCollector{}
 
-func (c *nopCollector) ApiErrorOccurred()                        {}
-func (c *nopCollector) TraceDownloadFailed()                     {}
-func (c *nopCollector) ServerPanicked(string)                    {}
-func (c *nopCollector) EVMHeightIndexed(uint64)                  {}
-func (c *nopCollector) EVMAccountInteraction(string)             {}
-func (c *nopCollector) MeasureRequestDuration(time.Time, string) {}
+func (c *nopCollector) ApiErrorOccurred()                                           {}
+func (c *nopCollector) TraceDownloadFailed()                                        {}
+func (c *nopCollector) ServerPanicked(string)                                       {}
+func (c *nopCollector) EVMHeightIndexed(uint64)                                     {}
+func (c *nopCollector) EVMAccountInteraction(string)                                {}
+func (c *nopCollector) MeasureRequestDuration(time.Time, string)                    {}
+func (c *nopCollector) EVMFeesCollected(models.Transaction, *models.StorageReceipt) {}
+func (c *nopCollector) FlowFeesCollected(sdk.Address, []sdk.Event)                  {}

services/ingestion/engine.go

@@ -159,10 +159,12 @@ func (e *Engine) processEvents(events *models.CadenceEvents) error {
 	for i, tx := range events.Transactions() {
 		receipt := events.Receipts()[i]
 
-		err := e.indexTransaction(tx, receipt, batch)
+		err = e.indexTransaction(tx, receipt, batch)
 		if err != nil {
 			return fmt.Errorf("failed to index transaction %s event: %w", tx.Hash().String(), err)
 		}
+
+		e.collector.EVMFeesCollected(tx, receipt)
 	}
 
 	err = e.indexReceipts(events.Receipts(), events.Block(), batch)

services/requester/pool.go

@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/sethvargo/go-retry"
 
+	"github.com/onflow/flow-evm-gateway/metrics"
 	"github.com/onflow/flow-evm-gateway/models"
 	errs "github.com/onflow/flow-evm-gateway/models/errors"
 )
@@ -30,19 +31,22 @@ type TxPool struct {
 	client      *CrossSporkClient
 	pool        *sync.Map
 	txPublisher *models.Publisher
+	collector   metrics.Collector
 	// todo add methods to inspect transaction pool state
 }
 
 func NewTxPool(
 	client *CrossSporkClient,
 	transactionsPublisher *models.Publisher,
 	logger zerolog.Logger,
+	collector metrics.Collector,
 ) *TxPool {
 	return &TxPool{
 		logger:      logger.With().Str("component", "tx-pool").Logger(),
 		client:      client,
 		txPublisher: transactionsPublisher,
 		pool:        &sync.Map{},
+		collector:   collector,
 	}
 }
 
@@ -67,7 +71,8 @@ func (t *TxPool) Send(
 
 	backoff := retry.WithMaxDuration(time.Minute*3, retry.NewFibonacci(time.Millisecond*100))
 
-	return retry.Do(ctx, backoff, func(ctx context.Context) error {
+	var txRes flow.TransactionResult
+	err := retry.Do(ctx, backoff, func(ctx context.Context) error {
 		res, err := t.client.GetTransactionResult(ctx, flowTx.ID())
 		if err != nil {
 			return fmt.Errorf("failed to retrieve flow transaction result %s: %w", flowTx.ID(), err)
@@ -91,8 +96,12 @@ func (t *TxPool) Send(
 			return fmt.Errorf("failed to submit flow evm transaction %s", evmTx.Hash())
 		}
 
+		txRes = *res
 		return nil
 	})
+
+	t.collector.FlowFeesCollected(flowTx.Payer, txRes.Events)
+	return err
 }
 
 // this will extract the evm specific error from the Flow transaction error message

tests/go.mod

@@ -242,3 +242,5 @@ require (
 )
 
 replace github.com/onflow/flow-evm-gateway => ../
+
+replace github.com/onflow/flow-go v0.37.1 => github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc

tests/go.sum

@@ -981,6 +981,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
 github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc h1:Cs/omBT4gRwW/L/GF4A8KQw0VPvCPQ6F8T9DVG6mGLI=
+github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE=
 github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
 github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
@@ -2090,8 +2092,6 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE=
-github.com/onflow/flow-go v0.37.1 h1:DHvadojDigTOjLBLrwwKyyWXVRawmlefAk/DNVbK8cQ=
-github.com/onflow/flow-go v0.37.1/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE=
 github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo=
 github.com/onflow/flow-go-sdk v1.0.0-preview.50 h1:j5HotrV/ieo5JckmMxR2dMxO3x1j7YO8SP2EuGMEwRQ=
 github.com/onflow/flow-go-sdk v1.0.0-preview.50/go.mod h1:Ykk4PS7fgWuc6BB073tdzHu/VtzOd0CVNIoDjaqFHLg=