diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 92cb4681..36db1112 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -290,7 +290,7 @@ func startServer( } // create transaction pool - txPool := requester.NewTxPool(client, transactionsPublisher, logger) + txPool := requester.NewTxPool(client, transactionsPublisher, logger, collector) evm, err := requester.NewEVM( client, diff --git a/go.mod b/go.mod index 4e0bda73..d0ad4a6d 100644 --- a/go.mod +++ b/go.mod @@ -211,3 +211,5 @@ require ( lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/onflow/flow-go v0.37.1 => github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc diff --git a/go.sum b/go.sum index 99d397bc..c21a6814 100644 --- a/go.sum +++ b/go.sum @@ -978,6 +978,8 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc h1:Cs/omBT4gRwW/L/GF4A8KQw0VPvCPQ6F8T9DVG6mGLI= +github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= @@ -1861,8 +1863,6 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.37.1 h1:DHvadojDigTOjLBLrwwKyyWXVRawmlefAk/DNVbK8cQ= -github.com/onflow/flow-go v0.37.1/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.50 h1:j5HotrV/ieo5JckmMxR2dMxO3x1j7YO8SP2EuGMEwRQ= github.com/onflow/flow-go-sdk v1.0.0-preview.50/go.mod h1:Ykk4PS7fgWuc6BB073tdzHu/VtzOd0CVNIoDjaqFHLg= diff --git a/metrics/collector.go b/metrics/collector.go index c585a534..248a3b6b 100644 --- a/metrics/collector.go +++ b/metrics/collector.go @@ -2,10 +2,17 @@ package metrics import ( "fmt" + "math/big" + "strings" "time" + sdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/model/flow" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" + + "github.com/onflow/flow-evm-gateway/models" ) type Collector interface { @@ -15,18 +22,22 @@ type Collector interface { EVMHeightIndexed(height uint64) EVMAccountInteraction(address string) MeasureRequestDuration(start time.Time, method string) + EVMFeesCollected(tx models.Transaction, receipt *models.StorageReceipt) + FlowFeesCollected(from sdk.Address, txEvents []sdk.Event) } var _ Collector = &DefaultCollector{} type DefaultCollector struct { - // TODO: for now we cannot differentiate which api request failed number of times + logger zerolog.Logger apiErrorsCounter prometheus.Counter traceDownloadErrorCounter prometheus.Counter serverPanicsCounters *prometheus.CounterVec evmBlockHeight prometheus.Gauge evmAccountCallCounters *prometheus.CounterVec requestDurations *prometheus.HistogramVec + evmFees *prometheus.GaugeVec + flowFees *prometheus.GaugeVec } func NewCollector(logger zerolog.Logger) Collector { @@ -62,6 +73,20 @@ func NewCollector(logger zerolog.Logger) Collector { Buckets: prometheus.DefBuckets, }, []string{"method"}) + evmFees := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prefixedName("evm_fees_total"), + Help: "The total amount of fees collected on EVM side in gas", + }, []string{"account"}, + ) + + flowFees := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prefixedName("flow_fees_total"), + Help: "The total amount of fees collected on Flow side in FLOW", + }, []string{"account"}, + ) + metrics := []prometheus.Collector{ apiErrors, traceDownloadErrorCounter, @@ -69,19 +94,25 @@ func NewCollector(logger zerolog.Logger) Collector { evmBlockHeight, evmAccountCallCounters, requestDurations, + evmFees, + flowFees, } + if err := registerMetrics(logger, metrics...); err != nil { logger.Info().Msg("using noop collector as metric register failed") return NopCollector } return &DefaultCollector{ + logger: logger, apiErrorsCounter: apiErrors, traceDownloadErrorCounter: traceDownloadErrorCounter, serverPanicsCounters: serverPanicsCounters, evmBlockHeight: evmBlockHeight, evmAccountCallCounters: evmAccountCallCounters, requestDurations: requestDurations, + evmFees: evmFees, + flowFees: flowFees, } } @@ -123,6 +154,62 @@ func (c *DefaultCollector) MeasureRequestDuration(start time.Time, method string Observe(float64(time.Since(start))) } +func (c *DefaultCollector) EVMFeesCollected(tx models.Transaction, receipt *models.StorageReceipt) { + if receipt == nil { + c.logger.Debug().Msg("storage receipt is nil") + return + } + + from, err := tx.From() + if err != nil { + c.logger.Debug().Msg("could not retrieve transaction sender") + return + } + + gasUsed := receipt.GasUsed + gasPrice := receipt.EffectiveGasPrice + + gasUsedBigInt := new(big.Int).SetUint64(gasUsed) + gasBigInt := new(big.Int).Mul(gasUsedBigInt, gasPrice) + + gasFloat64, accuracy := gasBigInt.Float64() + if accuracy != big.Exact { + c.logger.Debug().Msg("precision lost when converting gas price to float64 in metrics collector") + return + } + + gas := float64(gasUsed) * gasFloat64 + c.evmFees.With(prometheus.Labels{"account": from.String()}).Add(gas) +} + +func (c *DefaultCollector) FlowFeesCollected(from sdk.Address, txEvents []sdk.Event) { + feesDeducted, err := findEvent(events.EventTypeFlowFeesDeducted, txEvents) + if err != nil { + c.logger.Debug().Err(err).Msg("fees metric will not be collected as appropriate event was not found") + return + } + + feesDeductedPayload, err := events.DecodeFlowFeesDeductedEventPayload(feesDeducted.Value) + if err != nil { + c.logger.Debug().Err(err).Msg("failed to decode fees deducted payload") + return + } + + c.flowFees. + With(prometheus.Labels{"account": from.String()}). + Add(float64(feesDeductedPayload.Amount)) +} + func prefixedName(name string) string { return fmt.Sprintf("evm_gateway_%s", name) } + +func findEvent(eventType flow.EventType, events []sdk.Event) (sdk.Event, error) { + for _, event := range events { + if strings.Contains(event.Type, string(eventType)) { + return event, nil + } + } + + return sdk.Event{}, fmt.Errorf("no event with type %s found", eventType) +} diff --git a/metrics/nop.go b/metrics/nop.go index f113bfb6..731b5bae 100644 --- a/metrics/nop.go +++ b/metrics/nop.go @@ -2,6 +2,10 @@ package metrics import ( "time" + + sdk "github.com/onflow/flow-go-sdk" + + "github.com/onflow/flow-evm-gateway/models" ) type nopCollector struct{} @@ -10,9 +14,11 @@ var _ Collector = &nopCollector{} var NopCollector = &nopCollector{} -func (c *nopCollector) ApiErrorOccurred() {} -func (c *nopCollector) TraceDownloadFailed() {} -func (c *nopCollector) ServerPanicked(string) {} -func (c *nopCollector) EVMHeightIndexed(uint64) {} -func (c *nopCollector) EVMAccountInteraction(string) {} -func (c *nopCollector) MeasureRequestDuration(time.Time, string) {} +func (c *nopCollector) ApiErrorOccurred() {} +func (c *nopCollector) TraceDownloadFailed() {} +func (c *nopCollector) ServerPanicked(string) {} +func (c *nopCollector) EVMHeightIndexed(uint64) {} +func (c *nopCollector) EVMAccountInteraction(string) {} +func (c *nopCollector) MeasureRequestDuration(time.Time, string) {} +func (c *nopCollector) EVMFeesCollected(models.Transaction, *models.StorageReceipt) {} +func (c *nopCollector) FlowFeesCollected(sdk.Address, []sdk.Event) {} diff --git a/services/ingestion/engine.go b/services/ingestion/engine.go index 888c118d..522a1638 100644 --- a/services/ingestion/engine.go +++ b/services/ingestion/engine.go @@ -159,10 +159,12 @@ func (e *Engine) processEvents(events *models.CadenceEvents) error { for i, tx := range events.Transactions() { receipt := events.Receipts()[i] - err := e.indexTransaction(tx, receipt, batch) + err = e.indexTransaction(tx, receipt, batch) if err != nil { return fmt.Errorf("failed to index transaction %s event: %w", tx.Hash().String(), err) } + + e.collector.EVMFeesCollected(tx, receipt) } err = e.indexReceipts(events.Receipts(), events.Block(), batch) diff --git a/services/requester/pool.go b/services/requester/pool.go index 61405a29..8be7cc1a 100644 --- a/services/requester/pool.go +++ b/services/requester/pool.go @@ -12,6 +12,7 @@ import ( "github.com/rs/zerolog" "github.com/sethvargo/go-retry" + "github.com/onflow/flow-evm-gateway/metrics" "github.com/onflow/flow-evm-gateway/models" errs "github.com/onflow/flow-evm-gateway/models/errors" ) @@ -30,6 +31,7 @@ type TxPool struct { client *CrossSporkClient pool *sync.Map txPublisher *models.Publisher + collector metrics.Collector // todo add methods to inspect transaction pool state } @@ -37,12 +39,14 @@ func NewTxPool( client *CrossSporkClient, transactionsPublisher *models.Publisher, logger zerolog.Logger, + collector metrics.Collector, ) *TxPool { return &TxPool{ logger: logger.With().Str("component", "tx-pool").Logger(), client: client, txPublisher: transactionsPublisher, pool: &sync.Map{}, + collector: collector, } } @@ -67,7 +71,8 @@ func (t *TxPool) Send( backoff := retry.WithMaxDuration(time.Minute*3, retry.NewFibonacci(time.Millisecond*100)) - return retry.Do(ctx, backoff, func(ctx context.Context) error { + var txRes flow.TransactionResult + err := retry.Do(ctx, backoff, func(ctx context.Context) error { res, err := t.client.GetTransactionResult(ctx, flowTx.ID()) if err != nil { return fmt.Errorf("failed to retrieve flow transaction result %s: %w", flowTx.ID(), err) @@ -91,8 +96,12 @@ func (t *TxPool) Send( return fmt.Errorf("failed to submit flow evm transaction %s", evmTx.Hash()) } + txRes = *res return nil }) + + t.collector.FlowFeesCollected(flowTx.Payer, txRes.Events) + return err } // this will extract the evm specific error from the Flow transaction error message diff --git a/tests/go.mod b/tests/go.mod index 32366e9a..acfbe8c4 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -242,3 +242,5 @@ require ( ) replace github.com/onflow/flow-evm-gateway => ../ + +replace github.com/onflow/flow-go v0.37.1 => github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc diff --git a/tests/go.sum b/tests/go.sum index 15fe9a2e..22296d62 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -981,6 +981,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc h1:Cs/omBT4gRwW/L/GF4A8KQw0VPvCPQ6F8T9DVG6mGLI= +github.com/The-K-R-O-K/flow-go v0.35.15-crescendo-preview.28-atree-inlining.0.20240814093750-db66de6115bc/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= @@ -2090,8 +2092,6 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go v0.37.1 h1:DHvadojDigTOjLBLrwwKyyWXVRawmlefAk/DNVbK8cQ= -github.com/onflow/flow-go v0.37.1/go.mod h1:hLFem+cwkq6650p4+0DUp36GH2QEuuFDCDUzDg0QZNE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= github.com/onflow/flow-go-sdk v1.0.0-preview.50 h1:j5HotrV/ieo5JckmMxR2dMxO3x1j7YO8SP2EuGMEwRQ= github.com/onflow/flow-go-sdk v1.0.0-preview.50/go.mod h1:Ykk4PS7fgWuc6BB073tdzHu/VtzOd0CVNIoDjaqFHLg=