diff --git a/client/http.go b/client/http.go index ce1ae3a6..59626676 100644 --- a/client/http.go +++ b/client/http.go @@ -53,6 +53,15 @@ func (c *Client) GetBlock(blockNum uint64) (*core_types.ResultBlock, error) { return block, nil } +func (c *Client) GetGenesis() (*core_types.ResultGenesis, error) { + genesis, err := c.client.Genesis() + if err != nil { + return nil, fmt.Errorf("unable to get genesis block, %w", err) + } + + return genesis, nil +} + func (c *Client) GetBlockResults(blockNum uint64) (*core_types.ResultBlockResults, error) { bn := int64(blockNum) diff --git a/fetch/fetch.go b/fetch/fetch.go index 16024f78..199f3dad 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -11,6 +11,9 @@ import ( queue "github.com/madz-lab/insertion-queue" "go.uber.org/zap" + "github.com/gnolang/gno/gno.land/pkg/gnoland" + "github.com/gnolang/gno/tm2/pkg/amino" + bft_types "github.com/gnolang/gno/tm2/pkg/bft/types" "github.com/gnolang/tx-indexer/storage" storageErrors "github.com/gnolang/tx-indexer/storage/errors" "github.com/gnolang/tx-indexer/types" @@ -21,6 +24,8 @@ const ( DefaultMaxChunkSize = 100 ) +var errInvalidGenesisState = errors.New("invalid genesis state") + // Fetcher is an instance of the block indexer // fetcher type Fetcher struct { @@ -67,9 +72,71 @@ func New( return f } +func (f *Fetcher) fetchGenesisData() error { + _, err := f.storage.GetLatestHeight() + // Possible cases: + // - err is ErrNotFound: the storage is empty, we execute the rest of the routine and fetch+write genesis data + // - err is nil: the storage has a latest height, this means at least the genesis data has been written, + // or some blocks past it, we do nothing and return nil + // - err is something else: there has been a storage error, we do nothing and return this error + if !errors.Is(err, storageErrors.ErrNotFound) { + return err + } + + f.logger.Info("Fetching genesis") + + block, err := getGenesisBlock(f.client) + if err != nil { + return fmt.Errorf("failed to fetch genesis block: %w", err) + } + + results, err := f.client.GetBlockResults(0) + if err != nil { + return fmt.Errorf("failed to fetch genesis results: %w", err) + } + + if results.Results == nil { + return errors.New("nil results") + } + + txResults := make([]*bft_types.TxResult, len(block.Txs)) + + for txIndex, tx := range block.Txs { + result := &bft_types.TxResult{ + Height: 0, + Index: uint32(txIndex), + Tx: tx, + Response: results.Results.DeliverTxs[txIndex], + } + + txResults[txIndex] = result + } + + s := &slot{ + chunk: &chunk{ + blocks: []*bft_types.Block{block}, + results: [][]*bft_types.TxResult{txResults}, + }, + chunkRange: chunkRange{ + from: 0, + to: 0, + }, + } + + return f.writeSlot(s) +} + // FetchChainData starts the fetching process that indexes // blockchain data func (f *Fetcher) FetchChainData(ctx context.Context) error { + // Attempt to fetch the genesis data + if err := f.fetchGenesisData(); err != nil { + // We treat this error as soft, to ease migration, since + // some versions of gno networks don't support this. + // In the future, we should hard fail if genesis is not fetch-able + f.logger.Error("unable to fetch genesis data", zap.Error(err)) + } + collectorCh := make(chan *workerResponse, DefaultMaxSlots) // attemptRangeFetch compares local and remote state @@ -178,68 +245,114 @@ func (f *Fetcher) FetchChainData(ctx context.Context) error { // Pop the next chunk f.chunkBuffer.PopFront() - wb := f.storage.WriteBatch() + if err := f.writeSlot(item); err != nil { + return err + } + } + } + } +} + +func (f *Fetcher) writeSlot(s *slot) error { + wb := f.storage.WriteBatch() - // Save the fetched data - for blockIndex, block := range item.chunk.blocks { - if saveErr := wb.SetBlock(block); saveErr != nil { - // This is a design choice that really highlights the strain - // of keeping legacy testnets running. Current TM2 testnets - // have blocks / transactions that are no longer compatible - // with latest "master" changes for Amino, so these blocks / txs are ignored, - // as opposed to this error being a show-stopper for the fetcher - f.logger.Error("unable to save block", zap.String("err", saveErr.Error())) + // Save the fetched data + for blockIndex, block := range s.chunk.blocks { + if saveErr := wb.SetBlock(block); saveErr != nil { + // This is a design choice that really highlights the strain + // of keeping legacy testnets running. Current TM2 testnets + // have blocks / transactions that are no longer compatible + // with latest "master" changes for Amino, so these blocks / txs are ignored, + // as opposed to this error being a show-stopper for the fetcher + f.logger.Error("unable to save block", zap.String("err", saveErr.Error())) - continue - } + continue + } - f.logger.Debug("Added block data to batch", zap.Int64("number", block.Height)) + f.logger.Debug("Added block data to batch", zap.Int64("number", block.Height)) - // Get block results - txResults := item.chunk.results[blockIndex] + // Get block results + txResults := s.chunk.results[blockIndex] - // Save the fetched transaction results - for _, txResult := range txResults { - if err := wb.SetTx(txResult); err != nil { - f.logger.Error("unable to save tx", zap.String("err", err.Error())) + // Save the fetched transaction results + for _, txResult := range txResults { + if err := wb.SetTx(txResult); err != nil { + f.logger.Error("unable to save tx", zap.String("err", err.Error())) - continue - } + continue + } - f.logger.Debug( - "Added tx to batch", - zap.String("hash", base64.StdEncoding.EncodeToString(txResult.Tx.Hash())), - ) - } + f.logger.Debug( + "Added tx to batch", + zap.String("hash", base64.StdEncoding.EncodeToString(txResult.Tx.Hash())), + ) + } - // Alert any listeners of a new saved block - event := &types.NewBlock{ - Block: block, - Results: txResults, - } + // Alert any listeners of a new saved block + event := &types.NewBlock{ + Block: block, + Results: txResults, + } - f.events.SignalEvent(event) - } + f.events.SignalEvent(event) + } - f.logger.Info( - "Added to batch block and tx data for range", - zap.Uint64("from", item.chunkRange.from), - zap.Uint64("to", item.chunkRange.to), - ) + f.logger.Info( + "Added to batch block and tx data for range", + zap.Uint64("from", s.chunkRange.from), + zap.Uint64("to", s.chunkRange.to), + ) - // Save the latest height data - if err := wb.SetLatestHeight(item.chunkRange.to); err != nil { - if rErr := wb.Rollback(); rErr != nil { - return fmt.Errorf("unable to save latest height info, %w, %w", err, rErr) - } + // Save the latest height data + if err := wb.SetLatestHeight(s.chunkRange.to); err != nil { + if rErr := wb.Rollback(); rErr != nil { + return fmt.Errorf("unable to save latest height info, %w, %w", err, rErr) + } - return fmt.Errorf("unable to save latest height info, %w", err) - } + return fmt.Errorf("unable to save latest height info, %w", err) + } - if err := wb.Commit(); err != nil { - return fmt.Errorf("error persisting block information into storage, %w", err) - } - } + if err := wb.Commit(); err != nil { + return fmt.Errorf("error persisting block information into storage, %w", err) + } + + return nil +} + +func getGenesisBlock(client Client) (*bft_types.Block, error) { + gblock, err := client.GetGenesis() + if err != nil { + return nil, fmt.Errorf("unable to get genesis block: %w", err) + } + + if gblock.Genesis == nil { + return nil, errInvalidGenesisState + } + + genesisState, ok := gblock.Genesis.AppState.(gnoland.GnoGenesisState) + if !ok { + return nil, fmt.Errorf("unknown genesis state kind '%T'", gblock.Genesis.AppState) + } + + txs := make([]bft_types.Tx, len(genesisState.Txs)) + for i, tx := range genesisState.Txs { + txs[i], err = amino.MarshalJSON(tx) + if err != nil { + return nil, fmt.Errorf("unable to marshal genesis tx: %w", err) } } + + block := &bft_types.Block{ + Header: bft_types.Header{ + NumTxs: int64(len(txs)), + TotalTxs: int64(len(txs)), + Time: gblock.Genesis.GenesisTime, + ChainID: gblock.Genesis.ChainID, + }, + Data: bft_types.Data{ + Txs: txs, + }, + } + + return block, nil } diff --git a/fetch/fetch_test.go b/fetch/fetch_test.go index b910aba4..8fcbca0f 100644 --- a/fetch/fetch_test.go +++ b/fetch/fetch_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/gnolang/gno/gno.land/pkg/gnoland" "github.com/gnolang/gno/tm2/pkg/amino" abci "github.com/gnolang/gno/tm2/pkg/bft/abci/types" core_types "github.com/gnolang/gno/tm2/pkg/bft/rpc/core/types" @@ -158,6 +159,16 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { }, }, nil }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, } ) @@ -183,8 +194,8 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { // Verify the transactions are saved correctly require.Len(t, savedTxs, blockNum*txCount) - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) for txIndex := 0; txIndex < txCount; txIndex++ { // since this is a linearized array of transactions @@ -199,9 +210,14 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { } // Make sure proper events were emitted - require.Len(t, capturedEvents, len(blocks)-1) + require.Len(t, capturedEvents, len(blocks)) for index, event := range capturedEvents { + if index == 0 { + // Dummy genesis block + continue + } + if event.GetType() != indexerTypes.NewBlockEvent { continue } @@ -210,13 +226,13 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { require.True(t, ok) // Make sure the block is valid - assert.Equal(t, blocks[index+1], eventData.Block) + assert.Equal(t, blocks[index], eventData.Block) // Make sure the transaction results are valid require.Len(t, eventData.Results, txCount) for txIndex, tx := range eventData.Results { - assert.EqualValues(t, blocks[index+1].Height, tx.Height) + assert.EqualValues(t, blocks[index].Height, tx.Height) assert.EqualValues(t, txIndex, tx.Index) assert.Equal(t, serializedTxs[txIndex], tx.Tx) } @@ -336,6 +352,29 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { getLatestBlockNumberFn: func() (uint64, error) { return uint64(blockNum), nil }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, + getBlockResultsFn: func(num uint64) (*core_types.ResultBlockResults, error) { + // Sanity check + if num > uint64(blockNum) { + t.Fatalf("invalid block requested, %d", num) + } + + return &core_types.ResultBlockResults{ + Height: int64(num), + Results: &state.ABCIResponses{ + DeliverTxs: make([]abci.ResponseDeliverTx, txCount), + }, + }, nil + }, } ) @@ -367,8 +406,8 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { // Verify the transactions are saved correctly require.Len(t, savedTxs, blockNum*txCount) - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) for txIndex := 0; txIndex < txCount; txIndex++ { // since this is a linearized array of transactions @@ -383,18 +422,23 @@ func TestFetcher_FetchTransactions_Valid_FullBlocks(t *testing.T) { } // Make sure proper events were emitted - require.Len(t, capturedEvents, len(blocks)-1) + require.Len(t, capturedEvents, len(blocks)) for index, event := range capturedEvents { + if index == 0 { + // Dummy genesis block + continue + } + // Make sure the block is valid eventData := event.(*indexerTypes.NewBlock) - assert.Equal(t, blocks[index+1], eventData.Block) + assert.Equal(t, blocks[index], eventData.Block) // Make sure the transaction results are valid require.Len(t, eventData.Results, txCount) for txIndex, tx := range eventData.Results { - assert.EqualValues(t, blocks[index+1].Height, tx.Height) + assert.EqualValues(t, blocks[index].Height, tx.Height) assert.EqualValues(t, txIndex, tx.Index) assert.Equal(t, serializedTxs[txIndex], tx.Tx) } @@ -507,6 +551,16 @@ func TestFetcher_FetchTransactions_Valid_FullTransactions(t *testing.T) { }, }, nil }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, } ) @@ -532,16 +586,16 @@ func TestFetcher_FetchTransactions_Valid_FullTransactions(t *testing.T) { // Verify the transactions are saved correctly require.Len(t, savedTxs, blockNum*txCount) - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) for txIndex := 0; txIndex < txCount; txIndex++ { // since this is a linearized array of transactions // we can access each item with: blockNum * length + txIndx // where blockNum is the y-axis, and txIndx is the x-axis - tx := savedTxs[blockIndex*txCount+txIndex] + tx := savedTxs[(blockIndex-1)*txCount+txIndex] - assert.EqualValues(t, blockIndex+1, tx.Height) + assert.EqualValues(t, blockIndex, tx.Height) assert.EqualValues(t, txIndex, tx.Index) assert.Equal(t, serializedTxs[txIndex], tx.Tx) } @@ -549,10 +603,15 @@ func TestFetcher_FetchTransactions_Valid_FullTransactions(t *testing.T) { // Make sure proper events were emitted // Blocks each have as many transactions as txCount. - txEventCount := (len(blocks) - 1) + txEventCount := len(blocks) require.Len(t, capturedEvents, txEventCount) for index, event := range capturedEvents { + if index == 0 { + // Dummy genesis block + continue + } + if event.GetType() != indexerTypes.NewBlockEvent { continue } @@ -561,13 +620,13 @@ func TestFetcher_FetchTransactions_Valid_FullTransactions(t *testing.T) { require.True(t, ok) // Make sure the block is valid - assert.Equal(t, blocks[index+1], eventData.Block) + assert.Equal(t, blocks[index], eventData.Block) // Make sure the transaction results are valid require.Len(t, eventData.Results, txCount) for txIndex, tx := range eventData.Results { - assert.EqualValues(t, blocks[index+1].Height, tx.Height) + assert.EqualValues(t, blocks[index].Height, tx.Height) assert.EqualValues(t, txIndex, tx.Index) assert.Equal(t, serializedTxs[txIndex], tx.Tx) } @@ -650,11 +709,30 @@ func TestFetcher_FetchTransactions_Valid_EmptyBlocks(t *testing.T) { Block: blocks[num], }, nil }, - getBlockResultsFn: func(_ uint64) (*core_types.ResultBlockResults, error) { + getBlockResultsFn: func(num uint64) (*core_types.ResultBlockResults, error) { + if num == 0 { + return &core_types.ResultBlockResults{ + Height: int64(num), + Results: &state.ABCIResponses{ + DeliverTxs: make([]abci.ResponseDeliverTx, 0), + }, + }, nil + } + t.Fatalf("should not request results") return nil, nil }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, } ) @@ -668,16 +746,21 @@ func TestFetcher_FetchTransactions_Valid_EmptyBlocks(t *testing.T) { // Run the fetch require.NoError(t, f.FetchChainData(ctx)) - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) } // Make sure proper events were emitted - require.Len(t, capturedEvents, len(blocks)-1) + require.Len(t, capturedEvents, len(blocks)) for index, event := range capturedEvents { + if index == 0 { + // Dummy genesis block + continue + } + // Make sure the block is valid - assert.Equal(t, blocks[index+1], event.Block) + assert.Equal(t, blocks[index], event.Block) // Make sure the transaction results are valid require.Len(t, event.Results, 0) @@ -769,9 +852,33 @@ func TestFetcher_FetchTransactions_Valid_EmptyBlocks(t *testing.T) { }, } }, + getBlockResultsFn: func(num uint64) (*core_types.ResultBlockResults, error) { + if num == 0 { + return &core_types.ResultBlockResults{ + Height: int64(num), + Results: &state.ABCIResponses{ + DeliverTxs: make([]abci.ResponseDeliverTx, 0), + }, + }, nil + } + + t.Fatalf("should not request results") + + return nil, nil + }, getLatestBlockNumberFn: func() (uint64, error) { return uint64(blockNum), nil }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, } ) @@ -785,16 +892,21 @@ func TestFetcher_FetchTransactions_Valid_EmptyBlocks(t *testing.T) { // Run the fetch require.NoError(t, f.FetchChainData(ctx)) - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) } // Make sure proper events were emitted - require.Len(t, capturedEvents, len(blocks)-1) + require.Len(t, capturedEvents, len(blocks)) for index, event := range capturedEvents { + if index == 0 { + // Dummy genesis block + continue + } + // Make sure the block is valid - assert.Equal(t, blocks[index+1], event.Block) + assert.Equal(t, blocks[index], event.Block) // Make sure the transaction results are valid require.Len(t, event.Results, 0) @@ -877,10 +989,29 @@ func TestFetcher_InvalidBlocks(t *testing.T) { }, nil }, getBlockResultsFn: func(num uint64) (*core_types.ResultBlockResults, error) { + if num == 0 { + return &core_types.ResultBlockResults{ + Height: int64(num), + Results: &state.ABCIResponses{ + DeliverTxs: make([]abci.ResponseDeliverTx, 0), + }, + }, nil + } + require.LessOrEqual(t, num, uint64(blockNum)) return nil, fmt.Errorf("unable to fetch result for block %d", num) }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{ + Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{ + Balances: []gnoland.Balance{}, + Txs: []std.Tx{}, + }, + }, + }, nil + }, } ) @@ -895,14 +1026,330 @@ func TestFetcher_InvalidBlocks(t *testing.T) { require.NoError(t, f.FetchChainData(ctx)) // Make sure correct blocks were attempted to be saved - for blockIndex := 0; blockIndex < blockNum; blockIndex++ { - assert.Equal(t, blocks[blockIndex+1], savedBlocks[blockIndex]) + for blockIndex := 1; blockIndex < blockNum; blockIndex++ { + assert.Equal(t, blocks[blockIndex], savedBlocks[blockIndex]) } // Make sure no events were emitted assert.Len(t, capturedEvents, 0) } +func TestFetcher_Genesis(t *testing.T) { + t.Parallel() + + var ( + txCount = 21 + txs = generateTransactions(t, txCount) + savedBlocks = map[int64]*types.Block{} + savedTxs = map[string]*types.TxResult{} + + capturedEvents = make([]*indexerTypes.NewBlock, 0) + + mockEvents = &mockEvents{ + signalEventFn: func(e events.Event) { + blockEvent, ok := e.(*indexerTypes.NewBlock) + require.True(t, ok) + + capturedEvents = append(capturedEvents, blockEvent) + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + return &mock.WriteBatch{ + SetBlockFn: func(block *types.Block) error { + _, ok := savedBlocks[block.Height] + require.False(t, ok) + savedBlocks[block.Height] = block + + return nil + }, + SetTxFn: func(tx *types.TxResult) error { + savedTxs[fmt.Sprintf("%d-%d", tx.Height, tx.Index)] = tx + + return nil + }, + } + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + localTxs := make([]std.Tx, len(txs)) + for i, tx := range txs { + localTxs[i] = *tx + } + + return &core_types.ResultGenesis{Genesis: &types.GenesisDoc{AppState: gnoland.GnoGenesisState{ + Txs: localTxs, + }}}, nil + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + return &core_types.ResultBlockResults{ + Results: &state.ABCIResponses{ + DeliverTxs: make([]abci.ResponseDeliverTx, len(txs)), + }, + }, nil + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.NoError(t, f.fetchGenesisData()) + + require.Len(t, capturedEvents, 1) + + _, ok := savedBlocks[0] + require.True(t, ok) + + for i := uint32(0); i < uint32(len(txs)); i++ { + tx, ok := savedTxs[fmt.Sprintf("0-%d", i)] + require.True(t, ok) + + expected := &types.TxResult{ + Height: 0, + Index: i, + Tx: amino.MustMarshalJSON(txs[i]), + Response: abci.ResponseDeliverTx{}, + } + require.Equal(t, expected, tx) + } +} + +func TestFetcher_GenesisAlreadyFetched(t *testing.T) { + t.Parallel() + + var ( + mockEvents = &mockEvents{} + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, nil + }, + } + + mockClient = &mockClient{} + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.NoError(t, f.fetchGenesisData()) +} + +func TestFetcher_GenesisFetchError(t *testing.T) { + t.Parallel() + + var ( + remoteErr = errors.New("remote error") + + mockEvents = &mockEvents{ + signalEventFn: func(_ events.Event) { + require.Fail(t, "should not emit events") + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + require.Fail(t, "should not attempt to write to storage") + + return nil + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return nil, remoteErr + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + require.Fail(t, "should not attempt to fetch block results") + + return nil, nil + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.ErrorIs(t, f.fetchGenesisData(), remoteErr) +} + +func TestFetcher_GenesisInvalidState(t *testing.T) { + t.Parallel() + + var ( + mockEvents = &mockEvents{ + signalEventFn: func(_ events.Event) { + require.Fail(t, "should not emit events") + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + require.Fail(t, "should not attempt to write to storage") + + return nil + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{Genesis: &types.GenesisDoc{AppState: 0xdeadbeef}}, nil + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + require.Fail(t, "should not attempt to fetch block results") + + return nil, nil + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.ErrorContains(t, f.fetchGenesisData(), "unknown genesis state kind 'int'") +} + +func TestFetcher_GenesisFetchResultsError(t *testing.T) { + t.Parallel() + + var ( + remoteErr = errors.New("remote error") + + mockEvents = &mockEvents{ + signalEventFn: func(_ events.Event) { + require.Fail(t, "should not emit events") + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + require.Fail(t, "should not attempt to write to storage") + + return nil + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{Txs: []std.Tx{{}}}, + }}, nil + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + return nil, remoteErr + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.ErrorIs(t, f.fetchGenesisData(), remoteErr) +} + +func TestFetcher_GenesisNilGenesisDoc(t *testing.T) { + t.Parallel() + + var ( + mockEvents = &mockEvents{ + signalEventFn: func(_ events.Event) { + require.Fail(t, "should not emit events") + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + require.Fail(t, "should not attempt to write") + + return nil + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{Genesis: nil}, nil + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + return &core_types.ResultBlockResults{Results: &state.ABCIResponses{}}, nil + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.Error(t, f.fetchGenesisData()) +} + +func TestFetcher_GenesisNilResults(t *testing.T) { + t.Parallel() + + var ( + mockEvents = &mockEvents{ + signalEventFn: func(_ events.Event) { + require.Fail(t, "should not emit events") + }, + } + + mockStorage = &mock.Storage{ + GetLatestSavedHeightFn: func() (uint64, error) { + return 0, storageErrors.ErrNotFound + }, + GetWriteBatchFn: func() storage.Batch { + require.Fail(t, "should not attempt to write") + + return nil + }, + } + + mockClient = &mockClient{ + getLatestBlockNumberFn: func() (uint64, error) { + return 0, nil + }, + getGenesisFn: func() (*core_types.ResultGenesis, error) { + return &core_types.ResultGenesis{Genesis: &types.GenesisDoc{ + AppState: gnoland.GnoGenesisState{Txs: []std.Tx{{}}}, + }}, nil + }, + getBlockResultsFn: func(uint64) (*core_types.ResultBlockResults, error) { + return &core_types.ResultBlockResults{Results: nil}, nil + }, + } + ) + + f := New(mockStorage, mockClient, mockEvents) + + require.Error(t, f.fetchGenesisData()) +} + // generateTransactions generates dummy transactions func generateTransactions(t *testing.T, count int) []*std.Tx { t.Helper() diff --git a/fetch/mocks_test.go b/fetch/mocks_test.go index 7d11e576..bd76e85c 100644 --- a/fetch/mocks_test.go +++ b/fetch/mocks_test.go @@ -13,6 +13,7 @@ type ( getLatestBlockNumberDelegate func() (uint64, error) getBlockDelegate func(uint64) (*core_types.ResultBlock, error) getBlockResultsDelegate func(uint64) (*core_types.ResultBlockResults, error) + getGenesisDelegate func() (*core_types.ResultGenesis, error) createBatchDelegate func() clientTypes.Batch ) @@ -21,6 +22,7 @@ type mockClient struct { getLatestBlockNumberFn getLatestBlockNumberDelegate getBlockFn getBlockDelegate getBlockResultsFn getBlockResultsDelegate + getGenesisFn getGenesisDelegate createBatchFn createBatchDelegate } @@ -41,6 +43,14 @@ func (m *mockClient) GetBlock(blockNum uint64) (*core_types.ResultBlock, error) return nil, nil } +func (m *mockClient) GetGenesis() (*core_types.ResultGenesis, error) { + if m.getGenesisFn != nil { + return m.getGenesisFn() + } + + return nil, nil +} + func (m *mockClient) GetBlockResults(blockNum uint64) (*core_types.ResultBlockResults, error) { if m.getBlockResultsFn != nil { return m.getBlockResultsFn(blockNum) diff --git a/fetch/types.go b/fetch/types.go index cb75e844..edd19f91 100644 --- a/fetch/types.go +++ b/fetch/types.go @@ -15,6 +15,9 @@ type Client interface { // GetBlock returns specified block GetBlock(uint64) (*core_types.ResultBlock, error) + // GetGenesis returns the genesis block + GetGenesis() (*core_types.ResultGenesis, error) + // GetBlockResults returns the results of executing the transactions // for the specified block GetBlockResults(uint64) (*core_types.ResultBlockResults, error) diff --git a/go.mod b/go.mod index bfb29fd8..17531a33 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( ) require ( + dario.cat/mergo v1.0.0 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -51,17 +52,21 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rs/cors v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sosodev/duration v1.3.1 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/urfave/cli/v2 v2.27.2 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + go.etcd.io/bbolt v1.3.9 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.28.0 // indirect diff --git a/go.sum b/go.sum index b943273f..b0465394 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/cosmos/ledger-cosmos-go v0.13.3 h1:7ehuBGuyIytsXbd4MP43mLeoN2LTOEnk5nvue4rK+yM= +github.com/cosmos/ledger-cosmos-go v0.13.3/go.mod h1:HENcEP+VtahZFw38HZ3+LS3Iv5XV6svsnkk9vdJtLr8= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -81,6 +83,7 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= @@ -108,6 +111,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -148,15 +153,19 @@ github.com/madz-lab/insertion-queue v0.0.0-20230520191346-295d3348f63a/go.mod h1 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olahol/melody v1.2.1 h1:xdwRkzHxf+B0w4TKbGpUSSkV516ZucQZJIWLztOWICQ= github.com/olahol/melody v1.2.1/go.mod h1:GgkTl6Y7yWj/HtfD48Q5vLKPVoZOH+Qqgfa7CvJgJM4= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -180,6 +189,8 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= +github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -202,6 +213,10 @@ github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= +github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= +github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw= +github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= @@ -266,6 +281,8 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -280,6 +297,7 @@ golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= @@ -299,6 +317,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=