From ca8b38b8715bc7b06209648f1609fa9a8694adc0 Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:17:08 +0000 Subject: [PATCH 01/16] Darren/logdb remove leading zeros (#865) --- logdb/logdb.go | 17 ++++++- logdb/logdb_bench_test.go | 45 +++++++++--------- logdb/logdb_test.go | 97 ++++++++++++++++++++++++++------------- logdb/types.go | 2 +- 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index bcd793e94..b1979813f 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -398,11 +398,23 @@ func (db *LogDB) NewWriterSyncOff() *Writer { func topicValue(topics []thor.Bytes32, i int) []byte { if i < len(topics) { - return topics[i][:] + return removeLeadingZeros(topics[i][:]) } return nil } +func removeLeadingZeros(bytes []byte) []byte { + i := 0 + // increase i until it reaches the first non-zero byte + for ; i < len(bytes) && bytes[i] == 0; i++ { + } + // ensure at least 1 byte exists + if i == len(bytes) { + return []byte{0} + } + return bytes[i:] +} + // Writer is the transactional log writer. type Writer struct { conn *sql.Conn @@ -481,7 +493,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { topicValue(ev.Topics, 1), topicValue(ev.Topics, 2), topicValue(ev.Topics, 3), - topicValue(ev.Topics, 4)); err != nil { + topicValue(ev.Topics, 4), + ); err != nil { return err } diff --git a/logdb/logdb_bench_test.go b/logdb/logdb_bench_test.go index e421ffce3..9e667999b 100644 --- a/logdb/logdb_bench_test.go +++ b/logdb/logdb_bench_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package logdb_test +package logdb import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" ) @@ -39,7 +38,7 @@ func init() { flag.StringVar(&dbPath, "dbPath", "", "Path to the database file") } -// TestLogDB_NewestBlockID performs a series of read/write benchmarks on the NewestBlockID functionality of the LogDB. +// TestLogDB_NewestBlockID performs a series of read/write benchmarks on the NewestBlockID functionality of LogDB. // It benchmarks the creating, writing, committing a new block, followed by fetching this new block as the NewestBlockID func BenchmarkFakeDB_NewestBlockID(t *testing.B) { db, err := createTempDB() @@ -155,7 +154,7 @@ func BenchmarkTestDB_HasBlockID(b *testing.B) { defer db.Close() // find the first 500k blocks with events - events, err := db.FilterEvents(context.Background(), &logdb.EventFilter{Options: &logdb.Options{Offset: 0, Limit: 500_000}}) + events, err := db.FilterEvents(context.Background(), &EventFilter{Options: &Options{Offset: 0, Limit: 500_000}}) require.NoError(b, err) require.GreaterOrEqual(b, len(events), 500_000, "there should be more than 500k events in the db") @@ -178,12 +177,12 @@ func BenchmarkTestDB_FilterEvents(b *testing.B) { vthoAddress := thor.MustParseAddress(VTHO_ADDRESS) topic := thor.MustParseBytes32(VTHO_TOPIC) - addressFilterCriteria := []*logdb.EventCriteria{ + addressFilterCriteria := []*EventCriteria{ { Address: &vthoAddress, }, } - topicFilterCriteria := []*logdb.EventCriteria{ + topicFilterCriteria := []*EventCriteria{ { Topics: [5]*thor.Bytes32{&topic, nil, nil, nil, nil}, }, @@ -191,14 +190,14 @@ func BenchmarkTestDB_FilterEvents(b *testing.B) { tests := []struct { name string - arg *logdb.EventFilter + arg *EventFilter }{ - {"AddressCriteriaFilter", &logdb.EventFilter{CriteriaSet: addressFilterCriteria, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"TopicCriteriaFilter", &logdb.EventFilter{CriteriaSet: topicFilterCriteria, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventLimit", &logdb.EventFilter{Order: logdb.ASC, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventLimitDesc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventRange", &logdb.EventFilter{Range: &logdb.Range{From: 500000, To: 1_000_000}}}, - {"EventRangeDesc", &logdb.EventFilter{Range: &logdb.Range{From: 500000, To: 1_000_000}, Order: logdb.DESC}}, + {"AddressCriteriaFilter", &EventFilter{CriteriaSet: addressFilterCriteria, Options: &Options{Offset: 0, Limit: 500000}}}, + {"TopicCriteriaFilter", &EventFilter{CriteriaSet: topicFilterCriteria, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventLimit", &EventFilter{Order: ASC, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventLimitDesc", &EventFilter{Order: DESC, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventRange", &EventFilter{Range: &Range{From: 500000, To: 1_000_000}}}, + {"EventRangeDesc", &EventFilter{Range: &Range{From: 500000, To: 1_000_000}, Order: DESC}}, } for _, tt := range tests { @@ -222,7 +221,7 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { defer db.Close() txOrigin := thor.MustParseAddress(TEST_ADDRESS) - transferCriteria := []*logdb.TransferCriteria{ + transferCriteria := []*TransferCriteria{ { TxOrigin: &txOrigin, Sender: nil, @@ -232,12 +231,12 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { tests := []struct { name string - arg *logdb.TransferFilter + arg *TransferFilter }{ - {"TransferCriteria", &logdb.TransferFilter{CriteriaSet: transferCriteria, Options: &logdb.Options{Offset: 0, Limit: 500_000}}}, - {"TransferCriteriaDesc", &logdb.TransferFilter{Order: logdb.DESC, CriteriaSet: transferCriteria, Options: &logdb.Options{Offset: 0, Limit: 500_000}}}, - {"Ranged500K", &logdb.TransferFilter{Range: &logdb.Range{From: 500_000, To: 1_000_000}}}, - {"Ranged500KDesc", &logdb.TransferFilter{Range: &logdb.Range{From: 500_000, To: 1_000_000}, Order: logdb.DESC}}, + {"TransferCriteria", &TransferFilter{CriteriaSet: transferCriteria, Options: &Options{Offset: 0, Limit: 500_000}}}, + {"TransferCriteriaDesc", &TransferFilter{Order: DESC, CriteriaSet: transferCriteria, Options: &Options{Offset: 0, Limit: 500_000}}}, + {"Ranged500K", &TransferFilter{Range: &Range{From: 500_000, To: 1_000_000}}}, + {"Ranged500KDesc", &TransferFilter{Range: &Range{From: 500_000, To: 1_000_000}, Order: DESC}}, } for _, tt := range tests { @@ -253,7 +252,7 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { } } -func createTempDB() (*logdb.LogDB, error) { +func createTempDB() (*LogDB, error) { dir, err := os.MkdirTemp("", "tempdir-") if err != nil { return nil, fmt.Errorf("failed to create temp directory: %w", err) @@ -268,7 +267,7 @@ func createTempDB() (*logdb.LogDB, error) { return nil, fmt.Errorf("failed to close temp file: %w", err) } - db, err := logdb.New(tmpFile.Name()) + db, err := New(tmpFile.Name()) if err != nil { return nil, fmt.Errorf("unable to load logdb: %w", err) } @@ -276,10 +275,10 @@ func createTempDB() (*logdb.LogDB, error) { return db, nil } -func loadDBFromDisk(b *testing.B) (*logdb.LogDB, error) { +func loadDBFromDisk(b *testing.B) (*LogDB, error) { if dbPath == "" { b.Fatal("Please provide a dbPath") } - return logdb.New(dbPath) + return New(dbPath) } diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index 7ffdd59b1..fc7c6af56 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package logdb_test +package logdb import ( "context" @@ -11,10 +11,10 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/block" - logdb "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" ) @@ -84,9 +84,9 @@ func newTransferOnlyReceipt() *tx.Receipt { } } -type eventLogs []*logdb.Event +type eventLogs []*Event -func (logs eventLogs) Filter(f func(ev *logdb.Event) bool) (ret eventLogs) { +func (logs eventLogs) Filter(f func(ev *Event) bool) (ret eventLogs) { for _, ev := range logs { if f(ev) { ret = append(ret, ev) @@ -102,9 +102,9 @@ func (logs eventLogs) Reverse() (ret eventLogs) { return } -type transferLogs []*logdb.Transfer +type transferLogs []*Transfer -func (logs transferLogs) Filter(f func(tr *logdb.Transfer) bool) (ret transferLogs) { +func (logs transferLogs) Filter(f func(tr *Transfer) bool) (ret transferLogs) { for _, tr := range logs { if f(tr) { ret = append(ret, tr) @@ -121,7 +121,7 @@ func (logs transferLogs) Reverse() (ret transferLogs) { } func TestEvents(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -144,7 +144,7 @@ func TestEvents(t *testing.T) { tx := b.Transactions()[j] receipt := receipts[j] origin, _ := tx.Origin() - allEvents = append(allEvents, &logdb.Event{ + allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), Index: uint32(j), BlockID: b.Header().ID(), @@ -157,7 +157,7 @@ func TestEvents(t *testing.T) { Data: receipt.Outputs[0].Events[0].Data, }) - allTransfers = append(allTransfers, &logdb.Transfer{ + allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), Index: uint32(j), BlockID: b.Header().ID(), @@ -184,21 +184,21 @@ func TestEvents(t *testing.T) { { tests := []struct { name string - arg *logdb.EventFilter + arg *EventFilter want eventLogs }{ - {"query all events", &logdb.EventFilter{}, allEvents}, + {"query all events", &EventFilter{}, allEvents}, {"query all events with nil option", nil, allEvents}, - {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents}, - {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()}, - {"query all events limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, - {"query all events range", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, - {"query events with range and desc", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, - {"query events with limit with desc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allEvents.Reverse()[0:10]}, - {"query all events with criteria", &logdb.EventFilter{CriteriaSet: []*logdb.EventCriteria{{Address: &allEvents[1].Address}}}, allEvents.Filter(func(ev *logdb.Event) bool { + {"query all events asc", &EventFilter{Order: ASC}, allEvents}, + {"query all events desc", &EventFilter{Order: DESC}, allEvents.Reverse()}, + {"query all events limit offset", &EventFilter{Options: &Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, + {"query all events range", &EventFilter{Range: &Range{From: 10, To: 20}}, allEvents.Filter(func(ev *Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, + {"query events with range and desc", &EventFilter{Range: &Range{From: 10, To: 20}, Order: DESC}, allEvents.Filter(func(ev *Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, + {"query events with limit with desc", &EventFilter{Order: DESC, Options: &Options{Limit: 10}}, allEvents.Reverse()[0:10]}, + {"query all events with criteria", &EventFilter{CriteriaSet: []*EventCriteria{{Address: &allEvents[1].Address}}}, allEvents.Filter(func(ev *Event) bool { return ev.Address == allEvents[1].Address })}, - {"query all events with multi-criteria", &logdb.EventFilter{CriteriaSet: []*logdb.EventCriteria{{Address: &allEvents[1].Address}, {Topics: [5]*thor.Bytes32{allEvents[2].Topics[0]}}, {Topics: [5]*thor.Bytes32{allEvents[3].Topics[0]}}}}, allEvents.Filter(func(ev *logdb.Event) bool { + {"query all events with multi-criteria", &EventFilter{CriteriaSet: []*EventCriteria{{Address: &allEvents[1].Address}, {Topics: [5]*thor.Bytes32{allEvents[2].Topics[0]}}, {Topics: [5]*thor.Bytes32{allEvents[3].Topics[0]}}}}, allEvents.Filter(func(ev *Event) bool { return ev.Address == allEvents[1].Address || *ev.Topics[0] == *allEvents[2].Topics[0] || *ev.Topics[0] == *allEvents[3].Topics[0] })}, } @@ -215,21 +215,21 @@ func TestEvents(t *testing.T) { { tests := []struct { name string - arg *logdb.TransferFilter + arg *TransferFilter want transferLogs }{ - {"query all transfers", &logdb.TransferFilter{}, allTransfers}, + {"query all transfers", &TransferFilter{}, allTransfers}, {"query all transfers with nil option", nil, allTransfers}, - {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers}, - {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()}, - {"query all transfers limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, - {"query all transfers range", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, - {"query transfers with range and desc", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, - {"query transfers with limit with desc", &logdb.TransferFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allTransfers.Reverse()[0:10]}, - {"query all transfers with criteria", &logdb.TransferFilter{CriteriaSet: []*logdb.TransferCriteria{{Sender: &allTransfers[1].Sender}}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { + {"query all transfers asc", &TransferFilter{Order: ASC}, allTransfers}, + {"query all transfers desc", &TransferFilter{Order: DESC}, allTransfers.Reverse()}, + {"query all transfers limit offset", &TransferFilter{Options: &Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, + {"query all transfers range", &TransferFilter{Range: &Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, + {"query transfers with range and desc", &TransferFilter{Range: &Range{From: 10, To: 20}, Order: DESC}, allTransfers.Filter(func(tr *Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, + {"query transfers with limit with desc", &TransferFilter{Order: DESC, Options: &Options{Limit: 10}}, allTransfers.Reverse()[0:10]}, + {"query all transfers with criteria", &TransferFilter{CriteriaSet: []*TransferCriteria{{Sender: &allTransfers[1].Sender}}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.Sender == allTransfers[1].Sender })}, - {"query all transfers with multi-criteria", &logdb.TransferFilter{CriteriaSet: []*logdb.TransferCriteria{{Sender: &allTransfers[1].Sender}, {Recipient: &allTransfers[2].Recipient}}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { + {"query all transfers with multi-criteria", &TransferFilter{CriteriaSet: []*TransferCriteria{{Sender: &allTransfers[1].Sender}, {Recipient: &allTransfers[2].Recipient}}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.Sender == allTransfers[1].Sender || tr.Recipient == allTransfers[2].Recipient })}, } @@ -244,10 +244,10 @@ func TestEvents(t *testing.T) { } } -// TestLogDB_NewestBlockID performs a series of read/write tests on the NewestBlockID functionality of the LogDB. +// TestLogDB_NewestBlockID performs a series of read/write tests on the NewestBlockID functionality of the // It validates the correctness of the NewestBlockID method under various scenarios. func TestLogDB_NewestBlockID(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -368,9 +368,9 @@ func TestLogDB_NewestBlockID(t *testing.T) { } } -// TestLogDB_HasBlockID performs a series of tests on the HasBlockID functionality of the LogDB. +// TestLogDB_HasBlockID performs a series of tests on the HasBlockID functionality of the func TestLogDB_HasBlockID(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -431,3 +431,34 @@ func TestLogDB_HasBlockID(t *testing.T) { } assert.True(t, has) } + +func TestRemoveLeadingZeros(t *testing.T) { + tests := []struct { + name string + input []byte + expected []byte + }{ + { + "should remove leading zeros", + common.Hex2Bytes("0000000000000000000000006d95e6dca01d109882fe1726a2fb9865fa41e7aa"), + common.Hex2Bytes("6d95e6dca01d109882fe1726a2fb9865fa41e7aa"), + }, + { + "should not remove any bytes", + common.Hex2Bytes("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + common.Hex2Bytes("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + }, + { + "should have at least 1 byte", + common.Hex2Bytes("00000000000000000"), + []byte{0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := removeLeadingZeros(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/logdb/types.go b/logdb/types.go index e4ebb1be4..7aa5ce990 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -71,7 +71,7 @@ func (c *EventCriteria) toWhereCondition() (cond string, args []interface{}) { for i, topic := range c.Topics { if topic != nil { cond += fmt.Sprintf(" AND topic%v = ", i) + refIDQuery - args = append(args, topic.Bytes()) + args = append(args, removeLeadingZeros(topic.Bytes())) } } return From f9173e422ee102c3306457d25098c0a83f7837fa Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Tue, 24 Sep 2024 09:57:36 +0200 Subject: [PATCH 02/16] feat: add new txIndex column to event meta response --- api/events/types.go | 4 ++++ logdb/logdb.go | 18 +++++++++++++++--- logdb/logdb_test.go | 1 + logdb/schema.go | 1 + logdb/types.go | 1 + 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 0dce06aa4..8cd03590e 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -21,6 +21,7 @@ type LogMeta struct { BlockNumber uint32 `json:"blockNumber"` BlockTimestamp uint64 `json:"blockTimestamp"` TxID thor.Bytes32 `json:"txID"` + TxIndex uint32 `json:"txIndex"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` } @@ -51,6 +52,7 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockNumber: event.BlockNumber, BlockTimestamp: event.BlockTime, TxID: event.TxID, + TxIndex: event.TxIndex, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, @@ -74,6 +76,7 @@ func (e *FilteredEvent) String() string { blockNumber %v, blockTimestamp %v), txID %v, + txIndex %v, txOrigin %v, clauseIndex %v) )`, @@ -84,6 +87,7 @@ func (e *FilteredEvent) String() string { e.Meta.BlockNumber, e.Meta.BlockTimestamp, e.Meta.TxID, + e.Meta.TxIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, ) diff --git a/logdb/logdb.go b/logdb/logdb.go index b1979813f..67f6f5c90 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -95,7 +95,7 @@ func (db *LogDB) Path() string { } func (db *LogDB) FilterEvents(ctx context.Context, filter *EventFilter) ([]*Event, error) { - const query = `SELECT e.seq, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data + const query = `SELECT e.seq, e.txIndex, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data FROM (%v) e LEFT JOIN ref r0 ON e.blockID = r0.id LEFT JOIN ref r1 ON e.txID = r1.id @@ -244,6 +244,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } var ( seq sequence + txIndex uint32 blockID []byte blockTime uint64 txID []byte @@ -255,6 +256,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac ) if err := rows.Scan( &seq, + &txIndex, &blockID, &blockTime, &txID, @@ -276,6 +278,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: txIndex, TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Address: thor.BytesToAddress(address), @@ -455,6 +458,11 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) + indexes := make(map[thor.Bytes32]int, len(txs)) + for i, tx := range txs { + indexes[tx.ID()] = i + } + for i, r := range receipts { if isReceiptEmpty(r) { continue @@ -478,6 +486,9 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { txID = tx.ID() txOrigin, _ = tx.Origin() } + + txIndex := indexes[txID] + if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", txID[:], txOrigin[:]); err != nil { @@ -498,8 +509,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { return err } - const query = "INSERT OR IGNORE INTO event(seq, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + - "VALUES(?,?,?,?," + + const query = "INSERT OR IGNORE INTO event(seq, txIndex, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + + "VALUES(?,?,?,?,?," + refIDQuery + "," + refIDQuery + "," + refIDQuery + "," + @@ -518,6 +529,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, newSequence(blockNum, eventCount), + txIndex, blockTimestamp, clauseIndex, eventData, diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index fc7c6af56..aa1cb8df4 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -147,6 +147,7 @@ func TestEvents(t *testing.T) { allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), Index: uint32(j), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), diff --git a/logdb/schema.go b/logdb/schema.go index dccb33d35..1c60513e8 100644 --- a/logdb/schema.go +++ b/logdb/schema.go @@ -14,6 +14,7 @@ const ( // creates events table eventTableSchema = `CREATE TABLE IF NOT EXISTS event ( seq INTEGER PRIMARY KEY NOT NULL, + txIndex INTEGER NOT NULL, blockID INTEGER NOT NULL, blockTime INTEGER NOT NULL, txID INTEGER NOT NULL, diff --git a/logdb/types.go b/logdb/types.go index 7aa5ce990..697385d03 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -19,6 +19,7 @@ type Event struct { BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address //contract caller ClauseIndex uint32 Address thor.Address // always a contract address From f3bd272785a13f92c6e57967683badb29091b728 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Wed, 25 Sep 2024 09:57:16 +0200 Subject: [PATCH 03/16] test: add convert event test --- api/events/types.go | 4 +++ api/events/types_test.go | 71 ++++++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 8cd03590e..841c7c78b 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -22,6 +22,7 @@ type LogMeta struct { BlockTimestamp uint64 `json:"blockTimestamp"` TxID thor.Bytes32 `json:"txID"` TxIndex uint32 `json:"txIndex"` + LogIndex uint32 `json:"logIndex"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` } @@ -53,6 +54,7 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockTimestamp: event.BlockTime, TxID: event.TxID, TxIndex: event.TxIndex, + LogIndex: event.Index, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, @@ -77,6 +79,7 @@ func (e *FilteredEvent) String() string { blockTimestamp %v), txID %v, txIndex %v, + logIndex %v, txOrigin %v, clauseIndex %v) )`, @@ -88,6 +91,7 @@ func (e *FilteredEvent) String() string { e.Meta.BlockTimestamp, e.Meta.TxID, e.Meta.TxIndex, + e.Meta.LogIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, ) diff --git a/api/events/types_test.go b/api/events/types_test.go index a02f441c5..ec418b7b7 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -3,19 +3,20 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package events_test +package events import ( "math" "testing" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/stretchr/testify/assert" - "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" ) func TestEventsTypes(t *testing.T) { @@ -33,13 +34,13 @@ func TestEventsTypes(t *testing.T) { } func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.BlockRangeType, + rng := &Range{ + Unit: BlockRangeType, From: 1, To: 2, } - convertedRng, err := events.ConvertRange(chain, rng) + convertedRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, uint32(rng.From), convertedRng.From) @@ -47,8 +48,8 @@ func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { } func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: 2, } @@ -57,7 +58,7 @@ func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -68,8 +69,8 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: genesis.Timestamp(), } @@ -78,7 +79,7 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { To: 0, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedZeroRange, convRng) @@ -89,8 +90,8 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: genesis.Timestamp() + 1_000, To: genesis.Timestamp() + 10_000, } @@ -99,7 +100,7 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -123,3 +124,45 @@ func initChain(t *testing.T) *chain.Chain { return repo.NewBestChain() } + +func TestConvertEvent(t *testing.T) { + event := &logdb.Event{ + Address: thor.Address{0x01}, + Data: []byte{0x02, 0x03}, + BlockID: thor.Bytes32{0x04}, + BlockNumber: 5, + BlockTime: 6, + TxID: thor.Bytes32{0x07}, + TxIndex: 8, + Index: 9, + TxOrigin: thor.Address{0x0A}, + ClauseIndex: 10, + Topics: [5]*thor.Bytes32{ + {0x0B}, + {0x0C}, + nil, + nil, + nil, + }, + } + + expectedTopics := []*thor.Bytes32{ + {0x0B}, + {0x0C}, + } + expectedData := hexutil.Encode(event.Data) + + result := convertEvent(event) + + assert.Equal(t, event.Address, result.Address) + assert.Equal(t, expectedData, result.Data) + assert.Equal(t, event.BlockID, result.Meta.BlockID) + assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) + assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) + assert.Equal(t, event.TxID, result.Meta.TxID) + assert.Equal(t, event.TxIndex, result.Meta.TxIndex) + assert.Equal(t, event.Index, result.Meta.LogIndex) + assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) + assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) + assert.Equal(t, expectedTopics, result.Topics) +} From c5b3c632a68c1ca5d76be1fda4aaa4cc29de7ab3 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 26 Sep 2024 09:33:43 +0200 Subject: [PATCH 04/16] feat: make txLog and txIndex as optional return params --- api/events/events.go | 2 +- api/events/events_test.go | 73 +++++++++++++++++++++++++++++++++ api/events/types.go | 85 +++++++++++++++++++++++++++++---------- api/events/types_test.go | 41 +++++++++++++++++-- 4 files changed, 175 insertions(+), 26 deletions(-) diff --git a/api/events/events.go b/api/events/events.go index 40dff7b09..62bdec355 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e) + fes[i] = convertEvent(e, ef.OptionalData) } return fes, nil } diff --git a/api/events/events_test.go b/api/events/events_test.go index b1268d378..513745d58 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -56,6 +56,79 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } +func TestOptionalData(t *testing.T) { + db := createDb(t) + initEventServer(t, db, defaultLogLimit) + defer ts.Close() + insertBlocks(t, db, 5) + + testCases := []struct { + name string + optData *events.EventOptionalData + expected *events.LogOptionalData + }{ + { + name: "empty optional data", + optData: &events.EventOptionalData{}, + expected: nil, + }, + { + name: "optional data with txIndex", + optData: &events.EventOptionalData{ + TxIndex: true, + }, + expected: &events.LogOptionalData{ + TxIndex: new(uint32), + }, + }, + { + name: "optional data with logIndex", + optData: &events.EventOptionalData{ + LogIndex: true, + }, + expected: &events.LogOptionalData{ + LogIndex: new(uint32), + }, + }, + { + name: "optional data with txIndex and logIndex", + optData: &events.EventOptionalData{ + TxIndex: true, + LogIndex: true, + }, + expected: &events.LogOptionalData{ + TxIndex: new(uint32), + LogIndex: new(uint32), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := events.EventFilter{ + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + OptionalData: tc.optData, + } + + res, statusCode := httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.OptionalData) + } + }) + } +} + func TestOption(t *testing.T) { thorChain := initEventServer(t, 5) defer ts.Close() diff --git a/api/events/types.go b/api/events/types.go index 841c7c78b..a80ef81d4 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,14 +17,33 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxIndex uint32 `json:"txIndex"` - LogIndex uint32 `json:"logIndex"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + OptionalData *LogOptionalData `json:"optionalData,omitempty"` +} + +type LogOptionalData struct { + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` +} + +func (opt *LogOptionalData) Empty() bool { + return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) +} + +func (opt *LogOptionalData) String() string { + var parts []string + if opt.TxIndex != nil { + parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) + } + if opt.LogIndex != nil { + parts = append(parts, fmt.Sprintf("logIndex: %v", *opt.LogIndex)) + } + return fmt.Sprintf("%v", parts) } type TopicSet struct { @@ -44,8 +63,8 @@ type FilteredEvent struct { } // convert a logdb.Event into a json format Event -func convertEvent(event *logdb.Event) *FilteredEvent { - fe := FilteredEvent{ +func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { + fe := &FilteredEvent{ Address: event.Address, Data: hexutil.Encode(event.Data), Meta: LogMeta{ @@ -53,19 +72,37 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockNumber: event.BlockNumber, BlockTimestamp: event.BlockTime, TxID: event.TxID, - TxIndex: event.TxIndex, - LogIndex: event.Index, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, } + fe = addOptionalData(fe, event, eventOptionalData) + fe.Topics = make([]*thor.Bytes32, 0) for i := 0; i < 5; i++ { if event.Topics[i] != nil { fe.Topics = append(fe.Topics, event.Topics[i]) } } - return &fe + return fe +} + +func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { + if eventOptionalData != nil { + opt := &LogOptionalData{} + + if eventOptionalData.LogIndex { + opt.LogIndex = &event.Index + } + if eventOptionalData.TxIndex { + opt.TxIndex = &event.TxIndex + } + + if !opt.Empty() { + fe.Meta.OptionalData = opt + } + } + return fe } func (e *FilteredEvent) String() string { @@ -78,10 +115,9 @@ func (e *FilteredEvent) String() string { blockNumber %v, blockTimestamp %v), txID %v, - txIndex %v, - logIndex %v, txOrigin %v, - clauseIndex %v) + clauseIndex %v, + optionalData (%v)) )`, e.Address, e.Topics, @@ -90,10 +126,9 @@ func (e *FilteredEvent) String() string { e.Meta.BlockNumber, e.Meta.BlockTimestamp, e.Meta.TxID, - e.Meta.TxIndex, - e.Meta.LogIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, + e.Meta.OptionalData, ) } @@ -103,10 +138,16 @@ type EventCriteria struct { } type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` + CriteriaSet []*EventCriteria `json:"criteriaSet"` + Range *Range `json:"range"` + Options *logdb.Options `json:"options"` + Order logdb.Order `json:"order"` + OptionalData *EventOptionalData `json:"optionalData,omitempty"` +} + +type EventOptionalData struct { + LogIndex bool `json:"logIndex,omitempty"` + TxIndex bool `json:"txIndex,omitempty"` } func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { diff --git a/api/events/types_test.go b/api/events/types_test.go index ec418b7b7..850e16f06 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -145,6 +145,10 @@ func TestConvertEvent(t *testing.T) { nil, }, } + eventOptData := &EventOptionalData{ + LogIndex: true, + TxIndex: true, + } expectedTopics := []*thor.Bytes32{ {0x0B}, @@ -152,7 +156,7 @@ func TestConvertEvent(t *testing.T) { } expectedData := hexutil.Encode(event.Data) - result := convertEvent(event) + result := convertEvent(event, eventOptData) assert.Equal(t, event.Address, result.Address) assert.Equal(t, expectedData, result.Data) @@ -160,9 +164,40 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, result.Meta.TxIndex) - assert.Equal(t, event.Index, result.Meta.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.OptionalData.TxIndex) + assert.Equal(t, event.Index, *result.Meta.OptionalData.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) } + +func TestIsEmpty(t *testing.T) { + // Empty cases + var nilCase *LogOptionalData + assert.True(t, nilCase.Empty()) + + emptyCase := &LogOptionalData{} + assert.True(t, emptyCase.Empty()) + + emptyCase = &LogOptionalData{ + LogIndex: nil, + } + assert.True(t, emptyCase.Empty()) + + emptyCase = &LogOptionalData{ + TxIndex: nil, + } + assert.True(t, emptyCase.Empty()) + + // Not empty cases + val := uint32(1) + notEmptyCase := &LogOptionalData{ + LogIndex: &val, + } + assert.False(t, notEmptyCase.Empty()) + + notEmptyCase = &LogOptionalData{ + TxIndex: &val, + } + assert.False(t, notEmptyCase.Empty()) +} From 76a38d90eca98fe7a1087125acf16fa4e5853433 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 26 Sep 2024 10:27:56 +0200 Subject: [PATCH 05/16] chore: update swagger with new event optional data --- api/doc/thor.yaml | 84 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 732a5f1a3..9ba9735bc 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1009,6 +1009,8 @@ components: enum: - asc - desc + optionalData: + $ref: '#/components/schemas/EventOptionalData' EventLogsResponse: type: array @@ -1020,7 +1022,7 @@ components: - $ref: '#/components/schemas/Event' - properties: meta: - $ref: '#/components/schemas/LogMeta' + $ref: '#/components/schemas/EventLogMeta' TransferLogFilterRequest: type: object @@ -1325,6 +1327,66 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false + + EventLogMeta: + title: EventLogMeta + type: object + description: The event or transfer log metadata such as block number, block timestamp, etc. + properties: + blockID: + type: string + format: hex + description: The block identifier in which the log was included. + example: '0x0004f6cc88bb4626a92907718e82f255b8fa511453a78e8797eb8cea3393b215' + nullable: false + pattern: '^0x[0-9a-f]{64}$' + blockNumber: + type: integer + format: uint32 + description: The block number (height) of the block in which the log was included. + example: 325324 + nullable: false + blockTimestamp: + type: integer + format: uint64 + description: The UNIX timestamp of the block in which the log was included. + example: 1533267900 + nullable: false + txID: + type: string + format: hex + description: The transaction identifier, from which the log was generated. + example: '0x284bba50ef777889ff1a367ed0b38d5e5626714477c40de38d71cedd6f9fa477' + nullable: false + pattern: '^0x[0-9a-f]{64}$' + txOrigin: + type: string + description: The account from which the transaction was sent. + example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d' + nullable: false + pattern: '^0x[0-9a-f]{40}$' + clauseIndex: + type: integer + format: uint32 + description: The index of the clause in the transaction, from which the log was generated. + example: 0 + nullable: false + optionalData: + $ref: '#/components/schemas/LogOptionalData' + + LogOptionalData: + title: optionalData + type: object + nullable: true + properties: + txIndex: + type: integer + nullable: true + example: 1 + logIndex: + type: integer + nullable: true + example: 1 Block: title: Block @@ -1916,6 +1978,26 @@ components: } ``` This refers to the range from block 10 to block 1000. + + EventOptionalData: + nullable: true + type: object + title: EventOptionalData + properties: + txIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event transaction index. + loglIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event log index. + description: | + Specifies all the optional data that can be included in the response. EventCriteria: type: object From ad6204e451fd5c010e19e10470f717eb7c65952a Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Mon, 7 Oct 2024 10:22:26 +0200 Subject: [PATCH 06/16] feat: save logIndex in sequence --- logdb/logdb.go | 32 ++++++++++++++------------------ logdb/schema.go | 1 - logdb/sequence.go | 41 ++++++++++++++++++++++++++++++++--------- logdb/sequence_test.go | 36 +++++++++++++++++++++++------------- 4 files changed, 69 insertions(+), 41 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index 67f6f5c90..457a34395 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -9,7 +9,6 @@ import ( "context" "database/sql" "fmt" - "math" "math/big" sqlite3 "github.com/mattn/go-sqlite3" @@ -95,7 +94,7 @@ func (db *LogDB) Path() string { } func (db *LogDB) FilterEvents(ctx context.Context, filter *EventFilter) ([]*Event, error) { - const query = `SELECT e.seq, e.txIndex, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data + const query = `SELECT e.seq, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data FROM (%v) e LEFT JOIN ref r0 ON e.blockID = r0.id LEFT JOIN ref r1 ON e.txID = r1.id @@ -118,10 +117,10 @@ FROM (%v) e if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -184,10 +183,10 @@ FROM (%v) t if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -244,7 +243,6 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } var ( seq sequence - txIndex uint32 blockID []byte blockTime uint64 txID []byte @@ -256,7 +254,6 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac ) if err := rows.Scan( &seq, - &txIndex, &blockID, &blockTime, &txID, @@ -274,11 +271,11 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } event := &Event{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + Index: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), - TxIndex: txIndex, + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Address: thor.BytesToAddress(address), @@ -337,7 +334,7 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter } trans := &Transfer{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + Index: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), @@ -379,7 +376,7 @@ func (db *LogDB) HasBlockID(id thor.Bytes32) (bool, error) { UNION SELECT * FROM (SELECT seq FROM event WHERE seq=? AND blockID=` + refIDQuery + ` LIMIT 1))` - seq := newSequence(block.Number(id), 0) + seq := newSequence(block.Number(id), 0, 0) row := db.stmtCache.MustPrepare(query).QueryRow(seq, id[:], seq, id[:]) var count int if err := row.Scan(&count); err != nil { @@ -429,7 +426,7 @@ type Writer struct { // Truncate truncates the database by deleting logs after blockNum (included). func (w *Writer) Truncate(blockNum uint32) error { - seq := newSequence(blockNum, 0) + seq := newSequence(blockNum, 0, 0) if err := w.exec("DELETE FROM event WHERE seq >= ?", seq); err != nil { return err } @@ -509,8 +506,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { return err } - const query = "INSERT OR IGNORE INTO event(seq, txIndex, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + - "VALUES(?,?,?,?,?," + + const query = "INSERT OR IGNORE INTO event(seq, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + + "VALUES(?,?,?,?," + refIDQuery + "," + refIDQuery + "," + refIDQuery + "," + @@ -528,8 +525,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, eventCount), - txIndex, + newSequence(blockNum, uint32(txIndex), eventCount), blockTimestamp, clauseIndex, eventData, @@ -564,7 +560,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, transferCount), + newSequence(blockNum, uint32(txIndex), transferCount), blockTimestamp, clauseIndex, tr.Amount.Bytes(), diff --git a/logdb/schema.go b/logdb/schema.go index 1c60513e8..dccb33d35 100644 --- a/logdb/schema.go +++ b/logdb/schema.go @@ -14,7 +14,6 @@ const ( // creates events table eventTableSchema = `CREATE TABLE IF NOT EXISTS event ( seq INTEGER PRIMARY KEY NOT NULL, - txIndex INTEGER NOT NULL, blockID INTEGER NOT NULL, blockTime INTEGER NOT NULL, txID INTEGER NOT NULL, diff --git a/logdb/sequence.go b/logdb/sequence.go index 52909ffe4..9b5c29f0c 100644 --- a/logdb/sequence.go +++ b/logdb/sequence.go @@ -5,21 +5,44 @@ package logdb -import "math" - type sequence int64 -func newSequence(blockNum uint32, index uint32) sequence { - if (index & math.MaxInt32) != index { - panic("index too large") +// Adjust these constants based on your bit allocation requirements +const ( + blockNumBits = 31 + txIndexBits = 12 + logIndexBits = 21 + // Max = 2^31 - 1 = 2,147,483,647 + blockNumMask = (1 << blockNumBits) - 1 + // Max = 2^12 - 1 = 4,095 + txIndexMask = (1 << txIndexBits) - 1 + // Max = 2^21 - 1 = 2,097,151 + logIndexMask = (1 << logIndexBits) - 1 +) + +func newSequence(blockNum uint32, txIndex uint32, logIndex uint32) sequence { + if blockNum > blockNumMask { + panic("block number too large") + } + if txIndex > txIndexMask { + panic("transaction index too large") } - return (sequence(blockNum) << 31) | sequence(index) + if logIndex > logIndexMask { + panic("log index too large") + } + return (sequence(blockNum) << (txIndexBits + logIndexBits)) | + (sequence(txIndex) << logIndexBits) | + sequence(logIndex) } func (s sequence) BlockNumber() uint32 { - return uint32(s >> 31) + return uint32(s>>(txIndexBits+logIndexBits)) & blockNumMask +} + +func (s sequence) TxIndex() uint32 { + return uint32((s >> logIndexBits) & txIndexMask) } -func (s sequence) Index() uint32 { - return uint32(s & math.MaxInt32) +func (s sequence) LogIndex() uint32 { + return uint32(s & logIndexMask) } diff --git a/logdb/sequence_test.go b/logdb/sequence_test.go index 9fa19fff0..b16e2d0da 100644 --- a/logdb/sequence_test.go +++ b/logdb/sequence_test.go @@ -6,33 +6,36 @@ package logdb import ( - "math" "testing" ) func TestSequence(t *testing.T) { type args struct { blockNum uint32 - index uint32 + txIndex uint32 + logIndex uint32 } tests := []struct { name string args args - want args }{ - {"regular", args{1, 2}, args{1, 2}}, - {"max bn", args{math.MaxUint32, 1}, args{math.MaxUint32, 1}}, - {"max index", args{5, math.MaxInt32}, args{5, math.MaxInt32}}, - {"both max", args{math.MaxUint32, math.MaxInt32}, args{math.MaxUint32, math.MaxInt32}}, + {"regular", args{1, 2, 3}}, + {"max bn", args{blockNumMask, 1, 2}}, + {"max tx index", args{5, txIndexMask, 4}}, + {"max log index", args{5, 4, logIndexMask}}, + {"both max", args{blockNumMask, txIndexMask, logIndexMask}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := newSequence(tt.args.blockNum, tt.args.index) - if bn := got.BlockNumber(); bn != tt.want.blockNum { - t.Errorf("seq.blockNum() = %v, want %v", bn, tt.want.blockNum) + got := newSequence(tt.args.blockNum, tt.args.txIndex, tt.args.logIndex) + if bn := got.BlockNumber(); bn != tt.args.blockNum { + t.Errorf("seq.blockNum() = %v, want %v", bn, tt.args.blockNum) } - if i := got.Index(); i != tt.want.index { - t.Errorf("seq.index() = %v, want %v", i, tt.want.index) + if ti := got.TxIndex(); ti != tt.args.txIndex { + t.Errorf("seq.txIndex() = %v, want %v", ti, tt.args.txIndex) + } + if i := got.LogIndex(); i != tt.args.logIndex { + t.Errorf("seq.index() = %v, want %v", i, tt.args.logIndex) } }) } @@ -42,5 +45,12 @@ func TestSequence(t *testing.T) { t.Errorf("newSequence should panic on 2nd arg > math.MaxInt32") } }() - newSequence(1, math.MaxInt32+1) + newSequence(1, txIndexMask+1, 5) + + defer func() { + if e := recover(); e == nil { + t.Errorf("newSequence should panic on 3rd arg > math.MaxInt32") + } + }() + newSequence(1, 5, logIndexMask+1) } From 1415b40cb62a8a04e66aef9cd935121a21e60d93 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Fri, 11 Oct 2024 09:49:17 +0200 Subject: [PATCH 07/16] feat: tweaked bits in sequence --- logdb/sequence.go | 8 ++++---- thor/params.go | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/logdb/sequence.go b/logdb/sequence.go index 9b5c29f0c..b76ad4821 100644 --- a/logdb/sequence.go +++ b/logdb/sequence.go @@ -9,12 +9,12 @@ type sequence int64 // Adjust these constants based on your bit allocation requirements const ( - blockNumBits = 31 - txIndexBits = 12 + blockNumBits = 28 + txIndexBits = 15 logIndexBits = 21 - // Max = 2^31 - 1 = 2,147,483,647 + // Max = 2^28 - 1 = 268,435,455 blockNumMask = (1 << blockNumBits) - 1 - // Max = 2^12 - 1 = 4,095 + // Max = 2^15 - 1 = 32,767 txIndexMask = (1 << txIndexBits) - 1 // Max = 2^21 - 1 = 2,097,151 logIndexMask = (1 << logIndexBits) - 1 diff --git a/thor/params.go b/thor/params.go index 5912c46c9..6750f2577 100644 --- a/thor/params.go +++ b/thor/params.go @@ -12,6 +12,11 @@ import ( "github.com/ethereum/go-ethereum/params" ) +/* + NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in sequence.go: + - an increase in gas limit may require more bits for txIndex; + - if block frequency is increased, blockNumber will increment faster, potentially exhausting the allocated bits sooner than expected. +*/ // Constants of block chain. const ( BlockInterval uint64 = 10 // time interval between two consecutive blocks. From 94f4070428729dce7fd3e02413f7c9a038e1c3f5 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Fri, 11 Oct 2024 10:03:15 +0200 Subject: [PATCH 08/16] refactor: rename optional log meta field --- api/doc/thor.yaml | 8 ++++---- api/events/events_test.go | 10 +++++----- api/events/types.go | 26 +++++++++++++------------- api/events/types_test.go | 16 ++++++++-------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 9ba9735bc..66ff3eba7 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1371,11 +1371,11 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false - optionalData: - $ref: '#/components/schemas/LogOptionalData' + extendedLogMeta: + $ref: '#/components/schemas/ExtendedLogMeta' - LogOptionalData: - title: optionalData + ExtendedLogMeta: + title: ExtendedLogMeta type: object nullable: true properties: diff --git a/api/events/events_test.go b/api/events/events_test.go index 513745d58..2a82a9e20 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -65,7 +65,7 @@ func TestOptionalData(t *testing.T) { testCases := []struct { name string optData *events.EventOptionalData - expected *events.LogOptionalData + expected *events.ExtendedLogMeta }{ { name: "empty optional data", @@ -77,7 +77,7 @@ func TestOptionalData(t *testing.T) { optData: &events.EventOptionalData{ TxIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ TxIndex: new(uint32), }, }, @@ -86,7 +86,7 @@ func TestOptionalData(t *testing.T) { optData: &events.EventOptionalData{ LogIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ LogIndex: new(uint32), }, }, @@ -96,7 +96,7 @@ func TestOptionalData(t *testing.T) { TxIndex: true, LogIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ TxIndex: new(uint32), LogIndex: new(uint32), }, @@ -123,7 +123,7 @@ func TestOptionalData(t *testing.T) { assert.Equal(t, 5, len(tLogs)) for _, tLog := range tLogs { - assert.Equal(t, tc.expected, tLog.Meta.OptionalData) + assert.Equal(t, tc.expected, tLog.Meta.ExtendedLogMeta) } }) } diff --git a/api/events/types.go b/api/events/types.go index a80ef81d4..f9c2f612a 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,25 +17,25 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` - OptionalData *LogOptionalData `json:"optionalData,omitempty"` + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + ExtendedLogMeta *ExtendedLogMeta `json:"extendedLogMeta,omitempty"` } -type LogOptionalData struct { +type ExtendedLogMeta struct { TxIndex *uint32 `json:"txIndex,omitempty"` LogIndex *uint32 `json:"logIndex,omitempty"` } -func (opt *LogOptionalData) Empty() bool { +func (opt *ExtendedLogMeta) Empty() bool { return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) } -func (opt *LogOptionalData) String() string { +func (opt *ExtendedLogMeta) String() string { var parts []string if opt.TxIndex != nil { parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) @@ -89,7 +89,7 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { if eventOptionalData != nil { - opt := &LogOptionalData{} + opt := &ExtendedLogMeta{} if eventOptionalData.LogIndex { opt.LogIndex = &event.Index @@ -99,7 +99,7 @@ func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *E } if !opt.Empty() { - fe.Meta.OptionalData = opt + fe.Meta.ExtendedLogMeta = opt } } return fe @@ -128,7 +128,7 @@ func (e *FilteredEvent) String() string { e.Meta.TxID, e.Meta.TxOrigin, e.Meta.ClauseIndex, - e.Meta.OptionalData, + e.Meta.ExtendedLogMeta, ) } diff --git a/api/events/types_test.go b/api/events/types_test.go index 850e16f06..e6216094d 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -164,8 +164,8 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, *result.Meta.OptionalData.TxIndex) - assert.Equal(t, event.Index, *result.Meta.OptionalData.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.ExtendedLogMeta.TxIndex) + assert.Equal(t, event.Index, *result.Meta.ExtendedLogMeta.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) @@ -173,30 +173,30 @@ func TestConvertEvent(t *testing.T) { func TestIsEmpty(t *testing.T) { // Empty cases - var nilCase *LogOptionalData + var nilCase *ExtendedLogMeta assert.True(t, nilCase.Empty()) - emptyCase := &LogOptionalData{} + emptyCase := &ExtendedLogMeta{} assert.True(t, emptyCase.Empty()) - emptyCase = &LogOptionalData{ + emptyCase = &ExtendedLogMeta{ LogIndex: nil, } assert.True(t, emptyCase.Empty()) - emptyCase = &LogOptionalData{ + emptyCase = &ExtendedLogMeta{ TxIndex: nil, } assert.True(t, emptyCase.Empty()) // Not empty cases val := uint32(1) - notEmptyCase := &LogOptionalData{ + notEmptyCase := &ExtendedLogMeta{ LogIndex: &val, } assert.False(t, notEmptyCase.Empty()) - notEmptyCase = &LogOptionalData{ + notEmptyCase = &ExtendedLogMeta{ TxIndex: &val, } assert.False(t, notEmptyCase.Empty()) From 464f530dbf11190e6cfb33a9a63cb8b90f362f8b Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Tue, 5 Nov 2024 09:37:55 +0100 Subject: [PATCH 09/16] refactor: comments, yaml and txIndex counts --- api/doc/thor.yaml | 2 ++ logdb/logdb.go | 7 +------ thor/params.go | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 66ff3eba7..fccae4437 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1380,10 +1380,12 @@ components: nullable: true properties: txIndex: + description: The index of the transaction in the block, from which the log was generated. type: integer nullable: true example: 1 logIndex: + descrption: The index of the log in the receipt's outputs. type: integer nullable: true example: 1 diff --git a/logdb/logdb.go b/logdb/logdb.go index 457a34395..817683286 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -455,11 +455,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) - indexes := make(map[thor.Bytes32]int, len(txs)) - for i, tx := range txs { - indexes[tx.ID()] = i - } - for i, r := range receipts { if isReceiptEmpty(r) { continue @@ -484,7 +479,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { txOrigin, _ = tx.Origin() } - txIndex := indexes[txID] + txIndex := i if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", diff --git a/thor/params.go b/thor/params.go index 6750f2577..3ec8462f8 100644 --- a/thor/params.go +++ b/thor/params.go @@ -13,7 +13,7 @@ import ( ) /* - NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in sequence.go: + NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in logdb/sequence.go: - an increase in gas limit may require more bits for txIndex; - if block frequency is increased, blockNumber will increment faster, potentially exhausting the allocated bits sooner than expected. */ From 9114c0b9de5e603e4489dd9f79289d4c827451ad Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:36:13 +0100 Subject: [PATCH 10/16] rebase to master --- api/doc/thor.yaml | 66 ++++------------------------ api/events/events.go | 2 +- api/events/events_test.go | 65 +++++++++------------------- api/events/types.go | 76 ++++++++++----------------------- api/events/types_test.go | 41 ++---------------- api/transfers/transfers.go | 2 +- api/transfers/transfers_test.go | 51 ++++++++++++++++++++++ api/transfers/types.go | 13 +++++- logdb/types.go | 6 ++- 9 files changed, 123 insertions(+), 199 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index fccae4437..075dedb8f 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1009,8 +1009,6 @@ components: enum: - asc - desc - optionalData: - $ref: '#/components/schemas/EventOptionalData' EventLogsResponse: type: array @@ -1022,7 +1020,7 @@ components: - $ref: '#/components/schemas/Event' - properties: meta: - $ref: '#/components/schemas/EventLogMeta' + $ref: '#/components/schemas/LogMeta' TransferLogFilterRequest: type: object @@ -1327,65 +1325,13 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false - - EventLogMeta: - title: EventLogMeta - type: object - description: The event or transfer log metadata such as block number, block timestamp, etc. - properties: - blockID: - type: string - format: hex - description: The block identifier in which the log was included. - example: '0x0004f6cc88bb4626a92907718e82f255b8fa511453a78e8797eb8cea3393b215' - nullable: false - pattern: '^0x[0-9a-f]{64}$' - blockNumber: - type: integer - format: uint32 - description: The block number (height) of the block in which the log was included. - example: 325324 - nullable: false - blockTimestamp: - type: integer - format: uint64 - description: The UNIX timestamp of the block in which the log was included. - example: 1533267900 - nullable: false - txID: - type: string - format: hex - description: The transaction identifier, from which the log was generated. - example: '0x284bba50ef777889ff1a367ed0b38d5e5626714477c40de38d71cedd6f9fa477' - nullable: false - pattern: '^0x[0-9a-f]{64}$' - txOrigin: - type: string - description: The account from which the transaction was sent. - example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d' - nullable: false - pattern: '^0x[0-9a-f]{40}$' - clauseIndex: - type: integer - format: uint32 - description: The index of the clause in the transaction, from which the log was generated. - example: 0 - nullable: false - extendedLogMeta: - $ref: '#/components/schemas/ExtendedLogMeta' - - ExtendedLogMeta: - title: ExtendedLogMeta - type: object - nullable: true - properties: txIndex: description: The index of the transaction in the block, from which the log was generated. type: integer nullable: true example: 1 logIndex: - descrption: The index of the log in the receipt's outputs. + description: The index of the log in the receipt's outputs. type: integer nullable: true example: 1 @@ -1919,6 +1865,11 @@ components: The limit of records to be included in the output. Use this parameter for pagination. Default's to all results. + includeIndexes: + type: boolean + example: true + nullable: true + description: Include both transaction and log index in the response. description: | Include these parameters to receive filtered results in a paged format. @@ -1929,7 +1880,8 @@ components: { "options": { "offset": 0, - "limit": 10 + "limit": 10, + "includeIndexes": true } } ``` diff --git a/api/events/events.go b/api/events/events.go index 62bdec355..0001280df 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e, ef.OptionalData) + fes[i] = convertEvent(e, ef.Options.IncludeIndexes) } return fes, nil } diff --git a/api/events/events_test.go b/api/events/events_test.go index 2a82a9e20..0f924e8af 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -56,64 +56,40 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } -func TestOptionalData(t *testing.T) { - db := createDb(t) - initEventServer(t, db, defaultLogLimit) +func TestOptionalIndexes(t *testing.T) { + thorChain := initEventServer(t, defaultLogLimit) defer ts.Close() - insertBlocks(t, db, 5) + insertBlocks(t, thorChain.LogDB(), 5) + tclient = thorclient.New(ts.URL) testCases := []struct { - name string - optData *events.EventOptionalData - expected *events.ExtendedLogMeta + name string + includeIndexes bool + expected *uint32 }{ { - name: "empty optional data", - optData: &events.EventOptionalData{}, - expected: nil, - }, - { - name: "optional data with txIndex", - optData: &events.EventOptionalData{ - TxIndex: true, - }, - expected: &events.ExtendedLogMeta{ - TxIndex: new(uint32), - }, + name: "do not include indexes", + includeIndexes: false, + expected: nil, }, { - name: "optional data with logIndex", - optData: &events.EventOptionalData{ - LogIndex: true, - }, - expected: &events.ExtendedLogMeta{ - LogIndex: new(uint32), - }, - }, - { - name: "optional data with txIndex and logIndex", - optData: &events.EventOptionalData{ - TxIndex: true, - LogIndex: true, - }, - expected: &events.ExtendedLogMeta{ - TxIndex: new(uint32), - LogIndex: new(uint32), - }, + name: "include indexes", + includeIndexes: true, + expected: new(uint32), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { filter := events.EventFilter{ - CriteriaSet: make([]*events.EventCriteria, 0), - Range: nil, - Options: &logdb.Options{Limit: 6}, - Order: logdb.DESC, - OptionalData: tc.optData, + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/events", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) + assert.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -123,7 +99,8 @@ func TestOptionalData(t *testing.T) { assert.Equal(t, 5, len(tLogs)) for _, tLog := range tLogs { - assert.Equal(t, tc.expected, tLog.Meta.ExtendedLogMeta) + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) } }) } diff --git a/api/events/types.go b/api/events/types.go index f9c2f612a..278b66f76 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,33 +17,14 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` - ExtendedLogMeta *ExtendedLogMeta `json:"extendedLogMeta,omitempty"` -} - -type ExtendedLogMeta struct { - TxIndex *uint32 `json:"txIndex,omitempty"` - LogIndex *uint32 `json:"logIndex,omitempty"` -} - -func (opt *ExtendedLogMeta) Empty() bool { - return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) -} - -func (opt *ExtendedLogMeta) String() string { - var parts []string - if opt.TxIndex != nil { - parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) - } - if opt.LogIndex != nil { - parts = append(parts, fmt.Sprintf("logIndex: %v", *opt.LogIndex)) - } - return fmt.Sprintf("%v", parts) + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type TopicSet struct { @@ -63,7 +44,7 @@ type FilteredEvent struct { } // convert a logdb.Event into a json format Event -func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { +func convertEvent(event *logdb.Event, addIndexes bool) *FilteredEvent { fe := &FilteredEvent{ Address: event.Address, Data: hexutil.Encode(event.Data), @@ -76,7 +57,11 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil ClauseIndex: event.ClauseIndex, }, } - fe = addOptionalData(fe, event, eventOptionalData) + + if addIndexes { + fe.Meta.TxIndex = &event.TxIndex + fe.Meta.LogIndex = &event.Index + } fe.Topics = make([]*thor.Bytes32, 0) for i := 0; i < 5; i++ { @@ -87,24 +72,6 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil return fe } -func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { - if eventOptionalData != nil { - opt := &ExtendedLogMeta{} - - if eventOptionalData.LogIndex { - opt.LogIndex = &event.Index - } - if eventOptionalData.TxIndex { - opt.TxIndex = &event.TxIndex - } - - if !opt.Empty() { - fe.Meta.ExtendedLogMeta = opt - } - } - return fe -} - func (e *FilteredEvent) String() string { return fmt.Sprintf(` Event( @@ -117,7 +84,8 @@ func (e *FilteredEvent) String() string { txID %v, txOrigin %v, clauseIndex %v, - optionalData (%v)) + txIndex: %v, + logIndex: %v) )`, e.Address, e.Topics, @@ -128,7 +96,8 @@ func (e *FilteredEvent) String() string { e.Meta.TxID, e.Meta.TxOrigin, e.Meta.ClauseIndex, - e.Meta.ExtendedLogMeta, + e.Meta.TxIndex, + e.Meta.LogIndex, ) } @@ -138,11 +107,10 @@ type EventCriteria struct { } type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` - OptionalData *EventOptionalData `json:"optionalData,omitempty"` + CriteriaSet []*EventCriteria `json:"criteriaSet"` + Range *Range `json:"range"` + Options *logdb.Options `json:"options"` + Order logdb.Order `json:"order"` } type EventOptionalData struct { diff --git a/api/events/types_test.go b/api/events/types_test.go index e6216094d..cefc56768 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -145,10 +145,6 @@ func TestConvertEvent(t *testing.T) { nil, }, } - eventOptData := &EventOptionalData{ - LogIndex: true, - TxIndex: true, - } expectedTopics := []*thor.Bytes32{ {0x0B}, @@ -156,7 +152,7 @@ func TestConvertEvent(t *testing.T) { } expectedData := hexutil.Encode(event.Data) - result := convertEvent(event, eventOptData) + result := convertEvent(event, true) assert.Equal(t, event.Address, result.Address) assert.Equal(t, expectedData, result.Data) @@ -164,40 +160,9 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, *result.Meta.ExtendedLogMeta.TxIndex) - assert.Equal(t, event.Index, *result.Meta.ExtendedLogMeta.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.TxIndex) + assert.Equal(t, event.Index, *result.Meta.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) } - -func TestIsEmpty(t *testing.T) { - // Empty cases - var nilCase *ExtendedLogMeta - assert.True(t, nilCase.Empty()) - - emptyCase := &ExtendedLogMeta{} - assert.True(t, emptyCase.Empty()) - - emptyCase = &ExtendedLogMeta{ - LogIndex: nil, - } - assert.True(t, emptyCase.Empty()) - - emptyCase = &ExtendedLogMeta{ - TxIndex: nil, - } - assert.True(t, emptyCase.Empty()) - - // Not empty cases - val := uint32(1) - notEmptyCase := &ExtendedLogMeta{ - LogIndex: &val, - } - assert.False(t, notEmptyCase.Empty()) - - notEmptyCase = &ExtendedLogMeta{ - TxIndex: &val, - } - assert.False(t, notEmptyCase.Empty()) -} diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index cad4ee6b3..7d548b29d 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -50,7 +50,7 @@ func (t *Transfers) filter(ctx context.Context, filter *TransferFilter) ([]*Filt } tLogs := make([]*FilteredTransfer, len(transfers)) for i, trans := range transfers { - tLogs[i] = convertTransfer(trans) + tLogs[i] = convertTransfer(trans, filter.Options.IncludeIndexes) } return tLogs, nil } diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 04a8c7b42..a41e0ca08 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -100,6 +100,57 @@ func TestOption(t *testing.T) { assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) } +func TestOptionalData(t *testing.T) { + db := createDb(t) + initTransferServer(t, db, defaultLogLimit) + defer ts.Close() + insertBlocks(t, db, 5) + tclient = thorclient.New(ts.URL) + + testCases := []struct { + name string + includeIndexes bool + expected *uint32 + }{ + { + name: "do not include indexes", + includeIndexes: false, + expected: nil, + }, + { + name: "include indexes", + includeIndexes: true, + expected: new(uint32), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := transfers.TransferFilter{ + CriteriaSet: make([]*logdb.TransferCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 5, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, + } + + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*transfers.FilteredTransfer + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) + } + }) + } +} + // Test functions func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} diff --git a/api/transfers/types.go b/api/transfers/types.go index 29ad9b328..440c89c5d 100644 --- a/api/transfers/types.go +++ b/api/transfers/types.go @@ -19,6 +19,8 @@ type LogMeta struct { TxID thor.Bytes32 `json:"txID"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type FilteredTransfer struct { @@ -28,9 +30,9 @@ type FilteredTransfer struct { Meta LogMeta `json:"meta"` } -func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { +func convertTransfer(transfer *logdb.Transfer, addIndexes bool) *FilteredTransfer { v := math.HexOrDecimal256(*transfer.Amount) - return &FilteredTransfer{ + ft := &FilteredTransfer{ Sender: transfer.Sender, Recipient: transfer.Recipient, Amount: &v, @@ -43,6 +45,13 @@ func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { ClauseIndex: transfer.ClauseIndex, }, } + + if addIndexes { + ft.Meta.TxIndex = &transfer.TxIndex + ft.Meta.LogIndex = &transfer.Index + } + + return ft } type TransferFilter struct { diff --git a/logdb/types.go b/logdb/types.go index 697385d03..298f20680 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -34,6 +34,7 @@ type Transfer struct { BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address ClauseIndex uint32 Sender thor.Address @@ -54,8 +55,9 @@ type Range struct { } type Options struct { - Offset uint64 - Limit uint64 + Offset uint64 + Limit uint64 + IncludeIndexes bool } type EventCriteria struct { From a80a62a432e134b1cbee8d7b4f9c578aeac47aef Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Wed, 6 Nov 2024 16:31:48 +0100 Subject: [PATCH 11/16] fix: remove stale struct --- api/events/types.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 278b66f76..65fca444f 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -113,11 +113,6 @@ type EventFilter struct { Order logdb.Order `json:"order"` } -type EventOptionalData struct { - LogIndex bool `json:"logIndex,omitempty"` - TxIndex bool `json:"txIndex,omitempty"` -} - func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { rng, err := ConvertRange(chain, filter.Range) if err != nil { From 9217741a09ceb30563c956033916abb0df6a49a0 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 11:10:05 +0100 Subject: [PATCH 12/16] add txIndex to returned logdb query --- logdb/logdb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/logdb/logdb.go b/logdb/logdb.go index 817683286..a94afd4a6 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -338,6 +338,7 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Sender: thor.BytesToAddress(sender), From 297ffd529719def60f5470f8ef6fdaba5427991e Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 13:11:29 +0100 Subject: [PATCH 13/16] reset to 0 eventCount and transferCount each receipt and write blockId only once --- logdb/logdb.go | 9 ++++++--- logdb/logdb_test.go | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index a94afd4a6..b986dae5c 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -444,8 +444,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { blockNum = b.Header().Number() blockTimestamp = b.Header().Timestamp() txs = b.Transactions() - eventCount, - transferCount uint32 isReceiptEmpty = func(r *tx.Receipt) bool { for _, o := range r.Outputs { if len(o.Events) > 0 || len(o.Transfers) > 0 { @@ -456,18 +454,23 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) + writeBlockId := true + for i, r := range receipts { + eventCount, transferCount := uint32(0), uint32(0) + if isReceiptEmpty(r) { continue } - if eventCount == 0 && transferCount == 0 { + if writeBlockId { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } + writeBlockId = false } var ( diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index aa1cb8df4..f40b23a4a 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -146,7 +146,7 @@ func TestEvents(t *testing.T) { origin, _ := tx.Origin() allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), - Index: uint32(j), + Index: uint32(0), TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), @@ -160,7 +160,8 @@ func TestEvents(t *testing.T) { allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), - Index: uint32(j), + Index: uint32(0), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), From 9ff47cb887baa8084f8c5edca45e5359d7e0b65a Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:40:01 +0100 Subject: [PATCH 14/16] fix lint --- logdb/logdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index b986dae5c..e49f869c9 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -454,7 +454,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) - writeBlockId := true + writeBlockID := true for i, r := range receipts { eventCount, transferCount := uint32(0), uint32(0) @@ -463,14 +463,14 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { continue } - if writeBlockId { + if writeBlockID { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } - writeBlockId = false + writeBlockID = false } var ( From 6f4c9eff3e4c5c0fedd9ab2c18b2b13d49d81b07 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:54:46 +0100 Subject: [PATCH 15/16] rephrase logIndex description in yaml file --- api/doc/thor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 075dedb8f..2a0d30b9e 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1331,7 +1331,7 @@ components: nullable: true example: 1 logIndex: - description: The index of the log in the receipt's outputs. + description: The index of the log in the receipt's outputs. This is an overall index among all clauses. type: integer nullable: true example: 1 From f81bc85fd1c8a2852aaaea9fb5f6f99d0fb3e00e Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 17:35:00 +0100 Subject: [PATCH 16/16] refactor: use filter.Option instead of eventFilter.Option --- api/events/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/events/events.go b/api/events/events.go index 0001280df..8c1550471 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e, ef.Options.IncludeIndexes) + fes[i] = convertEvent(e, filter.Options.IncludeIndexes) } return fes, nil }