From 872ba931e98c5cad1dc8742cd158925f93db04ee Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:49:27 +0000 Subject: [PATCH] fix: docker build not working with sqlite (#648) * fix: docker build not working with sqlite * fix: typos * fix: add a unit test to ensure NewestBlockID is working * fix: rename unit test * fix: desc not working with range/limit filters * remove whitespace * revert whitespace change * fix: remove tags and labels from docker build * improve logdb query, set order inside subquery only when limit option is set * more tests for logdb --------- Co-authored-by: tony --- ...{build_docker.yaml => release-docker.yaml} | 0 .github/workflows/test-docker-build.yaml | 31 +++ go.mod | 2 +- go.sum | 2 + logdb/logdb.go | 46 ++-- logdb/logdb_test.go | 220 ++++++++++++++++++ 6 files changed, 286 insertions(+), 15 deletions(-) rename .github/workflows/{build_docker.yaml => release-docker.yaml} (100%) create mode 100644 .github/workflows/test-docker-build.yaml diff --git a/.github/workflows/build_docker.yaml b/.github/workflows/release-docker.yaml similarity index 100% rename from .github/workflows/build_docker.yaml rename to .github/workflows/release-docker.yaml diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml new file mode 100644 index 000000000..b80be30d5 --- /dev/null +++ b/.github/workflows/test-docker-build.yaml @@ -0,0 +1,31 @@ +name: Docker + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + docker_build: + runs-on: ubuntu-latest + name: Test Build + + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + provenance: false diff --git a/go.mod b/go.mod index 390233ba4..c051824e8 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/holiman/uint256 v1.2.0 github.com/inconshreveable/log15 v0.0.0-20171019012758-0decfc6c20d9 github.com/mattn/go-isatty v0.0.3 - github.com/mattn/go-sqlite3 v1.14.9 + github.com/mattn/go-sqlite3 v1.14.19 github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c github.com/pkg/errors v0.8.0 diff --git a/go.sum b/go.sum index 0ba6db36b..f5f7abdb3 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a h1:8TGB3DFRNl06DB1Q6zBX+I7FDoCUZY2fmMS9WGUIIpw= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= diff --git a/logdb/logdb.go b/logdb/logdb.go index 8c7de879d..985d9cd18 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -140,20 +140,29 @@ FROM (%v) e subQuery += ")" } - if filter.Order == DESC { - subQuery += " ORDER BY seq DESC " - } else { - subQuery += " ORDER BY seq ASC " - } - + // if there is limit option, set order inside subquery if filter.Options != nil { + if filter.Order == DESC { + subQuery += " ORDER BY seq DESC " + } else { + subQuery += " ORDER BY seq ASC " + } subQuery += " LIMIT ?, ?" args = append(args, filter.Options.Offset, filter.Options.Limit) } subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN event e ON s.seq = e.seq" - return db.queryEvents(ctx, fmt.Sprintf(query, subQuery), args...) + eventQuery := fmt.Sprintf(query, subQuery) + // if there is no limit option, set order outside + if filter.Options == nil { + if filter.Order == DESC { + eventQuery += " ORDER BY seq DESC " + } else { + eventQuery += " ORDER BY seq ASC " + } + } + return db.queryEvents(ctx, eventQuery, args...) } func (db *LogDB) FilterTransfers(ctx context.Context, filter *TransferFilter) ([]*Transfer, error) { @@ -197,19 +206,28 @@ FROM (%v) t subQuery += ")" } - if filter.Order == DESC { - subQuery += " ORDER BY seq DESC" - } else { - subQuery += " ORDER BY seq ASC" - } - + // if there is limit option, set order inside subquery if filter.Options != nil { + if filter.Order == DESC { + subQuery += " ORDER BY seq DESC" + } else { + subQuery += " ORDER BY seq ASC" + } subQuery += " LIMIT ?, ?" args = append(args, filter.Options.Offset, filter.Options.Limit) } subQuery = "SELECT e.* FROM (" + subQuery + ") s LEFT JOIN transfer e ON s.seq = e.seq" - return db.queryTransfers(ctx, fmt.Sprintf(query, subQuery), args...) + transferQuery := fmt.Sprintf(query, subQuery) + // if there is no limit option, set order outside + if filter.Options == nil { + if filter.Order == DESC { + transferQuery += " ORDER BY seq DESC " + } else { + transferQuery += " ORDER BY seq ASC " + } + } + return db.queryTransfers(ctx, transferQuery, args...) } func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interface{}) ([]*Event, error) { diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index ffa2bb0d5..87fbd8ea2 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -56,6 +56,34 @@ func newReceipt() *tx.Receipt { } } +func newEventOnlyReceipt() *tx.Receipt { + return &tx.Receipt{ + Outputs: []*tx.Output{ + { + Events: tx.Events{{ + Address: randAddress(), + Topics: []thor.Bytes32{randBytes32()}, + Data: randBytes32().Bytes(), + }}, + }, + }, + } +} + +func newTransferOnlyReceipt() *tx.Receipt { + return &tx.Receipt{ + Outputs: []*tx.Output{ + { + Transfers: tx.Transfers{{ + Sender: randAddress(), + Recipient: randAddress(), + Amount: new(big.Int).SetBytes(randAddress().Bytes()), + }}, + }, + }, + } +} + type eventLogs []*logdb.Event func (logs eventLogs) Filter(f func(ev *logdb.Event) bool) (ret eventLogs) { @@ -161,10 +189,13 @@ func TestEvents(t *testing.T) { want eventLogs }{ {"query all events", &logdb.EventFilter{}, allEvents}, + {"query all events with nil option", nil, allEvents}, {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents}, {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()}, {"query all events limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, {"query all events range", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, + {"query events with range and desc", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, + {"query events with limit with desc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allEvents.Reverse()[0:10]}, {"query all events with criteria", &logdb.EventFilter{CriteriaSet: []*logdb.EventCriteria{{Address: &allEvents[1].Address}}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.Address == allEvents[1].Address })}, @@ -189,10 +220,13 @@ func TestEvents(t *testing.T) { want transferLogs }{ {"query all transfers", &logdb.TransferFilter{}, allTransfers}, + {"query all transfers with nil option", nil, allTransfers}, {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers}, {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()}, {"query all transfers limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, {"query all transfers range", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, + {"query transfers with range and desc", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, + {"query transfers with limit with desc", &logdb.TransferFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allTransfers.Reverse()[0:10]}, {"query all transfers with criteria", &logdb.TransferFilter{CriteriaSet: []*logdb.TransferCriteria{{Sender: &allTransfers[1].Sender}}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.Sender == allTransfers[1].Sender })}, @@ -210,3 +244,189 @@ func TestEvents(t *testing.T) { } } } + +func TestLogDB_NewestBlockID(t *testing.T) { + db, err := logdb.NewMem() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + b := new(block.Builder).Build() + + b = new(block.Builder). + ParentID(b.Header().ID()). + Transaction(newTx()). + Build() + receipts := tx.Receipts{newReceipt()} + + w := db.NewWriter() + if err := w.Write(b, receipts); err != nil { + t.Fatal(err) + } + if err := w.Commit(); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + prepare func() (thor.Bytes32, error) + }{ + { + "newest block id", + func() (thor.Bytes32, error) { + return b.Header().ID(), nil + }, + }, { + "add empty block, best should remain unchanged", + func() (thor.Bytes32, error) { + wanted := b.Header().ID() + b = new(block.Builder).ParentID(b.Header().ID()).Build() + receipts = tx.Receipts{} + + w := db.NewWriter() + if err := w.Write(b, receipts); err != nil { + return thor.Bytes32{}, nil + } + if err := w.Commit(); err != nil { + return thor.Bytes32{}, nil + } + return wanted, nil + }, + }, + { + "add both event and transfer, best should change", + func() (thor.Bytes32, error) { + b = new(block.Builder). + ParentID(b.Header().ID()). + Transaction(newTx()). + Build() + receipts := tx.Receipts{newReceipt()} + + w := db.NewWriter() + if err := w.Write(b, receipts); err != nil { + return thor.Bytes32{}, nil + } + if err := w.Commit(); err != nil { + return thor.Bytes32{}, nil + } + return b.Header().ID(), nil + }, + }, + { + "add event only, best should change", + func() (thor.Bytes32, error) { + b = new(block.Builder). + ParentID(b.Header().ID()). + Transaction(newTx()). + Build() + receipts := tx.Receipts{newEventOnlyReceipt()} + + w := db.NewWriter() + if err := w.Write(b, receipts); err != nil { + return thor.Bytes32{}, nil + } + if err := w.Commit(); err != nil { + return thor.Bytes32{}, nil + } + return b.Header().ID(), nil + }, + }, + { + "add transfer only, best should change", + func() (thor.Bytes32, error) { + b = new(block.Builder). + ParentID(b.Header().ID()). + Transaction(newTx()). + Build() + receipts := tx.Receipts{newTransferOnlyReceipt()} + + w := db.NewWriter() + if err := w.Write(b, receipts); err != nil { + return thor.Bytes32{}, nil + } + if err := w.Commit(); err != nil { + return thor.Bytes32{}, nil + } + return b.Header().ID(), nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + want, err := tt.prepare() + if err != nil { + t.Fatal(err) + } + got, err := db.NewestBlockID() + if err != nil { + t.Fatal(err) + } + assert.Equal(t, want, got) + }) + } +} + +func TestLogDB_HasBlockID(t *testing.T) { + db, err := logdb.NewMem() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + b0 := new(block.Builder).Build() + + b := new(block.Builder). + ParentID(b0.Header().ID()). + Transaction(newTx()). + Build() + b1 := b.Header().ID() + receipts := tx.Receipts{newReceipt()} + + w := db.NewWriter() + _ = w.Write(b, receipts) + + b = new(block.Builder). + ParentID(b1). + Build() + b2 := b.Header().ID() + receipts = tx.Receipts{} + _ = w.Write(b, receipts) + + b = new(block.Builder). + ParentID(b2). + Transaction(newTx()). + Build() + b3 := b.Header().ID() + receipts = tx.Receipts{newEventOnlyReceipt()} + _ = w.Write(b, receipts) + + if err := w.Commit(); err != nil { + t.Fatal(err) + } + + has, err := db.HasBlockID(b0.Header().ID()) + if err != nil { + t.Fatal(err) + } + assert.False(t, has) + + has, err = db.HasBlockID(b1) + if err != nil { + t.Fatal(err) + } + assert.True(t, has) + + has, err = db.HasBlockID(b2) + if err != nil { + t.Fatal(err) + } + assert.False(t, has) + + has, err = db.HasBlockID(b3) + if err != nil { + t.Fatal(err) + } + assert.True(t, has) + +}