From a0d1ed75bf6bb8da01a36e9426b5118761b1a46e Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Thu, 4 Jul 2024 17:17:22 +0800 Subject: [PATCH 01/12] doc: update the roadmap For #318 --- README.md | 8 +++++++- README_ZH.md | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e7ebf4f95..e90ef78b0 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ English | [中文](README_ZH.md) # 🚀 Features +## 🦖 Milestone + - [x] [High-performance](#-performance) event-driven looping based on a networking model of multiple threads/goroutines - [x] Built-in goroutine pool powered by the library [ants](https://github.com/panjf2000/ants) - [x] Lock-free during the entire runtime @@ -43,8 +45,12 @@ English | [中文](README_ZH.md) - [x] Running on `Linux`, `macOS`, `Windows`, and *BSD: `Darwin`/`DragonFlyBSD`/`FreeBSD`/`NetBSD`/`OpenBSD` - [x] **Edge-triggered** I/O support - [x] Multiple network addresses binding + +## 🕊 Roadmap + - [ ] **TLS** support -- [ ] [io_uring](https://kernel.dk/io_uring.pdf) support +- [ ] [io_uring](https://github.com/axboe/liburing/wiki/io_uring-and-networking-in-2023) support +- [ ] **KCP** support ***Windows version of `gnet` should only be used in development for developing and testing, it shouldn't be used in production.*** diff --git a/README_ZH.md b/README_ZH.md index a6a2b2e7d..aec90e2cd 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -31,6 +31,8 @@ # 🚀 功能 +## 🦖 当前支持 + - [x] 基于多线程/协程网络模型的[高性能](#-性能测试)事件驱动循环 - [x] 内置 goroutine 池，由开源库 [ants](https://github.com/panjf2000/ants) 提供支持 - [x] 整个生命周期是无锁的 @@ -43,8 +45,12 @@ - [x] 支持 `Linux`, `macOS`, `Windows` 和 *BSD 操作系统: `Darwin`/`DragonFlyBSD`/`FreeBSD`/`NetBSD`/`OpenBSD` - [x] **Edge-triggered** I/O 支持 - [x] 多网络地址绑定 -- [ ] **TLS** 支持 -- [ ] [io_uring](https://kernel.dk/io_uring.pdf) 支持 + +## 🕊 未来计划 + +- [ ] 支持 **TLS** +- [ ] 支持 [io_uring](https://github.com/axboe/liburing/wiki/io_uring-and-networking-in-2023) +- [ ] 支持 **KCP** ***`gnet` 的 Windows 版本应该仅用于开发阶段的开发和测试，切勿用于生产环境***。 From 7162941088af404fb71b3a9452cfd072a581ee03 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Fri, 5 Jul 2024 14:22:51 +0800 Subject: [PATCH 02/12] doc: add new use case --- README.md | 23 ++++++++++++++--------- README_ZH.md | 23 ++++++++++++++--------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e90ef78b0..1466ef73b 100644 --- a/README.md +++ b/README.md @@ -78,27 +78,27 @@ The following companies/organizations use `gnet` as the underlying network servi - - + + - + - - + + - - - + + + @@ -109,11 +109,16 @@ The following companies/organizations use `gnet` as the underlying network servi + + + + + -If you have `gnet` integrated into projects, feel free to open a pull request refreshing this list. +If your company is also using `gnet` in production, please help us enrich this list by opening a pull request. # 📊 Performance diff --git a/README_ZH.md b/README_ZH.md index aec90e2cd..0ac69c15e 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -78,27 +78,27 @@ go get -u github.com/panjf2000/gnet - - + + - + - - + + - - - + + + @@ -109,11 +109,16 @@ go get -u github.com/panjf2000/gnet + + + + + -如果你的项目也在使用 `gnet`，欢迎给我提 Pull Request 来更新这份列表。 +如果你的公司也在生产环境上使用 `gnet`，欢迎提 Pull Request 来丰富这份列表。 # 📊 性能测试 From 31cbb950833d3d02af0363dc56cb7efc517e16e9 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Fri, 5 Jul 2024 15:16:55 +0800 Subject: [PATCH 03/12] Fix a few broken image links --- README.md | 2 +- README_ZH.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1466ef73b..3cf81cf2c 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ The following companies/organizations use `gnet` as the underlying network servi - + diff --git a/README_ZH.md b/README_ZH.md index 0ac69c15e..7be4f5236 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -111,7 +111,7 @@ go get -u github.com/panjf2000/gnet - + From 66ec945d617e85d65e1debaf1e51ad1d674b3d62 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Thu, 11 Jul 2024 09:27:28 +0800 Subject: [PATCH 04/12] doc: add new use case --- README.md | 5 +++++ README_ZH.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 3cf81cf2c..01fd29412 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,11 @@ The following companies/organizations use `gnet` as the underlying network servi + + + + + diff --git a/README_ZH.md b/README_ZH.md index 7be4f5236..347ed58bf 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -114,6 +114,11 @@ go get -u github.com/panjf2000/gnet + + + + + From ca1713031a1032154f32b90cce516740805769eb Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Tue, 23 Jul 2024 07:06:57 +0800 Subject: [PATCH 05/12] chore: update READMEs --- README.md | 4 ++-- README_ZH.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 01fd29412..b64b68bf5 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ go get -u github.com/panjf2000/gnet # 🎡 Use cases -The following companies/organizations use `gnet` as the underlying network service in production. +The following corporations/organizations use `gnet` as the underlying network service in production. @@ -123,7 +123,7 @@ The following companies/organizations use `gnet` as the underlying network servi
-If your company is also using `gnet` in production, please help us enrich this list by opening a pull request. +If you're also using `gnet` in production, please help us enrich this list by opening a pull request. # 📊 Performance diff --git a/README_ZH.md b/README_ZH.md index 347ed58bf..96143a775 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -123,7 +123,7 @@ go get -u github.com/panjf2000/gnet -如果你的公司也在生产环境上使用 `gnet`，欢迎提 Pull Request 来丰富这份列表。 +如果你也正在生产环境上使用 `gnet`，欢迎提 Pull Request 来丰富这份列表。 # 📊 性能测试 From 2750d402b7df42e81805aa2e6ed965be3a6d6df3 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Tue, 30 Jul 2024 12:13:09 +0800 Subject: [PATCH 06/12] chore: update READMEs --- README.md | 16 ++++++++-------- README_ZH.md | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b64b68bf5..439338a05 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,11 @@ The following corporations/organizations use `gnet` as the underlying network se + + + + + @@ -92,30 +97,25 @@ The following corporations/organizations use `gnet` as the underlying network se + + - - - - - - - - + diff --git a/README_ZH.md b/README_ZH.md index 96143a775..f5f22c867 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -82,6 +82,11 @@ go get -u github.com/panjf2000/gnet + + + + + @@ -92,30 +97,25 @@ go get -u github.com/panjf2000/gnet + + - - - - - - - - + From 7b23091b035d0d6a2ea15732ed271023d56302ca Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sun, 18 Aug 2024 09:55:03 +0800 Subject: [PATCH 07/12] chore: clean up needless test code --- gnet_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gnet_test.go b/gnet_test.go index 3147bb70f..50abb6e0c 100644 --- a/gnet_test.go +++ b/gnet_test.go @@ -501,9 +501,8 @@ func (s *testServer) OnClose(c Conn, err error) (action Action) { if err != nil { logging.Debugf("error occurred on closed, %v\n", err) } - if c.LocalAddr().Network() != "udp" { - require.Equal(s.tester, c.Context(), c, "invalid context") - } + + require.Equal(s.tester, c.Context(), c, "invalid context") atomic.AddInt32(&s.disconnected, 1) return From b210186c29032b3c42f9920c3e987252163ae31a Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Mon, 19 Aug 2024 11:16:10 +0800 Subject: [PATCH 08/12] Move the list of patrons elsewhere Relocated to https://andypan.me/donation/#-patrons --- README.md | 44 -------------------------------------------- README_ZH.md | 44 -------------------------------------------- 2 files changed, 88 deletions(-) diff --git a/README.md b/README.md index 439338a05..bdba1a6f3 100644 --- a/README.md +++ b/README.md @@ -244,50 +244,6 @@ Become a bronze sponsor with a monthly donation of $10 and get your logo on our       -# 💴 Patrons - - - - - - - - - - - - - -
- - Patrick Othmer - - - - Jimmy - - - - ChenZhen - - - - Mai Yang - - - - 王开帅 - - - - Unger Alejandro - - - - Weng Wei - -
- # 🔑 JetBrains OS licenses `gnet` had been being developed with `GoLand` IDE under the **free JetBrains Open Source license(s)** granted by JetBrains s.r.o., hence I would like to express my thanks here. diff --git a/README_ZH.md b/README_ZH.md index f5f22c867..fe5a4f28f 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -244,50 +244,6 @@ Test duration : 15s       -# 💴 资助者 - - - - - - - - - - - - - -
- - Patrick Othmer - - - - Jimmy - - - - ChenZhen - - - - Mai Yang - - - - 王开帅 - - - - Unger Alejandro - - - - Weng Wei - -
- # 🔑 JetBrains 开源证书支持 `gnet` 项目一直以来都是在 JetBrains 公司旗下的 GoLand 集成开发环境中进行开发，基于 **free JetBrains Open Source license(s)** 正版免费授权，在此表达我的谢意。 From 8c129aeb226d1758e6f85d33a6f1b289652e1356 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sat, 21 Sep 2024 20:21:21 +0800 Subject: [PATCH 09/12] chore: clean up unused code and deprecated function calls --- eventloop_unix_test.go | 2 +- gnet.go | 2 -- gnet_test.go | 12 +++++++++--- pkg/errors/errors.go | 2 -- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/eventloop_unix_test.go b/eventloop_unix_test.go index e21bb83f1..c166dde0a 100644 --- a/eventloop_unix_test.go +++ b/eventloop_unix_test.go @@ -275,7 +275,7 @@ func (s *testServerGC) GC(secs int) { runtime.GC() gcTime := time.Since(now) gcAllTime += gcTime - s.tester.Log(s.tester.Name(), s.network, " server gc:", gcTime, ", average gc time:", gcAllTime/gcAllCount) + s.tester.Log(s.tester.Name(), s.network, "server gc:", gcTime, "average gc time:", gcAllTime/gcAllCount) if time.Since(gcStart) >= time.Second*time.Duration(secs) { break } diff --git a/gnet.go b/gnet.go index 3e2d9ab9a..99f6f4038 100644 --- a/gnet.go +++ b/gnet.go @@ -293,8 +293,6 @@ type Socket interface { // algorithm). // The default is true (no delay), meaning that data is sent as soon as possible after a Write. SetNoDelay(noDelay bool) error - // CloseRead() error - // CloseWrite() error } // Conn is an interface of underlying connection. diff --git a/gnet_test.go b/gnet_test.go index 50abb6e0c..e26f085fb 100644 --- a/gnet_test.go +++ b/gnet_test.go @@ -1206,9 +1206,15 @@ type testStopServer struct { *BuiltinEventEngine tester *testing.T network, addr, protoAddr string + eng Engine action bool } +func (t *testStopServer) OnBoot(eng Engine) (action Action) { + t.eng = eng + return +} + func (t *testStopServer) OnClose(Conn, error) (action Action) { logging.Debugf("closing connection...") return @@ -1237,7 +1243,7 @@ func (t *testStopServer) OnTick() (delay time.Duration, action Action) { go func() { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - logging.Debugf("stop engine...", Stop(ctx, t.protoAddr)) + logging.Debugf("stop engine...", t.eng.Stop(ctx)) }() // waiting the engine shutdown. @@ -1365,7 +1371,7 @@ type testClosedWakeUpServer struct { clientClosed chan struct{} } -func (s *testClosedWakeUpServer) OnBoot(_ Engine) (action Action) { +func (s *testClosedWakeUpServer) OnBoot(eng Engine) (action Action) { go func() { c, err := net.Dial(s.network, s.addr) require.NoError(s.tester, err) @@ -1380,7 +1386,7 @@ func (s *testClosedWakeUpServer) OnBoot(_ Engine) (action Action) { close(s.clientClosed) <-s.serverClosed - logging.Debugf("stop engine...", Stop(context.TODO(), s.protoAddr)) + logging.Debugf("stop engine...", eng.Stop(context.TODO())) }() return None diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 835ddedf0..3aaf1b6ba 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -35,8 +35,6 @@ var ( ErrUnsupportedUDPProtocol = errors.New("gnet: only udp/udp4/udp6 are supported") // ErrUnsupportedUDSProtocol occurs when trying to use an unsupported Unix protocol. ErrUnsupportedUDSProtocol = errors.New("gnet: only unix is supported") - // ErrUnsupportedPlatform occurs when running gnet on an unsupported platform. - ErrUnsupportedPlatform = errors.New("gnet: unsupported platform in gnet") // ErrUnsupportedOp occurs when calling some methods that has not been implemented yet. ErrUnsupportedOp = errors.New("gnet: unsupported operation") // ErrNegativeSize occurs when trying to pass a negative size to a buffer. From 894263821665ed6dfe3cb38ab9e461823be15f0b Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sat, 21 Sep 2024 20:48:17 +0800 Subject: [PATCH 10/12] test: deprecate math/rand.Seed and math/rand.Read --- .github/workflows/test.yml | 4 +-- .github/workflows/test_gc_opt.yml | 4 +-- .github/workflows/test_poll_opt.yml | 4 +-- .github/workflows/test_poll_opt_gc_opt.yml | 4 +-- client_test.go | 4 +-- gnet_test.go | 7 ++-- os_unix_test.go | 4 +-- pkg/buffer/elastic/elastic_buffer_test.go | 23 +++++++------ pkg/buffer/linkedlist/llbuffer_test.go | 29 +++++++++------- pkg/buffer/ring/ring_buffer_test.go | 40 +++++++++++++--------- 10 files changed, 68 insertions(+), 55 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 202ac0c89..752c62135 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,9 +47,9 @@ jobs: cache: false - name: Setup and run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.56.2 + version: v1.61.0 args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot --timeout 5m test: needs: lint diff --git a/.github/workflows/test_gc_opt.yml b/.github/workflows/test_gc_opt.yml index 36e592aa5..a494a1589 100644 --- a/.github/workflows/test_gc_opt.yml +++ b/.github/workflows/test_gc_opt.yml @@ -47,9 +47,9 @@ jobs: cache: false - name: Setup and run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.56.2 + version: v1.61.0 args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot --timeout 5m test: needs: lint diff --git a/.github/workflows/test_poll_opt.yml b/.github/workflows/test_poll_opt.yml index 91092992a..22991b967 100644 --- a/.github/workflows/test_poll_opt.yml +++ b/.github/workflows/test_poll_opt.yml @@ -46,9 +46,9 @@ jobs: cache: false - name: Setup and run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.56.2 + version: v1.61.0 args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot test: needs: lint diff --git a/.github/workflows/test_poll_opt_gc_opt.yml b/.github/workflows/test_poll_opt_gc_opt.yml index 3cefb09a5..1d658f449 100644 --- a/.github/workflows/test_poll_opt_gc_opt.yml +++ b/.github/workflows/test_poll_opt_gc_opt.yml @@ -46,9 +46,9 @@ jobs: cache: false - name: Setup and run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.56.2 + version: v1.61.0 args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot test: needs: lint diff --git a/client_test.go b/client_test.go index 1f446501c..3f270ba0e 100644 --- a/client_test.go +++ b/client_test.go @@ -5,6 +5,7 @@ package gnet import ( "bytes" + crand "crypto/rand" "io" "math/rand" "net" @@ -462,7 +463,6 @@ func runClient(t *testing.T, network, addr string, et, reuseport, multicore, asy } func startGnetClient(t *testing.T, cli *Client, network, addr string, multicore, async, netDial bool) { - rand.Seed(time.Now().UnixNano()) var ( c Conn err error @@ -492,7 +492,7 @@ func startGnetClient(t *testing.T, cli *Client, network, addr string, multicore, if network == "udp" { reqData = reqData[:datagramLen] } - _, err = rand.Read(reqData) + _, err = crand.Read(reqData) require.NoError(t, err) err = c.AsyncWrite(reqData, nil) require.NoError(t, err) diff --git a/gnet_test.go b/gnet_test.go index e26f085fb..3e20b0c00 100644 --- a/gnet_test.go +++ b/gnet_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + crand "crypto/rand" "encoding/binary" "errors" "io" @@ -660,7 +661,6 @@ func runServer(t *testing.T, addrs []string, et, reuseport, multicore, async, wr } func startClient(t *testing.T, network, addr string, multicore, async bool) { - rand.Seed(time.Now().UnixNano()) c, err := net.Dial(network, addr) require.NoError(t, err) defer c.Close() @@ -678,7 +678,7 @@ func startClient(t *testing.T, network, addr string, multicore, async bool) { if network == "udp" { reqData = reqData[:datagramLen] } - _, err = rand.Read(reqData) + _, err = crand.Read(reqData) require.NoError(t, err) _, err = c.Write(reqData) require.NoError(t, err) @@ -1659,7 +1659,6 @@ func runSimServer(t *testing.T, addr string, et bool, nclients, packetSize, pack } func runSimClient(t *testing.T, network, addr string, packetSize, batch int) { - rand.Seed(time.Now().UnixNano()) c, err := net.Dial(network, addr) require.NoError(t, err) defer c.Close() @@ -1695,7 +1694,7 @@ func batchSendAndRecv(t *testing.T, c net.Conn, rd *bufio.Reader, packetSize, ba ) for i := 0; i < batch; i++ { req := make([]byte, packetSize) - _, err := rand.Read(req) + _, err := crand.Read(req) require.NoError(t, err) requests = append(requests, req) packet, _ := codec.Encode(req) diff --git a/os_unix_test.go b/os_unix_test.go index 0612cfd06..696ec1514 100644 --- a/os_unix_test.go +++ b/os_unix_test.go @@ -5,6 +5,7 @@ package gnet import ( "context" + crand "crypto/rand" "errors" "fmt" "math/rand" @@ -104,7 +105,6 @@ type testMcastServer struct { } func (s *testMcastServer) startMcastClient() { - rand.Seed(time.Now().UnixNano()) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() c, err := net.Dial("udp", s.addr) @@ -117,7 +117,7 @@ func (s *testMcastServer) startMcastClient() { start := time.Now() for time.Since(start) < duration { reqData := make([]byte, 1024) - _, err = rand.Read(reqData) + _, err = crand.Read(reqData) require.NoError(s.t, err) _, err = c.Write(reqData) require.NoError(s.t, err) diff --git a/pkg/buffer/elastic/elastic_buffer_test.go b/pkg/buffer/elastic/elastic_buffer_test.go index f000ac00d..80c219fac 100644 --- a/pkg/buffer/elastic/elastic_buffer_test.go +++ b/pkg/buffer/elastic/elastic_buffer_test.go @@ -2,10 +2,10 @@ package elastic import ( "bytes" + crand "crypto/rand" "math/rand" "runtime" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -15,8 +15,8 @@ func TestMixedBuffer_Basic(t *testing.T) { mb, _ := New(maxStaticSize) const dataLen = 5 * 1024 data := make([]byte, dataLen) - rand.Seed(time.Now().Unix()) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) n, err := mb.Write(data) require.NoError(t, err) require.EqualValues(t, dataLen, n) @@ -27,7 +27,8 @@ func TestMixedBuffer_Basic(t *testing.T) { mb.Reset(-1) newDataLen := rbn + 2*1024 data = make([]byte, newDataLen) - rand.Read(data) + _, err = crand.Read(data) + require.NoError(t, err) n, err = mb.Write(data) require.NoError(t, err) require.EqualValues(t, newDataLen, n) @@ -71,7 +72,8 @@ func TestMixedBuffer_Basic(t *testing.T) { n := rand.Intn(512) + 128 cum += n data := make([]byte, n) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) buf.Write(data) if i < 3 { headCum += n @@ -107,15 +109,16 @@ func TestMixedBuffer_ReadFrom(t *testing.T) { mb, _ := New(maxStaticSize) const dataLen = 2 * 1024 data := make([]byte, dataLen) - rand.Seed(time.Now().Unix()) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) r := bytes.NewReader(data) n, err := mb.ReadFrom(r) require.NoError(t, err) require.EqualValues(t, dataLen, n) require.EqualValues(t, dataLen, mb.Buffered()) newData := make([]byte, dataLen) - rand.Read(newData) + _, err = crand.Read(newData) + require.NoError(t, err) r.Reset(newData) n, err = mb.ReadFrom(r) require.NoError(t, err) @@ -155,12 +158,12 @@ func TestMixedBuffer_WriteTo(t *testing.T) { buf bytes.Buffer ) - rand.Seed(time.Now().Unix()) for i := 0; i < maxBlocks; i++ { n := rand.Intn(512) + 128 cum += n data := make([]byte, n) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) buf.Write(data) if i < 3 { headCum += n diff --git a/pkg/buffer/linkedlist/llbuffer_test.go b/pkg/buffer/linkedlist/llbuffer_test.go index 1a755ca0d..4caabaa20 100644 --- a/pkg/buffer/linkedlist/llbuffer_test.go +++ b/pkg/buffer/linkedlist/llbuffer_test.go @@ -2,9 +2,9 @@ package linkedlist import ( "bytes" + crand "crypto/rand" "math/rand" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -16,12 +16,12 @@ func TestLinkedListBuffer_Basic(t *testing.T) { cum int buf bytes.Buffer ) - rand.Seed(time.Now().Unix()) for i := 0; i < maxBlocks; i++ { n := rand.Intn(1024) + 128 cum += n data := make([]byte, n) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) llb.PushBack(data) buf.Write(data) } @@ -39,8 +39,10 @@ func TestLinkedListBuffer_Basic(t *testing.T) { require.EqualValues(t, buf.Bytes()[:pn], p) tmpA := make([]byte, cum/16) tmpB := make([]byte, cum/16) - rand.Read(tmpA) - rand.Read(tmpB) + _, err = crand.Read(tmpA) + require.NoError(t, err) + _, err = crand.Read(tmpB) + require.NoError(t, err) bs, err = llb.PeekWithBytes(cum/4, tmpA, tmpB) require.NoError(t, err) p = p[:0] @@ -69,8 +71,8 @@ func TestLinkedListBuffer_ReadFrom(t *testing.T) { var llb Buffer const dataLen = 4 * 1024 data := make([]byte, dataLen) - rand.Seed(time.Now().Unix()) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) r := bytes.NewReader(data) n, err := llb.ReadFrom(r) require.NoError(t, err) @@ -80,9 +82,11 @@ func TestLinkedListBuffer_ReadFrom(t *testing.T) { llb.Reset() const headLen = 256 head := make([]byte, headLen) - rand.Read(head) + _, err = crand.Read(head) + require.NoError(t, err) llb.PushBack(head) - rand.Read(data) + _, err = crand.Read(data) + require.NoError(t, err) r.Reset(data) n, err = llb.ReadFrom(r) require.NoError(t, err) @@ -104,12 +108,12 @@ func TestLinkedListBuffer_WriteTo(t *testing.T) { cum int buf bytes.Buffer ) - rand.Seed(time.Now().Unix()) for i := 0; i < maxBlocks; i++ { n := rand.Intn(1024) + 128 cum += n data := make([]byte, n) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) llb.PushBack(data) buf.Write(data) } @@ -130,7 +134,8 @@ func TestLinkedListBuffer_WriteTo(t *testing.T) { n := rand.Intn(1024) + 128 cum += n data := make([]byte, n) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) llb.PushBack(data) buf.Write(data) } diff --git a/pkg/buffer/ring/ring_buffer_test.go b/pkg/buffer/ring/ring_buffer_test.go index e323342b1..2efb15509 100644 --- a/pkg/buffer/ring/ring_buffer_test.go +++ b/pkg/buffer/ring/ring_buffer_test.go @@ -2,10 +2,9 @@ package ring import ( "bytes" - "math/rand" + crand "crypto/rand" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -125,7 +124,7 @@ func TestRingBufferGrow(t *testing.T) { assert.Empty(t, head, "head should be empty") assert.Empty(t, tail, "tail should be empty") data := make([]byte, DefaultBufferSize+1) - n, err := rand.Read(data) + n, err := crand.Read(data) assert.NoError(t, err, "failed to generate random data") assert.EqualValuesf(t, DefaultBufferSize+1, n, "expect random data length is %d but got %d", DefaultBufferSize+1, n) n, err = rb.Write(data) @@ -139,7 +138,7 @@ func TestRingBufferGrow(t *testing.T) { rb = New(DefaultBufferSize) newData := make([]byte, 3*512) - n, err = rand.Read(newData) + n, err = crand.Read(newData) assert.NoError(t, err, "failed to generate random data") assert.EqualValuesf(t, 3*512, n, "expect random data length is %d but got %d", 3*512, n) n, err = rb.Write(newData) @@ -153,7 +152,7 @@ func TestRingBufferGrow(t *testing.T) { rb.Reset() data = make([]byte, bufferGrowThreshold) - n, err = rand.Read(data) + n, err = crand.Read(data) assert.NoError(t, err, "failed to generate random data") assert.EqualValuesf(t, bufferGrowThreshold, n, "expect random data length is %d but got %d", bufferGrowThreshold, n) n, err = rb.Write(data) @@ -165,7 +164,7 @@ func TestRingBufferGrow(t *testing.T) { assert.EqualValues(t, 0, rb.Available()) assert.EqualValues(t, data, rb.Bytes()) newData = make([]byte, bufferGrowThreshold/2+1) - n, err = rand.Read(newData) + n, err = crand.Read(newData) assert.NoError(t, err, "failed to generate random data") assert.EqualValuesf(t, bufferGrowThreshold/2+1, n, "expect random data length is %d but got %d", bufferGrowThreshold, n) n, err = rb.Write(newData) @@ -307,8 +306,8 @@ func TestRingBuffer_ReadFrom(t *testing.T) { rb := New(0) const dataLen = 4 * 1024 data := make([]byte, dataLen) - rand.Seed(time.Now().Unix()) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) r := bytes.NewReader(data) n, err := rb.ReadFrom(r) require.NoError(t, err) @@ -329,8 +328,10 @@ func TestRingBuffer_ReadFrom(t *testing.T) { rb = New(0) const prefixLen = 2 * 1024 prefix := make([]byte, prefixLen) - rand.Read(prefix) - rand.Read(data) + _, err = crand.Read(prefix) + require.NoError(t, err) + _, err = crand.Read(data) + require.NoError(t, err) r.Reset(data) m, err = rb.Write(prefix) require.NoError(t, err) @@ -354,8 +355,10 @@ func TestRingBuffer_ReadFrom(t *testing.T) { const initLen = 5 * 1024 rb = New(initLen) - rand.Read(prefix) - rand.Read(data) + _, err = crand.Read(prefix) + require.NoError(t, err) + _, err = crand.Read(data) + require.NoError(t, err) r.Reset(data) m, err = rb.Write(prefix) require.NoError(t, err) @@ -380,8 +383,8 @@ func TestRingBuffer_WriteTo(t *testing.T) { rb := New(5 * 1024) const dataLen = 4 * 1024 data := make([]byte, dataLen) - rand.Seed(time.Now().Unix()) - rand.Read(data) + _, err := crand.Read(data) + require.NoError(t, err) n, err := rb.Write(data) require.NoError(t, err) require.EqualValuesf(t, dataLen, n, "ringbuffer should write %d bytes, but got %d", dataLen, n) @@ -394,7 +397,8 @@ func TestRingBuffer_WriteTo(t *testing.T) { require.EqualValues(t, data, buf.Bytes()) buf.Reset() - rand.Read(data) + _, err = crand.Read(data) + require.NoError(t, err) rb = New(dataLen) n, err = rb.Write(data) require.NoError(t, err) @@ -408,7 +412,8 @@ func TestRingBuffer_WriteTo(t *testing.T) { buf.Reset() rb.Reset() - rand.Read(data) + _, err = crand.Read(data) + require.NoError(t, err) n, err = rb.Write(data) require.NoError(t, err) require.EqualValuesf(t, dataLen, n, "ringbuffer should write %d bytes, but got %d", dataLen, n) @@ -419,7 +424,8 @@ func TestRingBuffer_WriteTo(t *testing.T) { require.EqualValues(t, data[:partLen], head) _, _ = rb.Discard(partLen) partData := make([]byte, partLen/2) - rand.Read(partData) + _, err = crand.Read(partData) + require.NoError(t, err) n, err = rb.Write(partData) require.NoError(t, err) require.EqualValuesf(t, partLen/2, n, "ringbuffer should write %d bytes, but got %d", dataLen, n) From 18f54e311551af7e2f01603f3a0758111e826e70 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Sat, 21 Sep 2024 21:52:03 +0800 Subject: [PATCH 11/12] lint: disable lint checks for string<->bytes conversion temporarily --- internal/bs/bs.go | 8 ++++++-- pkg/pool/byteslice/byteslice.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/bs/bs.go b/internal/bs/bs.go index d23cb34a4..16b3861f7 100644 --- a/internal/bs/bs.go +++ b/internal/bs/bs.go @@ -19,6 +19,10 @@ import ( "unsafe" ) +// TODO(panjf2000): rework the implementation of BytesToString and StringToBytes by +// using unsafe.String/unsafe.StringData and unsafe.Slice/unsafe.SliceData when we +// bump up the minimum required Go version to 1.20. + // BytesToString converts byte slice to a string without memory allocation. // // Note it may break if the implementation of string or slice header changes in the future go versions. @@ -32,9 +36,9 @@ func BytesToString(b []byte) string { // Note it may break if the implementation of string or slice header changes in the future go versions. func StringToBytes(s string) (b []byte) { /* #nosec G103 */ - sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) //nolint:staticcheck /* #nosec G103 */ - bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) //nolint:staticcheck bh.Data, bh.Len, bh.Cap = sh.Data, sh.Len, sh.Len return b diff --git a/pkg/pool/byteslice/byteslice.go b/pkg/pool/byteslice/byteslice.go index 3fd9ac89b..efd063375 100644 --- a/pkg/pool/byteslice/byteslice.go +++ b/pkg/pool/byteslice/byteslice.go @@ -53,7 +53,7 @@ func (p *Pool) Get(size int) (buf []byte) { if ptr == nil { return make([]byte, 1< Date: Sun, 22 Sep 2024 00:00:24 +0800 Subject: [PATCH 12/12] opt: bump up minimum required Go version to 1.20 (#638) Employ the standard approach of string<->bytes conversion --- .github/workflows/cross-compile-bsd.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/test_gc_opt.yml | 4 ++-- .github/workflows/test_poll_opt.yml | 6 ++--- .github/workflows/test_poll_opt_gc_opt.yml | 6 ++--- README.md | 2 +- README_ZH.md | 2 +- go.mod | 6 ++--- go.sum | 10 ++++----- internal/bs/bs.go | 26 +++++----------------- pkg/pool/byteslice/byteslice.go | 20 ++++++----------- 11 files changed, 32 insertions(+), 56 deletions(-) diff --git a/.github/workflows/cross-compile-bsd.yml b/.github/workflows/cross-compile-bsd.yml index 0e782a704..e91a22672 100644 --- a/.github/workflows/cross-compile-bsd.yml +++ b/.github/workflows/cross-compile-bsd.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.17', '1.22'] + go: ['1.20', '1.23'] os: - ubuntu-latest name: Go ${{ matrix.go }} @ ${{ matrix.os }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 752c62135..3e964a415 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' cache: false - name: Setup and run golangci-lint @@ -56,7 +56,7 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.17', '1.22'] + go: ['1.20', '1.23'] os: - ubuntu-latest - macos-latest diff --git a/.github/workflows/test_gc_opt.yml b/.github/workflows/test_gc_opt.yml index a494a1589..df99ae8e2 100644 --- a/.github/workflows/test_gc_opt.yml +++ b/.github/workflows/test_gc_opt.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' cache: false - name: Setup and run golangci-lint @@ -56,7 +56,7 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.17', '1.22'] + go: ['1.20', '1.23'] os: - ubuntu-latest - macos-latest diff --git a/.github/workflows/test_poll_opt.yml b/.github/workflows/test_poll_opt.yml index 22991b967..5d5ae19fc 100644 --- a/.github/workflows/test_poll_opt.yml +++ b/.github/workflows/test_poll_opt.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' cache: false - name: Setup and run golangci-lint @@ -55,7 +55,7 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.17', '1.22'] + go: ['1.20', '1.23'] os: [ubuntu-latest, macos-latest] name: Go ${{ matrix.go }} @ ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -68,7 +68,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' - name: Print Go environment id: go-env diff --git a/.github/workflows/test_poll_opt_gc_opt.yml b/.github/workflows/test_poll_opt_gc_opt.yml index 1d658f449..a5aef4062 100644 --- a/.github/workflows/test_poll_opt_gc_opt.yml +++ b/.github/workflows/test_poll_opt_gc_opt.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' cache: false - name: Setup and run golangci-lint @@ -55,7 +55,7 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.17', '1.22'] + go: ['1.20', '1.23'] os: [ubuntu-latest, macos-latest] name: Go ${{ matrix.go }} @ ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -68,7 +68,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '^1.17' + go-version: '^1.20' - name: Print Go environment id: go-env diff --git a/README.md b/README.md index bdba1a6f3..8c85967b3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ - +
diff --git a/README_ZH.md b/README_ZH.md index fe5a4f28f..d166d1b76 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -4,7 +4,7 @@ - +
diff --git a/go.mod b/go.mod index 178d229c9..0ab63f761 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ require ( github.com/stretchr/testify v1.9.0 github.com/valyala/bytebufferpool v1.0.0 go.uber.org/zap v1.21.0 // don't upgrade this one - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.21.0 + golang.org/x/sync v0.8.0 + golang.org/x/sys v0.25.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -18,4 +18,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.17 +go 1.20 diff --git a/go.sum b/go.sum index 7ba46041c..6c36aa9be 100644 --- a/go.sum +++ b/go.sum @@ -17,13 +17,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -48,15 +46,15 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/bs/bs.go b/internal/bs/bs.go index 16b3861f7..7afb5136e 100644 --- a/internal/bs/bs.go +++ b/internal/bs/bs.go @@ -15,31 +15,15 @@ package bs import ( - "reflect" "unsafe" ) -// TODO(panjf2000): rework the implementation of BytesToString and StringToBytes by -// using unsafe.String/unsafe.StringData and unsafe.Slice/unsafe.SliceData when we -// bump up the minimum required Go version to 1.20. - -// BytesToString converts byte slice to a string without memory allocation. -// -// Note it may break if the implementation of string or slice header changes in the future go versions. +// BytesToString converts byte slice to a string without any memory allocation. func BytesToString(b []byte) string { - /* #nosec G103 */ - return *(*string)(unsafe.Pointer(&b)) + return unsafe.String(unsafe.SliceData(b), len(b)) } -// StringToBytes converts string to a byte slice without memory allocation. -// -// Note it may break if the implementation of string or slice header changes in the future go versions. -func StringToBytes(s string) (b []byte) { - /* #nosec G103 */ - sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) //nolint:staticcheck - /* #nosec G103 */ - bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) //nolint:staticcheck - - bh.Data, bh.Len, bh.Cap = sh.Data, sh.Len, sh.Len - return b +// StringToBytes converts string to a byte slice without any memory allocation. +func StringToBytes(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) } diff --git a/pkg/pool/byteslice/byteslice.go b/pkg/pool/byteslice/byteslice.go index efd063375..53397dac6 100644 --- a/pkg/pool/byteslice/byteslice.go +++ b/pkg/pool/byteslice/byteslice.go @@ -17,8 +17,6 @@ package byteslice import ( "math" "math/bits" - "reflect" - "runtime" "sync" "unsafe" ) @@ -41,7 +39,7 @@ func Put(buf []byte) { } // Get retrieves a byte slice of the length requested by the caller from pool or allocates a new one. -func (p *Pool) Get(size int) (buf []byte) { +func (p *Pool) Get(size int) []byte { if size <= 0 { return nil } @@ -49,16 +47,11 @@ func (p *Pool) Get(size int) (buf []byte) { return make([]byte, size) } idx := index(uint32(size)) - ptr, _ := p.pools[idx].Get().(unsafe.Pointer) + ptr, _ := p.pools[idx].Get().(*byte) if ptr == nil { - return make([]byte, 1<