Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bug]: Crash after servals hour on concurrency #71

Open
iamat8080 opened this issue Aug 15, 2023 · 9 comments
Open

[Bug]: Crash after servals hour on concurrency #71

iamat8080 opened this issue Aug 15, 2023 · 9 comments
Labels
bug Something isn't working

Comments

@iamat8080
Copy link

iamat8080 commented Aug 15, 2023 •
edited
Loading

TLS client version

v1.3.12

System information

windows 10
amd ryzen 7 3700x

Issue description

github.com/bogdanfinn/fhttp.setRequestCancel.func4()
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:398 +0x8b
created by github.com/bogdanfinn/fhttp.setRequestCancel
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:397 +0x44a
goroutine 439378 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x181b7c2e2e8, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc3afeae5b260f336?, 0xc005b4b6c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc004a09418, 0xe04580)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc004a09400, {0xc001098000, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc004a09400, {0xc001098000?, 0xc001098005?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00022f2d8, {0xc001098000?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc0044d6678, {0xc001098000?, 0xc0044d6678?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc001127430, {0xe74660, 0xc0044d6678})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc001127180, {0x181b72aa0f8?, 0xc00022f2d8}, 0xc000bf4000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc001127180, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc001127180, {0xc004104000, 0x1000, 0xc003ee5e00?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc002ec1b60, {0xc0036b6658, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xe740e0, 0xc002ec1b60}, {0xc0036b6658, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0036b6658?, 0x9?, 0xcd1ca2?}, {0xe740e0?, 0xc002ec1b60?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0036b6620)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc005b4bfa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc002c6c1a0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

Steps to reproduce / Code Sample

type request struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
        data string
}

func init_(thread int) request {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return request{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

func (ms *request) Today() bool {
	url := "https://api.mydomain.com/data"
	payload := fmt.Sprintf("data=today", ms.userdata)
	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(payload))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = bheader
	resp, err := ms.client.Do(req)
	if err != nil {
		ms.clientChanger()
		PrintLogE(err.Error(), ms.Thread)
		return false
	}
        defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	var data map[string]interface{}
	err = json.Unmarshal(readBytes, &data)
	if err != nil {
		PrintLogF("Error unmarshalling JSON:"+err.Error(), ms.Thread)
		return false, false
	}
        ms.data = data

	return true

}

func (ms *request) postdata() bool {

	url := "https://api.seconddomain.com/postdata"

	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(ms.data))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = ms.headerstream
	resp, err := ms.client.Do(req)
	if err != nil {
		PrintLogE(err.Error(), ms.Thread)
		time.Sleep(5 * time.Second)
		return false
	}
        defer resp.Body.Close()
	if resp.StatusCode == 201 {
		return true
	} else {
		readBytes, err := io.ReadAll(resp.Body)
		if err != nil {
			PrintLogF(err.Error(), ms.Thread)
			return false
		}
		PrintLogM(fmt.Sprintf("status : %d dat : %s", resp.StatusCode, readBytes), ms.Thread)
	}
	return false
}

func run(Threads int) bool {
	self := init_(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	for i := 0; i < 3; i++{
		anid := self.postdata()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL POST %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {
	
	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()
	PrintLog("DONE", 0)
}
@iamat8080 iamat8080 added the bug Something isn't working label Aug 15, 2023
@bogdanfinn
Copy link
Owner

@iamat8080 the code does not reproduce your issue.

@iamat8080
Copy link
Author

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

@iamat8080
Copy link
Author 

0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc003469fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc006b13a00)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

goroutine 774853 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x1ad6700f8e0, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xd81159c9a0d3a647?, 0xc002c976c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc000ca0018, 0x8a4540)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc000ca0000, {0xc00547b500, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc000ca0000, {0xc00547b500?, 0xc00547b505?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005dc378, {0xc00547b500?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc004cc36e0, {0xc00547b500?, 0xc004cc36e0?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc003027eb0, {0x914600, 0xc004cc36e0})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc003027c00, {0x1ad40a440d8?, 0xc0005dc378}, 0xc00504f000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc003027c00, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc003027c00, {0xc006ee9000, 0x1000, 0xc009dcd200?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc0090fed20, {0xc0059be8f8, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x914080, 0xc0090fed20}, {0xc0059be8f8, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0059be8f8?, 0x9?, 0x771ca2?}, {0x914080?, 0xc0090fed20?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0059be8c0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc002c97fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc0091fd040)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

this time it's different

@iamat8080
Copy link
Author

hello @gngpp I am using windows

@KhushC-03
Copy link

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

Getting the same exact issue, works perfectly fine but crashes after a few hours of running.

github.com/bogdanfinn/fhttp.setRequestCancel.func4()
C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:398 +0x8b
created by github.com/bogdanfinn/fhttp.setRequestCancel
C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:397 +0x44a

@bogdanfinn
Copy link
Owner

bogdanfinn commented Aug 28, 2023 •
edited
Loading

Still no code provided which reproduces the issue and can be compiled.

Bildschirmfoto vom 2023-08-28 09-44-30

@iamat8080
Copy link
Author

iamat8080 commented Aug 30, 2023 •
edited
Loading 

package main

import (
	"fmt"
	"io"
	"sync"
	"time"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/bogdanfinn/tls-client"
)

type re struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
	data string
}

func init__(thread int) re {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return re{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

var gg = http.Header{
	"accept":             {"application/json"},
	"Cache-Control":      {"no-cache"},
	"user-agent":         {"android/8.1.7.1053/NapsterGlobal"},
	"Accept-Encoding":    {"gzip, deflate"},
	"Origin": {"https://www.ssense.com"},
	"Referer":       {"https://www.ssense.com/"},
	"Connection":         {"Close"},
	http.HeaderOrderKey: {
		"accept",
		"Accept-Encoding",
		"Cache-Control",
		"Origin",
		"Referer",
		"User-Agent",
		}}

func (ms *re) Today() bool {
	url := "https://www.ssense.com/en-us/men"
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return false
	}

	req.Header = gg
	resp, err := ms.client.Do(req)
	if err != nil {
		return false
	}
	defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return false
	}
	fmt.Println(readBytes)

	return true

}

func run(Threads int) bool {
	self := init__(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {

	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()

}

it's sample code I didn't add another post request where I scarpe and post to DB using API

If you format any bot like this with a high number of threads and in an endless loop running for a while it might crash
or it would be great if you guide me on what to do I'll test debug on my side and provide you with all logs
else I have to write different code using different domains and APIs which can be similar to this one

PS: I believe running in an endless loop with few posts and get requests can reproduce the crash
@bogdanfinn

@huaibaoyy1
Copy link

Brother, I also encountered the same problem as you, have you solved it? It will crash after reaching millions of requests.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
bug Something isn't working
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
5 participants
@bogdanfinn @iamat8080 @KhushC-03 @huaibaoyy1 and others