Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support HTTP methods other than GET and POST #834

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Support HTTP methods other than GET and POST #834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
51a57d6
Support HTTP methods other than GET and POST
shelld3v May 29, 2022
fbb0414
Merge branch 'develop' into http-methods
caffix Jun 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/resolvers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ var PublicResolvers []string
// GetPublicDNSResolvers obtains the public DNS server addresses from public-dns.info and assigns them to PublicResolvers.
func GetPublicDNSResolvers() error {
url := "https://public-dns.info/nameservers-all.csv"
page, err := http.RequestWebPage(context.Background(), url, nil, nil, nil)
page, err := http.RequestWebPage(context.Background(), url, "get", nil, nil, nil)
if err != nil {
return fmt.Errorf("failed to obtain the Public DNS csv file at %s: %v", url, err)
}
Expand Down
10 changes: 5 additions & 5 deletions datasrcs/alienvault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (a *AlienVault) executeDNSQuery(ctx context.Context, req *requests.DNSReque
}

u := a.getURL(req.Domain) + "passive_dns"
page, err := http.RequestWebPage(ctx, u, nil, a.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, a.getHeaders(), nil)
if err != nil {
a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
return
Expand Down Expand Up @@ -173,7 +173,7 @@ func (a *AlienVault) executeURLQuery(ctx context.Context, req *requests.DNSReque

headers := a.getHeaders()
u := a.getURL(req.Domain) + "url_list"
page, err := http.RequestWebPage(ctx, u, nil, headers, nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, headers, nil)
if err != nil {
a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
return
Expand Down Expand Up @@ -208,7 +208,7 @@ func (a *AlienVault) executeURLQuery(ctx context.Context, req *requests.DNSReque
for cur := m.PageNum + 1; cur <= pages; cur++ {
a.CheckRateLimit()
pageURL := u + "?page=" + strconv.Itoa(cur)
page, err = http.RequestWebPage(ctx, pageURL, nil, headers, nil)
page, err = http.RequestWebPage(ctx, pageURL, "get", nil, headers, nil)
if err != nil {
a.sys.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
break
Expand Down Expand Up @@ -263,7 +263,7 @@ func (a *AlienVault) executeWhoisQuery(ctx context.Context, req *requests.WhoisR
headers := a.getHeaders()
for _, email := range emails {
pageURL := a.getReverseWhoisURL(email)
page, err := http.RequestWebPage(ctx, pageURL, nil, headers, nil)
page, err := http.RequestWebPage(ctx, pageURL, "get", nil, headers, nil)
if err != nil {
a.sys.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
continue
Expand Down Expand Up @@ -303,7 +303,7 @@ func (a *AlienVault) queryWhoisForEmails(ctx context.Context, req *requests.Whoi
defer emails.Close()

u := a.getWhoisURL(req.Domain)
page, err := http.RequestWebPage(ctx, u, nil, a.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, a.getHeaders(), nil)
if err != nil {
a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
return emails.Slice()
Expand Down
2 changes: 1 addition & 1 deletion datasrcs/dnsdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (d *DNSDB) dnsRequest(ctx context.Context, req *requests.DNSRequest) {
}

url := d.getURL(req.Domain)
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
d.sys.Config().Log.Printf("%s: %s: %v", d.String(), url, err)
return
Expand Down
20 changes: 10 additions & 10 deletions datasrcs/networksdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func (n *NetworksDB) asnRequest(ctx context.Context, req *requests.ASNRequest) {

func (n *NetworksDB) executeASNAddrQuery(ctx context.Context, addr string) {
u := n.getIPURL(addr)
page, err := http.RequestWebPage(ctx, u, nil, nil, nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return
Expand All @@ -141,7 +141,7 @@ func (n *NetworksDB) executeASNAddrQuery(ctx context.Context, addr string) {

numRateLimitChecks(n, 3)
u = networksdbBaseURL + matches[1]
page, err = http.RequestWebPage(ctx, u, nil, nil, nil)
page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return
Expand Down Expand Up @@ -178,7 +178,7 @@ func (n *NetworksDB) getIPURL(addr string) string {
func (n *NetworksDB) executeASNQuery(ctx context.Context, asn int, addr string, netblocks *stringset.Set) {
numRateLimitChecks(n, 3)
u := n.getASNURL(asn)
page, err := http.RequestWebPage(ctx, u, nil, nil, nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return
Expand Down Expand Up @@ -331,7 +331,7 @@ func (n *NetworksDB) apiIPQuery(ctx context.Context, addr string) (string, strin
u := n.getAPIIPURL()
params := url.Values{"ip": {addr}}
body := strings.NewReader(params.Encode())
page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return "", ""
Expand Down Expand Up @@ -372,7 +372,7 @@ func (n *NetworksDB) apiOrgInfoQuery(ctx context.Context, id string) []int {
u := n.getAPIOrgInfoURL()
params := url.Values{"id": {id}}
body := strings.NewReader(params.Encode())
page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return []int{}
Expand Down Expand Up @@ -408,7 +408,7 @@ func (n *NetworksDB) apiASNInfoQuery(ctx context.Context, asn int) *requests.ASN
u := n.getAPIASNInfoURL()
params := url.Values{"asn": {strconv.Itoa(asn)}}
body := strings.NewReader(params.Encode())
page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return nil
Expand Down Expand Up @@ -456,7 +456,7 @@ func (n *NetworksDB) apiNetblocksQuery(ctx context.Context, asn int) *stringset.
u := n.getAPINetblocksURL()
params := url.Values{"asn": {strconv.Itoa(asn)}}
body := strings.NewReader(params.Encode())
page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil)
page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return netblocks
Expand Down Expand Up @@ -508,7 +508,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques

numRateLimitChecks(n, 2)
u := n.getDomainToIPURL(req.Domain)
page, err := http.RequestWebPage(ctx, u, nil, nil, nil)
page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
return
Expand All @@ -531,7 +531,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques

numRateLimitChecks(n, 3)
u = networksdbBaseURL + match[1]
page, err = http.RequestWebPage(ctx, u, nil, nil, nil)
page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
continue
Expand All @@ -552,7 +552,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques
first, last := amassnet.FirstLast(cidr)
u := n.getDomainsInNetworkURL(first.String(), last.String())

page, err = http.RequestWebPage(ctx, u, nil, nil, nil)
page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil)
if err != nil {
n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
continue
Expand Down
6 changes: 3 additions & 3 deletions datasrcs/radb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func (r *RADb) asnRequest(ctx context.Context, req *requests.ASNRequest) {
func (r *RADb) executeASNAddrQuery(ctx context.Context, addr string) {
url := r.getIPURL("arin", addr)
headers := map[string]string{"Content-Type": "application/json"}
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
return
Expand Down Expand Up @@ -172,7 +172,7 @@ func (r *RADb) executeASNQuery(ctx context.Context, asn int, addr, prefix string
numRateLimitChecks(r, 2)
url := r.getASNURL("arin", strconv.Itoa(asn))
headers := map[string]string{"Content-Type": "application/json"}
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
return
Expand Down Expand Up @@ -251,7 +251,7 @@ func (r *RADb) netblocks(ctx context.Context, asn int) *stringset.Set {
numRateLimitChecks(r, 2)
url := r.getNetblocksURL(strconv.Itoa(asn))
headers := map[string]string{"Content-Type": "application/json"}
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
return netblocks
Expand Down
34 changes: 16 additions & 18 deletions datasrcs/scripting/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,19 @@ func (s *Script) request(L *lua.LState) int {
return 2
}

var data string
if method, ok := getStringField(L, opt, "method"); ok && strings.ToLower(method) == "post" {
if d, ok := getStringField(L, opt, "data"); ok {
data = d
}
}

url, found := getStringField(L, opt, "url")
if !found {
L.Push(lua.LNil)
L.Push(lua.LString("No URL found in the parameters"))
return 2
}

method, ok := getStringField(L, opt, "method")
if !ok {
method = "get"
}
data, _ := getStringField(L, opt, "data")

headers := make(map[string]string)
lv := L.GetField(opt, "headers")
if tbl, ok := lv.(*lua.LTable); ok {
Expand All @@ -53,7 +52,7 @@ func (s *Script) request(L *lua.LState) int {

id, _ := getStringField(L, opt, "id")
pass, _ := getStringField(L, opt, "pass")
page, err := s.req(ctx, url, data, headers, &http.BasicAuth{
page, err := s.req(ctx, url, method, data, headers, &http.BasicAuth{
Username: id,
Password: pass,
})
Expand Down Expand Up @@ -81,19 +80,18 @@ func (s *Script) scrape(L *lua.LState) int {
return 1
}

var data string
if method, ok := getStringField(L, opt, "method"); ok && strings.ToLower(method) == "post" {
if d, ok := getStringField(L, opt, "data"); ok {
data = d
}
}

url, found := getStringField(L, opt, "url")
if !found {
L.Push(lua.LFalse)
return 1
}

method, ok := getStringField(L, opt, "method")
if !ok {
method = "get"
}
data, _ := getStringField(L, opt, "data")

headers := make(map[string]string)
lv := L.GetField(opt, "headers")
if tbl, ok := lv.(*lua.LTable); ok {
Expand All @@ -106,7 +104,7 @@ func (s *Script) scrape(L *lua.LState) int {
pass, _ := getStringField(L, opt, "pass")

sucess := lua.LFalse
if resp, err := s.req(ctx, url, data, headers, &http.BasicAuth{
if resp, err := s.req(ctx, url, method, data, headers, &http.BasicAuth{
Username: id,
Password: pass,
}); err == nil {
Expand All @@ -121,7 +119,7 @@ func (s *Script) scrape(L *lua.LState) int {
return 1
}

func (s *Script) req(ctx context.Context, url, data string, headers map[string]string, auth *http.BasicAuth) (string, error) {
func (s *Script) req(ctx context.Context, url, method string, data string, headers map[string]string, auth *http.BasicAuth) (string, error) {
cfg := s.sys.Config()
// Check for cached responses first
dsc := cfg.GetDataSourceConfig(s.String())
Expand All @@ -137,7 +135,7 @@ func (s *Script) req(ctx context.Context, url, data string, headers map[string]s
}

numRateLimitChecks(s, s.seconds)
resp, err := http.RequestWebPage(ctx, url, body, headers, auth)
resp, err := http.RequestWebPage(ctx, url, method, body, headers, auth)
if err != nil {
if cfg.Verbose {
cfg.Log.Printf("%s: %s: %v", s.String(), url, err)
Expand Down
2 changes: 1 addition & 1 deletion datasrcs/twitter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func (t *Twitter) dnsRequest(ctx context.Context, req *requests.DNSRequest) {

func (t *Twitter) getBearerToken() (string, error) {
headers := map[string]string{"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"}
page, err := http.RequestWebPage(context.Background(), "https://api.twitter.com/oauth2/token",
page, err := http.RequestWebPage(context.Background(), "https://api.twitter.com/oauth2/token", "post",
strings.NewReader("grant_type=client_credentials"), headers,
&http.BasicAuth{
Username: t.creds.Key,
Expand Down
12 changes: 6 additions & 6 deletions datasrcs/umbrella.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (u *Umbrella) dnsRequest(ctx context.Context, req *requests.DNSRequest) {

headers := u.restHeaders()
url := u.restDNSURL(req.Domain)
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
return
Expand Down Expand Up @@ -138,7 +138,7 @@ func (u *Umbrella) addrRequest(ctx context.Context, req *requests.AddrRequest) {

headers := u.restHeaders()
url := u.restAddrURL(req.Address)
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
return
Expand Down Expand Up @@ -177,7 +177,7 @@ func (u *Umbrella) asnRequest(ctx context.Context, req *requests.ASNRequest) {
func (u *Umbrella) executeASNAddrQuery(ctx context.Context, req *requests.ASNRequest) {
headers := u.restHeaders()
url := u.restAddrToASNURL(req.Address)
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
return
Expand Down Expand Up @@ -235,7 +235,7 @@ func (u *Umbrella) executeASNAddrQuery(ctx context.Context, req *requests.ASNReq
func (u *Umbrella) executeASNQuery(ctx context.Context, req *requests.ASNRequest) {
headers := u.restHeaders()
url := u.restASNToCIDRsURL(req.ASN)
page, err := http.RequestWebPage(ctx, url, nil, headers, nil)
page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
return
Expand Down Expand Up @@ -333,7 +333,7 @@ func (u *Umbrella) queryWhois(ctx context.Context, domain string) *whoisRecord {
whoisURL := u.whoisRecordURL(domain)

u.CheckRateLimit()
record, err := http.RequestWebPage(ctx, whoisURL, nil, headers, nil)
record, err := http.RequestWebPage(ctx, whoisURL, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err)
return nil
Expand All @@ -357,7 +357,7 @@ func (u *Umbrella) queryReverseWhois(ctx context.Context, apiURL string) []strin
for count, more := 0, true; more; count = count + 500 {
u.CheckRateLimit()
fullAPIURL := fmt.Sprintf("%s&offset=%d", apiURL, count)
record, err := http.RequestWebPage(ctx, fullAPIURL, nil, headers, nil)
record, err := http.RequestWebPage(ctx, fullAPIURL, "get", nil, headers, nil)
if err != nil {
u.sys.Config().Log.Printf("%s: %s: %v", u.String(), apiURL, err)
return domains.Slice()
Expand Down
9 changes: 2 additions & 7 deletions net/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,8 @@ func CheckCookie(urlString string, cookieName string) bool {
}

// RequestWebPage returns a string containing the entire response for the provided URL when successful.
func RequestWebPage(ctx context.Context, u string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) {
method := "GET"
if body != nil {
method = "POST"
}

req, err := http.NewRequestWithContext(ctx, method, u, body)
func RequestWebPage(ctx context.Context, u string, m string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) {
req, err := http.NewRequestWithContext(ctx, strings.ToUpper(m), u, body)
if err != nil {
return "", err
}
Expand Down
6 changes: 3 additions & 3 deletions net/http/http_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,22 +115,22 @@ func TestRequestWebPage(t *testing.T) {
}))
defer ts.Close()

resp, err := RequestWebPage(context.TODO(), ts.URL, nil, nil, &BasicAuth{name, pass})
resp, err := RequestWebPage(context.TODO(), ts.URL, "get", nil, nil, &BasicAuth{name, pass})
if err == nil || resp == succ {
t.Errorf("Failed to detect the bad request")
}

body := strings.NewReader(post)
var headers = map[string]string{hkey: name}
resp, err = RequestWebPage(context.TODO(), ts.URL, body, headers, &BasicAuth{name, pass})
resp, err = RequestWebPage(context.TODO(), ts.URL, "post", body, headers, &BasicAuth{name, pass})
if err != nil || resp != succ {
t.Errorf(resp)
}

ctx, cancel := context.WithCancel(context.Background())
cancel()

resp, err = RequestWebPage(ctx, ts.URL, nil, nil, nil)
resp, err = RequestWebPage(ctx, ts.URL, "get", nil, nil, nil)
if err == nil || resp != "" {
t.Errorf("Failed to detect the expired context")
}
Expand Down