diff --git a/config/resolvers.go b/config/resolvers.go index 175fd9a93..885401713 100644 --- a/config/resolvers.go +++ b/config/resolvers.go @@ -53,7 +53,7 @@ var PublicResolvers []string // GetPublicDNSResolvers obtains the public DNS server addresses from public-dns.info and assigns them to PublicResolvers. func GetPublicDNSResolvers() error { url := "https://public-dns.info/nameservers-all.csv" - page, err := http.RequestWebPage(context.Background(), url, nil, nil, nil) + page, err := http.RequestWebPage(context.Background(), url, "get", nil, nil, nil) if err != nil { return fmt.Errorf("failed to obtain the Public DNS csv file at %s: %v", url, err) } diff --git a/datasrcs/alienvault.go b/datasrcs/alienvault.go index b90a8c3b5..8cebbf2c9 100644 --- a/datasrcs/alienvault.go +++ b/datasrcs/alienvault.go @@ -104,7 +104,7 @@ func (a *AlienVault) executeDNSQuery(ctx context.Context, req *requests.DNSReque } u := a.getURL(req.Domain) + "passive_dns" - page, err := http.RequestWebPage(ctx, u, nil, a.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, a.getHeaders(), nil) if err != nil { a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err) return @@ -173,7 +173,7 @@ func (a *AlienVault) executeURLQuery(ctx context.Context, req *requests.DNSReque headers := a.getHeaders() u := a.getURL(req.Domain) + "url_list" - page, err := http.RequestWebPage(ctx, u, nil, headers, nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, headers, nil) if err != nil { a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err) return @@ -208,7 +208,7 @@ func (a *AlienVault) executeURLQuery(ctx context.Context, req *requests.DNSReque for cur := m.PageNum + 1; cur <= pages; cur++ { a.CheckRateLimit() pageURL := u + "?page=" + strconv.Itoa(cur) - page, err = http.RequestWebPage(ctx, pageURL, nil, headers, nil) + page, err = http.RequestWebPage(ctx, pageURL, "get", nil, headers, nil) if err != nil { a.sys.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) break @@ -263,7 +263,7 @@ func (a *AlienVault) executeWhoisQuery(ctx context.Context, req *requests.WhoisR headers := a.getHeaders() for _, email := range emails { pageURL := a.getReverseWhoisURL(email) - page, err := http.RequestWebPage(ctx, pageURL, nil, headers, nil) + page, err := http.RequestWebPage(ctx, pageURL, "get", nil, headers, nil) if err != nil { a.sys.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) continue @@ -303,7 +303,7 @@ func (a *AlienVault) queryWhoisForEmails(ctx context.Context, req *requests.Whoi defer emails.Close() u := a.getWhoisURL(req.Domain) - page, err := http.RequestWebPage(ctx, u, nil, a.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, a.getHeaders(), nil) if err != nil { a.sys.Config().Log.Printf("%s: %s: %v", a.String(), u, err) return emails.Slice() diff --git a/datasrcs/dnsdb.go b/datasrcs/dnsdb.go index 999bde221..5e5e2bc83 100644 --- a/datasrcs/dnsdb.go +++ b/datasrcs/dnsdb.go @@ -104,7 +104,7 @@ func (d *DNSDB) dnsRequest(ctx context.Context, req *requests.DNSRequest) { } url := d.getURL(req.Domain) - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { d.sys.Config().Log.Printf("%s: %s: %v", d.String(), url, err) return diff --git a/datasrcs/networksdb.go b/datasrcs/networksdb.go index c4d9aebbe..8aee2441d 100644 --- a/datasrcs/networksdb.go +++ b/datasrcs/networksdb.go @@ -127,7 +127,7 @@ func (n *NetworksDB) asnRequest(ctx context.Context, req *requests.ASNRequest) { func (n *NetworksDB) executeASNAddrQuery(ctx context.Context, addr string) { u := n.getIPURL(addr) - page, err := http.RequestWebPage(ctx, u, nil, nil, nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return @@ -141,7 +141,7 @@ func (n *NetworksDB) executeASNAddrQuery(ctx context.Context, addr string) { numRateLimitChecks(n, 3) u = networksdbBaseURL + matches[1] - page, err = http.RequestWebPage(ctx, u, nil, nil, nil) + page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return @@ -178,7 +178,7 @@ func (n *NetworksDB) getIPURL(addr string) string { func (n *NetworksDB) executeASNQuery(ctx context.Context, asn int, addr string, netblocks *stringset.Set) { numRateLimitChecks(n, 3) u := n.getASNURL(asn) - page, err := http.RequestWebPage(ctx, u, nil, nil, nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return @@ -331,7 +331,7 @@ func (n *NetworksDB) apiIPQuery(ctx context.Context, addr string) (string, strin u := n.getAPIIPURL() params := url.Values{"ip": {addr}} body := strings.NewReader(params.Encode()) - page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return "", "" @@ -372,7 +372,7 @@ func (n *NetworksDB) apiOrgInfoQuery(ctx context.Context, id string) []int { u := n.getAPIOrgInfoURL() params := url.Values{"id": {id}} body := strings.NewReader(params.Encode()) - page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return []int{} @@ -408,7 +408,7 @@ func (n *NetworksDB) apiASNInfoQuery(ctx context.Context, asn int) *requests.ASN u := n.getAPIASNInfoURL() params := url.Values{"asn": {strconv.Itoa(asn)}} body := strings.NewReader(params.Encode()) - page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return nil @@ -456,7 +456,7 @@ func (n *NetworksDB) apiNetblocksQuery(ctx context.Context, asn int) *stringset. u := n.getAPINetblocksURL() params := url.Values{"asn": {strconv.Itoa(asn)}} body := strings.NewReader(params.Encode()) - page, err := http.RequestWebPage(ctx, u, body, n.getHeaders(), nil) + page, err := http.RequestWebPage(ctx, u, "post", body, n.getHeaders(), nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return netblocks @@ -508,7 +508,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques numRateLimitChecks(n, 2) u := n.getDomainToIPURL(req.Domain) - page, err := http.RequestWebPage(ctx, u, nil, nil, nil) + page, err := http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) return @@ -531,7 +531,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques numRateLimitChecks(n, 3) u = networksdbBaseURL + match[1] - page, err = http.RequestWebPage(ctx, u, nil, nil, nil) + page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) continue @@ -552,7 +552,7 @@ func (n *NetworksDB) whoisRequest(ctx context.Context, req *requests.WhoisReques first, last := amassnet.FirstLast(cidr) u := n.getDomainsInNetworkURL(first.String(), last.String()) - page, err = http.RequestWebPage(ctx, u, nil, nil, nil) + page, err = http.RequestWebPage(ctx, u, "get", nil, nil, nil) if err != nil { n.sys.Config().Log.Printf("%s: %s: %v", n.String(), u, err) continue diff --git a/datasrcs/radb.go b/datasrcs/radb.go index e5060f187..9c9b5b635 100644 --- a/datasrcs/radb.go +++ b/datasrcs/radb.go @@ -118,7 +118,7 @@ func (r *RADb) asnRequest(ctx context.Context, req *requests.ASNRequest) { func (r *RADb) executeASNAddrQuery(ctx context.Context, addr string) { url := r.getIPURL("arin", addr) headers := map[string]string{"Content-Type": "application/json"} - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) return @@ -172,7 +172,7 @@ func (r *RADb) executeASNQuery(ctx context.Context, asn int, addr, prefix string numRateLimitChecks(r, 2) url := r.getASNURL("arin", strconv.Itoa(asn)) headers := map[string]string{"Content-Type": "application/json"} - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) return @@ -251,7 +251,7 @@ func (r *RADb) netblocks(ctx context.Context, asn int) *stringset.Set { numRateLimitChecks(r, 2) url := r.getNetblocksURL(strconv.Itoa(asn)) headers := map[string]string{"Content-Type": "application/json"} - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) return netblocks diff --git a/datasrcs/scripting/http.go b/datasrcs/scripting/http.go index c460d5d61..33e0be81a 100644 --- a/datasrcs/scripting/http.go +++ b/datasrcs/scripting/http.go @@ -29,13 +29,6 @@ func (s *Script) request(L *lua.LState) int { return 2 } - var data string - if method, ok := getStringField(L, opt, "method"); ok && strings.ToLower(method) == "post" { - if d, ok := getStringField(L, opt, "data"); ok { - data = d - } - } - url, found := getStringField(L, opt, "url") if !found { L.Push(lua.LNil) @@ -43,6 +36,12 @@ func (s *Script) request(L *lua.LState) int { return 2 } + method, ok := getStringField(L, opt, "method") + if !ok { + method = "get" + } + data, _ := getStringField(L, opt, "data") + headers := make(map[string]string) lv := L.GetField(opt, "headers") if tbl, ok := lv.(*lua.LTable); ok { @@ -53,7 +52,7 @@ func (s *Script) request(L *lua.LState) int { id, _ := getStringField(L, opt, "id") pass, _ := getStringField(L, opt, "pass") - page, err := s.req(ctx, url, data, headers, &http.BasicAuth{ + page, err := s.req(ctx, url, method, data, headers, &http.BasicAuth{ Username: id, Password: pass, }) @@ -81,19 +80,18 @@ func (s *Script) scrape(L *lua.LState) int { return 1 } - var data string - if method, ok := getStringField(L, opt, "method"); ok && strings.ToLower(method) == "post" { - if d, ok := getStringField(L, opt, "data"); ok { - data = d - } - } - url, found := getStringField(L, opt, "url") if !found { L.Push(lua.LFalse) return 1 } + method, ok := getStringField(L, opt, "method") + if !ok { + method = "get" + } + data, _ := getStringField(L, opt, "data") + headers := make(map[string]string) lv := L.GetField(opt, "headers") if tbl, ok := lv.(*lua.LTable); ok { @@ -106,7 +104,7 @@ func (s *Script) scrape(L *lua.LState) int { pass, _ := getStringField(L, opt, "pass") sucess := lua.LFalse - if resp, err := s.req(ctx, url, data, headers, &http.BasicAuth{ + if resp, err := s.req(ctx, url, method, data, headers, &http.BasicAuth{ Username: id, Password: pass, }); err == nil { @@ -121,7 +119,7 @@ func (s *Script) scrape(L *lua.LState) int { return 1 } -func (s *Script) req(ctx context.Context, url, data string, headers map[string]string, auth *http.BasicAuth) (string, error) { +func (s *Script) req(ctx context.Context, url, method string, data string, headers map[string]string, auth *http.BasicAuth) (string, error) { cfg := s.sys.Config() // Check for cached responses first dsc := cfg.GetDataSourceConfig(s.String()) @@ -137,7 +135,7 @@ func (s *Script) req(ctx context.Context, url, data string, headers map[string]s } numRateLimitChecks(s, s.seconds) - resp, err := http.RequestWebPage(ctx, url, body, headers, auth) + resp, err := http.RequestWebPage(ctx, url, method, body, headers, auth) if err != nil { if cfg.Verbose { cfg.Log.Printf("%s: %s: %v", s.String(), url, err) diff --git a/datasrcs/twitter.go b/datasrcs/twitter.go index 1aa971a7f..f79076b9c 100644 --- a/datasrcs/twitter.go +++ b/datasrcs/twitter.go @@ -131,7 +131,7 @@ func (t *Twitter) dnsRequest(ctx context.Context, req *requests.DNSRequest) { func (t *Twitter) getBearerToken() (string, error) { headers := map[string]string{"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"} - page, err := http.RequestWebPage(context.Background(), "https://api.twitter.com/oauth2/token", + page, err := http.RequestWebPage(context.Background(), "https://api.twitter.com/oauth2/token", "post", strings.NewReader("grant_type=client_credentials"), headers, &http.BasicAuth{ Username: t.creds.Key, diff --git a/datasrcs/umbrella.go b/datasrcs/umbrella.go index 534671ac0..d3c2c1b87 100644 --- a/datasrcs/umbrella.go +++ b/datasrcs/umbrella.go @@ -109,7 +109,7 @@ func (u *Umbrella) dnsRequest(ctx context.Context, req *requests.DNSRequest) { headers := u.restHeaders() url := u.restDNSURL(req.Domain) - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err) return @@ -138,7 +138,7 @@ func (u *Umbrella) addrRequest(ctx context.Context, req *requests.AddrRequest) { headers := u.restHeaders() url := u.restAddrURL(req.Address) - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err) return @@ -177,7 +177,7 @@ func (u *Umbrella) asnRequest(ctx context.Context, req *requests.ASNRequest) { func (u *Umbrella) executeASNAddrQuery(ctx context.Context, req *requests.ASNRequest) { headers := u.restHeaders() url := u.restAddrToASNURL(req.Address) - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err) return @@ -235,7 +235,7 @@ func (u *Umbrella) executeASNAddrQuery(ctx context.Context, req *requests.ASNReq func (u *Umbrella) executeASNQuery(ctx context.Context, req *requests.ASNRequest) { headers := u.restHeaders() url := u.restASNToCIDRsURL(req.ASN) - page, err := http.RequestWebPage(ctx, url, nil, headers, nil) + page, err := http.RequestWebPage(ctx, url, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), url, err) return @@ -333,7 +333,7 @@ func (u *Umbrella) queryWhois(ctx context.Context, domain string) *whoisRecord { whoisURL := u.whoisRecordURL(domain) u.CheckRateLimit() - record, err := http.RequestWebPage(ctx, whoisURL, nil, headers, nil) + record, err := http.RequestWebPage(ctx, whoisURL, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err) return nil @@ -357,7 +357,7 @@ func (u *Umbrella) queryReverseWhois(ctx context.Context, apiURL string) []strin for count, more := 0, true; more; count = count + 500 { u.CheckRateLimit() fullAPIURL := fmt.Sprintf("%s&offset=%d", apiURL, count) - record, err := http.RequestWebPage(ctx, fullAPIURL, nil, headers, nil) + record, err := http.RequestWebPage(ctx, fullAPIURL, "get", nil, headers, nil) if err != nil { u.sys.Config().Log.Printf("%s: %s: %v", u.String(), apiURL, err) return domains.Slice() diff --git a/net/http/http.go b/net/http/http.go index 451b44883..649d98b03 100644 --- a/net/http/http.go +++ b/net/http/http.go @@ -110,13 +110,8 @@ func CheckCookie(urlString string, cookieName string) bool { } // RequestWebPage returns a string containing the entire response for the provided URL when successful. -func RequestWebPage(ctx context.Context, u string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) { - method := "GET" - if body != nil { - method = "POST" - } - - req, err := http.NewRequestWithContext(ctx, method, u, body) +func RequestWebPage(ctx context.Context, u string, m string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) { + req, err := http.NewRequestWithContext(ctx, strings.ToUpper(m), u, body) if err != nil { return "", err } diff --git a/net/http/http_test.go b/net/http/http_test.go index c73d75de4..7aba908d8 100644 --- a/net/http/http_test.go +++ b/net/http/http_test.go @@ -115,14 +115,14 @@ func TestRequestWebPage(t *testing.T) { })) defer ts.Close() - resp, err := RequestWebPage(context.TODO(), ts.URL, nil, nil, &BasicAuth{name, pass}) + resp, err := RequestWebPage(context.TODO(), ts.URL, "get", nil, nil, &BasicAuth{name, pass}) if err == nil || resp == succ { t.Errorf("Failed to detect the bad request") } body := strings.NewReader(post) var headers = map[string]string{hkey: name} - resp, err = RequestWebPage(context.TODO(), ts.URL, body, headers, &BasicAuth{name, pass}) + resp, err = RequestWebPage(context.TODO(), ts.URL, "post", body, headers, &BasicAuth{name, pass}) if err != nil || resp != succ { t.Errorf(resp) } @@ -130,7 +130,7 @@ func TestRequestWebPage(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - resp, err = RequestWebPage(ctx, ts.URL, nil, nil, nil) + resp, err = RequestWebPage(ctx, ts.URL, "get", nil, nil, nil) if err == nil || resp != "" { t.Errorf("Failed to detect the expired context") }