feat(retry): Add retry logic (#146)

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
+	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect

go.sum

@@ -30,6 +30,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE=
+github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
+github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=

pkg/common/utils/http.go

@@ -12,6 +12,23 @@ import (
 
 const httpGetTimeout = 60 * time.Second
 
+// HTTPGetWithRetry returns the body of the HTTP Get response. Retries if the call fails.
+func HTTPGetWithRetry(provider, url string) ([]byte, error) {
+	var body []byte
+	retryErr := WithDefaultRetry(func() error {
+		var err error
+		body, err = HTTPGet(url)
+		if err != nil {
+			return errors.Wrapf(err, "failed to fetch networks from %s with URL: %s", provider, url)
+		}
+		return nil
+	})
+	if retryErr != nil {
+		return nil, retryErr
+	}
+	return body, nil
+}
+
 // HTTPGet returns the body of the HTTP GET response
 func HTTPGet(url string) ([]byte, error) {
 	log.Printf("Getting from URL: %s...", url)

pkg/common/utils/retry.go

@@ -0,0 +1,25 @@
+package utils
+
+import (
+	"github.com/cenkalti/backoff/v3"
+	"log"
+	"time"
+)
+
+// WithDefaultRetry retries a given function with a default exponential backoff
+func WithDefaultRetry(do func() error) error {
+	return WithRetry(do, 2*time.Second, 10*time.Second, 5*time.Minute)
+}
+
+// WithRetry retries a given function with an exponential backoff until maxTime is reached
+func WithRetry(do func() error, interval, maxInterval, maxTime time.Duration) error {
+	exponential := backoff.NewExponentialBackOff()
+	exponential.MaxElapsedTime = maxTime
+	exponential.InitialInterval = interval
+	exponential.MaxInterval = maxInterval
+
+	err := backoff.RetryNotify(do, exponential, func(err error, d time.Duration) {
+		log.Printf("call failed, retrying in %s. Error: %v", d.Round(time.Second), err)
+	})
+	return err
+}

pkg/crawlers/aws/aws.go

@@ -67,11 +67,7 @@ func (c *awsNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR
 }
 
 func (c *awsNetworkCrawler) fetch() ([]byte, error) {
-	body, err := utils.HTTPGet(c.url)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to fetch networks from Google")
-	}
-	return body, nil
+	return utils.HTTPGetWithRetry("Amazon", c.url)
 }
 
 func (c *awsNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) {

pkg/crawlers/azure/azure.go

@@ -3,12 +3,11 @@ package azure
 import (
 	"encoding/json"
 	"fmt"
-	"log"
-	"os/exec"
-
 	"github.com/pkg/errors"
 	"github.com/stackrox/external-network-pusher/pkg/common"
 	"github.com/stackrox/external-network-pusher/pkg/common/utils"
+	"log"
+	"os/exec"
 )
 
 // With Microsoft, it is a little different in a sense that: it has four
@@ -278,23 +277,31 @@ func (c *azureNetworkCrawler) fetchAll() ([][]byte, error) {
 	// and the page then renders generated URLs to json files.
 	jsonURLs := make([]string, 0, len(c.urls))
 	for _, url := range c.urls {
-		jsonURL, err := c.redirectToJSONURL(url)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to crawl Azure with URL: %s. Error: %v. JSON URL: %s", url, err, jsonURL)
-		}
-		if jsonURL == "" {
-			return nil, errors.Errorf("failed to crawl Azure with URL: %s, empty JSON URL returned. This could indicate Azure's services are unavailable", url)
+		var jsonURL string
+		retryErr := utils.WithDefaultRetry(func() error {
+			var err error
+			jsonURL, err = c.redirectToJSONURL(url)
+			if err != nil {
+				return errors.Wrapf(err, "failed to crawl Azure with URL: %s. Error: %v. JSON URL: %s", url, err, jsonURL)
+			}
+			if jsonURL == "" {
+				return errors.Errorf("failed to crawl Azure with URL: %s, empty JSON URL returned. This could indicate Azure's services are unavailable", url)
+			}
+			return nil
+		})
+		if retryErr != nil {
+			return nil, retryErr
 		}
-		log.Printf("Received Azure network JSON URL: %s", jsonURL)
+		log.Printf("Success obtaining Azure network JSON URL %q from %q", jsonURL, url)
 		jsonURLs = append(jsonURLs, jsonURL)
 	}
 
 	contents := make([][]byte, 0, len(jsonURLs))
 	for _, jsonURL := range jsonURLs {
 		log.Printf("Current URL is: %s", jsonURL)
-		body, err := utils.HTTPGet(jsonURL)
+		body, err := utils.HTTPGetWithRetry("Azure", jsonURL)
 		if err != nil {
-			return nil, errors.Wrapf(err, "Failed to fetch networks from Azure with URL: %s. Error: %v", jsonURL, err)
+			return nil, err
 		}
 		contents = append(contents, body)
 	}
@@ -318,7 +325,7 @@ func (c *azureNetworkCrawler) redirectToJSONURL(rawURL string) (string, error) {
 		rawURL)
 	out, err := exec.Command("/bin/sh", "-c", cmd).Output()
 	if err != nil {
-		errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL)
+		err = errors.Wrapf(err, "failed to redirect to JSON URL %q while trying to crawl Azure with URL", rawURL)
 		return "", err
 	}
 	return string(out), nil

pkg/crawlers/cloudflare/cloudflare.go

@@ -60,11 +60,7 @@ func (c *cloudflareNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderN
 }
 
 func (c *cloudflareNetworkCrawler) fetch() ([]byte, error) {
-	body, err := utils.HTTPGet(c.url)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to fetch networks from Cloudflare with URL: %s", c.url)
-	}
-	return body, nil
+	return utils.HTTPGetWithRetry("Cloudflare", c.url)
 }
 
 func (c *cloudflareNetworkCrawler) parseNetworks(networks []byte) (*common.ProviderNetworkRanges, error) {

pkg/crawlers/gcp/gcp.go

@@ -58,11 +58,7 @@ func (c *gcpNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR
 }
 
 func (c *gcpNetworkCrawler) fetch() ([]byte, error) {
-	body, err := utils.HTTPGet(c.url)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to fetch networks from Google")
-	}
-	return body, nil
+	return utils.HTTPGetWithRetry("Google", c.url)
 }
 
 func (c *gcpNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) {

pkg/crawlers/oracle/oracle.go

@@ -61,11 +61,7 @@ func (c *ociNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR
 }
 
 func (c *ociNetworkCrawler) fetch() ([]byte, error) {
-	body, err := utils.HTTPGet(c.url)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to fetch networks from Oracle")
-	}
-	return body, nil
+	return utils.HTTPGetWithRetry("Oracle", c.url)
 }
 
 func (c *ociNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) {