Add Pagination support

.env.example

@@ -6,7 +6,6 @@ BITBUCKET_USERNAME=your_username
 BITBUCKET_PASSWORD=your_password
 
 USE_SSH_CLONING=false # Optional: default: false
-PAGING_LIMIT=100 # Optional: default: 100
 
 OUTPUT_DIR=./repos_zipped # Optional: default: ./repos_zipped
 CLONE_DIR=./repos_cloned # Optional: default: ./repos_cloned

bitbucket/bitbucketAPI.go

@@ -17,7 +17,9 @@ var client = http.Client{}
 
 // Outer wrapper for the JSON payload
 type RepositoryPayload struct {
-	Values []Repo `json:"values"`
+	Values        []Repo `json:"values"`
+	NextPageStart int    `json:"nextPageStart"`
+	IsLastPage    bool   `json:"isLastPage"`
 }
 
 func parseRepositoryPayloadJSON(jsonBytes []byte) (r RepositoryPayload) {
@@ -29,79 +31,68 @@ func parseRepositoryPayloadJSON(jsonBytes []byte) (r RepositoryPayload) {
 }
 
 func GetArchivedRepositories() ([]Repo, error) {
-	// Create a basic authentication header
-	auth := utils.Cfg.BitbucketUsername + ":" + utils.Cfg.BitbucketPassword
-	basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
-
-	bitbucketUrl := fmt.Sprintf("%s/rest/api/latest/repos?archived=ARCHIVED&limit=%d", utils.Cfg.BitbucketUrl, utils.Cfg.PagingLimit)
-
-	req, err := http.NewRequest("GET", bitbucketUrl, nil)
-	if err != nil {
-		log.WithError(err).Fatal("Error creating request")
-	}
-
-	// Set the Authorization header for basic authentication
-	req.Header.Add("Authorization", basicAuth)
-
-	// Send an HTTP GET request to the URL
-	resp, err := client.Do(req)
-	if err != nil {
-		log.WithError(err).Fatal("Error sending GET request")
-		return nil, err
-	}
-	defer resp.Body.Close()
+	return getPaginatedAPI(fmt.Sprintf("%s/rest/api/latest/repos?archived=ARCHIVED", utils.Cfg.BitbucketUrl))
+}
 
-	// Check if the response status code is 200 OK
-	if resp.StatusCode != http.StatusOK {
-		log.Error("Error: Unexpected status code:", resp.Status)
-		return nil, errors.New("Status Code: " + resp.Status)
-	}
+func GetRepositoriesForProject(projectKey string) ([]Repo, error) {
+	return getPaginatedAPI(fmt.Sprintf("%s/rest/api/latest/projects/%s/repos", utils.Cfg.BitbucketUrl, projectKey))
+}
 
-	// Read the response body
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.WithError(err).Fatal("Error reading response body")
-		return nil, err
+func getPaginatedAPI(url string) ([]Repo, error) {
+	isLastPage := false
+	currentPage := 0
+	var repos []Repo
+
+	for !isLastPage {
+		payload, err := getRepoPayload(url, currentPage)
+		if err != nil {
+			log.WithError(err).Fatal("Error getting repo payload")
+			return nil, err
+		}
+
+		repos = append(repos, payload.Values...)
+		isLastPage = payload.IsLastPage
+		currentPage = payload.NextPageStart
 	}
 
-	return parseRepositoryPayloadJSON(body).Values, nil
+	return repos, nil
 }
 
-func GetRepositoriesForProject(projectKey string) ([]Repo, error) {
-	// Create a basic authentication header
+func basicAuth() string {
 	auth := utils.Cfg.BitbucketUsername + ":" + utils.Cfg.BitbucketPassword
-	basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
-
-	bitbucketUrl := fmt.Sprintf("%s/rest/api/latest/projects/%s/repos?limit=%d", utils.Cfg.BitbucketUrl, projectKey, utils.Cfg.PagingLimit)
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
+}
 
+func getRepoPayload(url string, startPage int) (RepositoryPayload, error) {
+	bitbucketUrl := fmt.Sprintf("%s?limit=1000&start=%d", url, startPage)
+	log.Debugf("GET %s", bitbucketUrl)
 	req, err := http.NewRequest("GET", bitbucketUrl, nil)
 	if err != nil {
 		log.WithError(err).Fatal("Error creating request")
 	}
 
 	// Set the Authorization header for basic authentication
-	req.Header.Add("Authorization", basicAuth)
+	req.Header.Add("Authorization", basicAuth())
 
 	// Send an HTTP GET request to the URL
 	resp, err := client.Do(req)
 	if err != nil {
 		log.WithError(err).Fatal("Error sending GET request")
-		return nil, err
+		return RepositoryPayload{}, err
 	}
 	defer resp.Body.Close()
 
 	// Check if the response status code is 200 OK
 	if resp.StatusCode != http.StatusOK {
 		log.Error("Error: Unexpected status code:", resp.Status)
-		return nil, errors.New("Status Code: " + resp.Status)
+		return RepositoryPayload{}, errors.New("Status Code: " + resp.Status)
 	}
 
 	// Read the response body
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		log.WithError(err).Fatal("Error reading response body")
-		return nil, err
+		return RepositoryPayload{}, err
 	}
-
-	return parseRepositoryPayloadJSON(body).Values, nil
+	return parseRepositoryPayloadJSON(body), nil
 }

utils/cli.go

@@ -17,7 +17,6 @@ type Config struct {
 	BitbucketPassword string `env:"BITBUCKET_PASSWORD,notEmpty"`
 
 	UseSSHCloning bool `env:"USE_SSH_CLONING" envDefault:"false"`
-	PagingLimit   int  `env:"PAGING_LIMIT" envDefault:"100"`
 
 	OutputDir string `env:"OUTPUT_DIR" envDefault:"./repos_zipped"`
 	CloneDir  string `env:"CLONE_DIR" envDefault:"./repos_cloned"`