diff --git a/.env.example b/.env.example index 6438190..36c8f7e 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,6 @@ BITBUCKET_USERNAME=your_username BITBUCKET_PASSWORD=your_password USE_SSH_CLONING=false # Optional: default: false -PAGING_LIMIT=100 # Optional: default: 100 OUTPUT_DIR=./repos_zipped # Optional: default: ./repos_zipped CLONE_DIR=./repos_cloned # Optional: default: ./repos_cloned diff --git a/bitbucket/bitbucketAPI.go b/bitbucket/bitbucketAPI.go index 59f3fa6..7af5fbc 100644 --- a/bitbucket/bitbucketAPI.go +++ b/bitbucket/bitbucketAPI.go @@ -17,7 +17,9 @@ var client = http.Client{} // Outer wrapper for the JSON payload type RepositoryPayload struct { - Values []Repo `json:"values"` + Values []Repo `json:"values"` + NextPageStart int `json:"nextPageStart"` + IsLastPage bool `json:"isLastPage"` } func parseRepositoryPayloadJSON(jsonBytes []byte) (r RepositoryPayload) { @@ -29,79 +31,68 @@ func parseRepositoryPayloadJSON(jsonBytes []byte) (r RepositoryPayload) { } func GetArchivedRepositories() ([]Repo, error) { - // Create a basic authentication header - auth := utils.Cfg.BitbucketUsername + ":" + utils.Cfg.BitbucketPassword - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - - bitbucketUrl := fmt.Sprintf("%s/rest/api/latest/repos?archived=ARCHIVED&limit=%d", utils.Cfg.BitbucketUrl, utils.Cfg.PagingLimit) - - req, err := http.NewRequest("GET", bitbucketUrl, nil) - if err != nil { - log.WithError(err).Fatal("Error creating request") - } - - // Set the Authorization header for basic authentication - req.Header.Add("Authorization", basicAuth) - - // Send an HTTP GET request to the URL - resp, err := client.Do(req) - if err != nil { - log.WithError(err).Fatal("Error sending GET request") - return nil, err - } - defer resp.Body.Close() + return getPaginatedAPI(fmt.Sprintf("%s/rest/api/latest/repos?archived=ARCHIVED", utils.Cfg.BitbucketUrl)) +} - // Check if the response status code is 200 OK - if resp.StatusCode != http.StatusOK { - log.Error("Error: Unexpected status code:", resp.Status) - return nil, errors.New("Status Code: " + resp.Status) - } +func GetRepositoriesForProject(projectKey string) ([]Repo, error) { + return getPaginatedAPI(fmt.Sprintf("%s/rest/api/latest/projects/%s/repos", utils.Cfg.BitbucketUrl, projectKey)) +} - // Read the response body - body, err := io.ReadAll(resp.Body) - if err != nil { - log.WithError(err).Fatal("Error reading response body") - return nil, err +func getPaginatedAPI(url string) ([]Repo, error) { + isLastPage := false + currentPage := 0 + var repos []Repo + + for !isLastPage { + payload, err := getRepoPayload(url, currentPage) + if err != nil { + log.WithError(err).Fatal("Error getting repo payload") + return nil, err + } + + repos = append(repos, payload.Values...) + isLastPage = payload.IsLastPage + currentPage = payload.NextPageStart } - return parseRepositoryPayloadJSON(body).Values, nil + return repos, nil } -func GetRepositoriesForProject(projectKey string) ([]Repo, error) { - // Create a basic authentication header +func basicAuth() string { auth := utils.Cfg.BitbucketUsername + ":" + utils.Cfg.BitbucketPassword - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - - bitbucketUrl := fmt.Sprintf("%s/rest/api/latest/projects/%s/repos?limit=%d", utils.Cfg.BitbucketUrl, projectKey, utils.Cfg.PagingLimit) + return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) +} +func getRepoPayload(url string, startPage int) (RepositoryPayload, error) { + bitbucketUrl := fmt.Sprintf("%s?limit=1000&start=%d", url, startPage) + log.Debugf("GET %s", bitbucketUrl) req, err := http.NewRequest("GET", bitbucketUrl, nil) if err != nil { log.WithError(err).Fatal("Error creating request") } // Set the Authorization header for basic authentication - req.Header.Add("Authorization", basicAuth) + req.Header.Add("Authorization", basicAuth()) // Send an HTTP GET request to the URL resp, err := client.Do(req) if err != nil { log.WithError(err).Fatal("Error sending GET request") - return nil, err + return RepositoryPayload{}, err } defer resp.Body.Close() // Check if the response status code is 200 OK if resp.StatusCode != http.StatusOK { log.Error("Error: Unexpected status code:", resp.Status) - return nil, errors.New("Status Code: " + resp.Status) + return RepositoryPayload{}, errors.New("Status Code: " + resp.Status) } // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { log.WithError(err).Fatal("Error reading response body") - return nil, err + return RepositoryPayload{}, err } - - return parseRepositoryPayloadJSON(body).Values, nil + return parseRepositoryPayloadJSON(body), nil } diff --git a/utils/cli.go b/utils/cli.go index de2031f..6c29ee7 100644 --- a/utils/cli.go +++ b/utils/cli.go @@ -17,7 +17,6 @@ type Config struct { BitbucketPassword string `env:"BITBUCKET_PASSWORD,notEmpty"` UseSSHCloning bool `env:"USE_SSH_CLONING" envDefault:"false"` - PagingLimit int `env:"PAGING_LIMIT" envDefault:"100"` OutputDir string `env:"OUTPUT_DIR" envDefault:"./repos_zipped"` CloneDir string `env:"CLONE_DIR" envDefault:"./repos_cloned"`