Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ENG-13851: Use repo scan API #68

Merged
merged 4 commits into from
Apr 30, 2024
Merged

ENG-13851: Use repo scan API #68

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
90ff513
Use repo scan API
ccampo133 Apr 24, 2024
893eeee
Fix tests
ccampo133 Apr 24, 2024
dbcbc15
Merge branch 'main' into ENG-13851
ccampo133 Apr 26, 2024
a07a1cf
Update API references
ccampo133 Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ type CLI struct {

// version is the application version. It is intended to be set at compile time
// via the linker (e.g. -ldflags="-X main.version=...").
var version string
var version = "dev"

func main() {
cli := CLI{Globals: Globals{}}
Expand Down
3 changes: 2 additions & 1 deletion cmd/repo_scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ func (cmd *RepoScanCmd) Run(_ *Globals) error {
// Publish the results to the Dmap API.
if cmd.RepoID != "" {
client := api.NewDmapClient(cmd.ApiBaseUrl, cmd.ClientID, cmd.ClientSecret)
if err := client.PublishRepoScanResults(ctx, cmd.RepoID, results); err != nil {
agent := "dmap-cli_" + version
if err := client.PublishRepoScanResults(ctx, agent, cmd.RepoID, results); err != nil {
return fmt.Errorf("error publishing results to Dmap API: %w", err)
}
}
Expand Down
51 changes: 42 additions & 9 deletions internal/api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package api
import (
"context"
"fmt"
"net/url"

"github.com/go-resty/resty/v2"
log "github.com/sirupsen/logrus"
Expand All @@ -12,19 +11,27 @@ import (
)

const (
repoScanPath = "/v1/reposcans"
dataLabelsPath = "/v1/datalabels"
classificationsPath = "/v1/repositories/{repoExternalID}/classifications"
classificationsPath = "/v1/classifications"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We might want to prefix this with a /reposcans/{repoScanID}, based on the comments in the dmap-backend PR.

)

type RepoScan struct {
RepoExternalId string `json:"repoExternalId"`
Agent string `json:"agent"`
}

// Classifications is the Dmap API representation of a set of data
// classifications.
type Classifications struct {
RepoScanId string `json:"repoScanId"`
Classifications []Classification `json:"classifications"`
}

// Labels is the Dmap API representation of a set of data labels.
type Labels struct {
Labels []Label `json:"labels"`
RepoScanId string `json:"repoScanId"`
Labels []Label `json:"labels"`
}

// Classification is the Dmap API representation of a single data
Expand Down Expand Up @@ -72,6 +79,31 @@ func NewDmapClient(baseURL, clientID, clientSecret string) *DmapClient {
return &DmapClient{client: client}
}

// CreateRepoScan creates a new repository scan in the Dmap API with the given
// repository scan details. If an error occurs, it is returned. The ID of the
// created repository scan is returned if the request is successful.
func (c *DmapClient) CreateRepoScan(ctx context.Context, repoScan RepoScan) (string, error) {
id := struct {
ID string `json:"id"`
}{}
resp, err := c.client.R().
SetContext(ctx).
SetBody(repoScan).
SetResult(&id).
Post(repoScanPath)
if err != nil || resp.IsError() {
if err == nil {
err = RequestError{
StatusCode: resp.StatusCode(),
Status: resp.Status(),
Body: resp.String(),
}
}
return "", fmt.Errorf("HTTP request error creating repo scan: %w", err)
}
return id.ID, nil
}

// UpsertLabels upserts the given labels to the Dmap API. All existing labels
// are replaced with the given labels. If an error occurs, it is returned.
func (c *DmapClient) UpsertLabels(ctx context.Context, labels Labels) error {
Expand Down Expand Up @@ -104,9 +136,6 @@ func (c *DmapClient) UpsertClassifications(
resp, err := c.client.R().
SetContext(ctx).
SetBody(classifications).
// We need to escape the repoExternalID because it may contain reserved
// characters like slashes.
SetPathParams(map[string]string{"repoExternalID": url.QueryEscape(repoExternalID)}).
Put(classificationsPath)
if err != nil || resp.IsError() {
if err == nil {
Expand All @@ -131,9 +160,13 @@ func (c *DmapClient) UpsertClassifications(
// the Dmap API if an error occurs.
func (c *DmapClient) PublishRepoScanResults(
ctx context.Context,
repoExternalID string,
agent, repoExternalID string,
results *scan.RepoScanResults,
) error {
repoScanId, err := c.CreateRepoScan(ctx, RepoScan{Agent: agent, RepoExternalId: repoExternalID})
if err != nil {
return fmt.Errorf("error creating repo scan: %w", err)
}
labels := make([]Label, len(results.Labels))
for i, label := range results.Labels {
labels[i] = Label{
Expand All @@ -142,7 +175,7 @@ func (c *DmapClient) PublishRepoScanResults(
Tags: label.Tags,
}
}
if err := c.UpsertLabels(ctx, Labels{Labels: labels}); err != nil {
if err := c.UpsertLabels(ctx, Labels{RepoScanId: repoScanId, Labels: labels}); err != nil {
return fmt.Errorf("error upserting labels: %w", err)
}
classifications := make([]Classification, 0, len(results.Classifications))
Expand All @@ -160,7 +193,7 @@ func (c *DmapClient) PublishRepoScanResults(
if err := c.UpsertClassifications(
ctx,
repoExternalID,
Classifications{Classifications: classifications},
Classifications{RepoScanId: repoScanId, Classifications: classifications},
); err != nil {
return fmt.Errorf("error upserting classifications for repo %s: %w", repoExternalID, err)
}
Expand Down
146 changes: 125 additions & 21 deletions internal/api/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,75 @@ import (
"github.com/cyralinc/dmap/scan"
)

func TestDmapClient_CreateRepoScan_Success(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
agent := "dmap-test"
repoExternalID := "arn:aws:dynamodb:us-east-1:123456789012:table/test"
wantRepoScan := RepoScan{
Agent: agent,
RepoExternalId: repoExternalID,
}
wantRepoScanId := "66284f0bf29b853e7db81bd4"
svr := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requireBasicAuth(t, r, clientID, clientSecret)
require.Equal(t, http.MethodPost, r.Method)
require.Equal(t, repoScanPath, r.URL.Path)
var rs RepoScan
err := json.NewDecoder(r.Body).Decode(&rs)
require.NoError(t, err)
require.Equal(t, wantRepoScan, rs)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"id":"%s"}`, wantRepoScanId)
},
),
)
defer svr.Close()
c := NewDmapClient(svr.URL, clientID, clientSecret)
id, err := c.CreateRepoScan(context.Background(), wantRepoScan)
require.NoError(t, err)
require.Equal(t, wantRepoScanId, id)
}

func TestDmapClient_CreateRepoScan_ServerError(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
svr := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requireBasicAuth(t, r, clientID, clientSecret)
require.Equal(t, http.MethodPost, r.Method)
require.Equal(t, repoScanPath, r.URL.Path)
w.WriteHeader(http.StatusInternalServerError)
},
),
)
defer svr.Close()
c := NewDmapClient(svr.URL, clientID, clientSecret)
id, err := c.CreateRepoScan(context.Background(), RepoScan{})
var reqErr RequestError
require.ErrorAs(t, err, &reqErr)
expectedErr := RequestError{
StatusCode: http.StatusInternalServerError,
Status: fmt.Sprintf(
"%d %s",
http.StatusInternalServerError,
http.StatusText(http.StatusInternalServerError),
),
}
require.Equal(t, expectedErr, reqErr)
require.Empty(t, id)
}

func TestDmapClient_CreateRepoScan_Error(t *testing.T) {
c := NewDmapClient("invalid", "clientID", "clientSecret")
id, err := c.CreateRepoScan(context.Background(), RepoScan{})
require.Error(t, err)
require.Empty(t, id)
}

func TestDmapClient_UpsertLabels_Success(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
Expand Down Expand Up @@ -163,6 +232,7 @@ func TestDmapClient_UpsertClassifications_Error(t *testing.T) {
func TestDmapClientPublishRepoScanResults_Success(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
agent := "dmap-test"
repoExternalID := "arn:aws:dynamodb:us-east-1:123456789012:table/test"
results := scan.RepoScanResults{
Labels: []classification.Label{
Expand All @@ -181,6 +251,10 @@ func TestDmapClientPublishRepoScanResults_Success(t *testing.T) {
},
},
}
wantRepoScan := RepoScan{
Agent: agent,
RepoExternalId: repoExternalID,
}
wantLbls := Labels{
Labels: []Label{
{
Expand All @@ -202,43 +276,51 @@ func TestDmapClientPublishRepoScanResults_Success(t *testing.T) {
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requireBasicAuth(t, r, clientID, clientSecret)
require.Equal(t, http.MethodPut, r.Method)
if r.URL.Path == dataLabelsPath {
switch r.URL.Path {
case repoScanPath:
require.Equal(t, http.MethodPost, r.Method)
var rs RepoScan
err := json.NewDecoder(r.Body).Decode(&rs)
require.NoError(t, err)
require.Equal(t, wantRepoScan, rs)
case dataLabelsPath:
require.Equal(t, http.MethodPut, r.Method)
var l Labels
err := json.NewDecoder(r.Body).Decode(&l)
require.NoError(t, err)
require.Equal(t, wantLbls, l)
} else {
path := strings.Replace(classificationsPath, "{repoExternalID}", url.QueryEscape(repoExternalID), 1)
if r.URL.Path != path {
t.Fatalf("unexpected path %s", r.URL.Path)
}
case classificationsPath:
require.Equal(t, http.MethodPut, r.Method)
var c Classifications
err := json.NewDecoder(r.Body).Decode(&c)
require.NoError(t, err)
require.Equal(t, wantClassifications, c)
default:
t.Fatalf("unexpected path %s", r.URL.Path)
}
},
),
)
defer svr.Close()
c := NewDmapClient(svr.URL, clientID, clientSecret)
err := c.PublishRepoScanResults(context.Background(), repoExternalID, &results)
err := c.PublishRepoScanResults(context.Background(), agent, repoExternalID, &results)
require.NoError(t, err)
}

func TestDmapClientPublishRepoScanResults_Error(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
agent := "dmap-test"
repoExternalID := "arn:aws:dynamodb:us-east-1:123456789012:table/test"
c := NewDmapClient("", clientID, clientSecret)
err := c.PublishRepoScanResults(context.Background(), repoExternalID, &scan.RepoScanResults{})
err := c.PublishRepoScanResults(context.Background(), agent, repoExternalID, &scan.RepoScanResults{})
require.Error(t, err)
}

func TestDmapClientPublishRepoScanResults_UpsertLabelsServerError(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
agent := "dmap-test"
repoExternalID := "arn:aws:dynamodb:us-east-1:123456789012:table/test"
results := scan.RepoScanResults{
Labels: []classification.Label{
Expand All @@ -257,22 +339,33 @@ func TestDmapClientPublishRepoScanResults_UpsertLabelsServerError(t *testing.T)
},
},
}
wantRepoScan := RepoScan{
Agent: agent,
RepoExternalId: repoExternalID,
}
svr := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requireBasicAuth(t, r, clientID, clientSecret)
require.Equal(t, http.MethodPut, r.Method)
if r.URL.Path == dataLabelsPath {
switch r.URL.Path {
case repoScanPath:
require.Equal(t, http.MethodPost, r.Method)
var rs RepoScan
err := json.NewDecoder(r.Body).Decode(&rs)
require.NoError(t, err)
require.Equal(t, wantRepoScan, rs)
case dataLabelsPath:
require.Equal(t, http.MethodPut, r.Method)
w.WriteHeader(http.StatusInternalServerError)
} else {
default:
t.Fatalf("unexpected path %s", r.URL.Path)
}
},
),
)
defer svr.Close()
c := NewDmapClient(svr.URL, clientID, clientSecret)
err := c.PublishRepoScanResults(context.Background(), repoExternalID, &results)
err := c.PublishRepoScanResults(context.Background(), agent, repoExternalID, &results)
var reqErr RequestError
require.ErrorAs(t, err, &reqErr)
expectedErr := RequestError{
Expand All @@ -289,6 +382,7 @@ func TestDmapClientPublishRepoScanResults_UpsertLabelsServerError(t *testing.T)
func TestPublishRepoScanResults_UpsertClassificationsServerError(t *testing.T) {
clientID := "clientID"
clientSecret := "clientSecret"
agent := "dmap-test"
repoExternalID := "arn:aws:dynamodb:us-east-1:123456789012:table/test"
results := scan.RepoScanResults{
Labels: []classification.Label{
Expand All @@ -307,6 +401,10 @@ func TestPublishRepoScanResults_UpsertClassificationsServerError(t *testing.T) {
},
},
}
wantRepoScan := RepoScan{
Agent: agent,
RepoExternalId: repoExternalID,
}
wantLbls := Labels{
Labels: []Label{
{
Expand All @@ -320,25 +418,31 @@ func TestPublishRepoScanResults_UpsertClassificationsServerError(t *testing.T) {
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requireBasicAuth(t, r, clientID, clientSecret)
require.Equal(t, http.MethodPut, r.Method)
if r.URL.Path == dataLabelsPath {
switch r.URL.Path {
case repoScanPath:
require.Equal(t, http.MethodPost, r.Method)
var rs RepoScan
err := json.NewDecoder(r.Body).Decode(&rs)
require.NoError(t, err)
require.Equal(t, wantRepoScan, rs)
case dataLabelsPath:
require.Equal(t, http.MethodPut, r.Method)
var l Labels
err := json.NewDecoder(r.Body).Decode(&l)
require.NoError(t, err)
require.Equal(t, wantLbls, l)
} else {
path := strings.Replace(classificationsPath, "{repoExternalID}", url.QueryEscape(repoExternalID), 1)
if r.URL.Path != path {
t.Fatalf("unexpected path %s", r.URL.Path)
}
case classificationsPath:
require.Equal(t, http.MethodPut, r.Method)
w.WriteHeader(http.StatusInternalServerError)
default:
t.Fatalf("unexpected path %s", r.URL.Path)
}
},
),
)
defer svr.Close()
c := NewDmapClient(svr.URL, clientID, clientSecret)
err := c.PublishRepoScanResults(context.Background(), repoExternalID, &results)
err := c.PublishRepoScanResults(context.Background(), agent, repoExternalID, &results)
var reqErr RequestError
require.ErrorAs(t, err, &reqErr)
expectedErr := RequestError{
Expand Down