Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use map for scan results #23

Merged
merged 1 commit into from
Feb 15, 2024
Merged

Use map for scan results #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 16 additions & 26 deletions aws/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type scanFunction func(
) scanResponse

type scanResponse struct {
repositories []scan.Repository
repositories map[string]scan.Repository
scanErrors []error
}

Expand All @@ -33,12 +33,10 @@ func scanRDSClusterRepositories(
),
)
}
repositories := make([]scan.Repository, 0, len(rdsClusters))
repositories := make(map[string]scan.Repository, len(rdsClusters))
for _, cluster := range rdsClusters {
repositories = append(
repositories,
newRepositoryFromRDSCluster(cluster),
)
repo := newRepositoryFromRDSCluster(cluster)
repositories[repo.Id] = repo
}
return scanResponse{
repositories: repositories,
Expand All @@ -62,15 +60,13 @@ func scanRDSInstanceRepositories(
),
)
}
repositories := make([]scan.Repository, 0, len(rdsInstances))
repositories := make(map[string]scan.Repository, len(rdsInstances))
for _, instance := range rdsInstances {
// Skip cluster instances, since they were already added when retrieving
// the RDS clusters.
if instance.DBClusterIdentifier == nil {
repositories = append(
repositories,
newRepositoryFromRDSInstance(instance),
)
repo := newRepositoryFromRDSInstance(instance)
repositories[repo.Id] = repo
}
}
return scanResponse{
Expand All @@ -95,12 +91,10 @@ func scanRedshiftRepositories(
),
)
}
repositories := make([]scan.Repository, 0, len(redshiftClusters))
repositories := make(map[string]scan.Repository, len(redshiftClusters))
for _, cluster := range redshiftClusters {
repositories = append(
repositories,
newRepositoryFromRedshiftCluster(cluster),
)
repo := newRepositoryFromRedshiftCluster(cluster)
repositories[repo.Id] = repo
}
return scanResponse{
repositories: repositories,
Expand All @@ -124,12 +118,10 @@ func scanDynamoDBRepositories(
),
)
}
repositories := make([]scan.Repository, 0, len(dynamodbTables))
repositories := make(map[string]scan.Repository, len(dynamodbTables))
for _, table := range dynamodbTables {
repositories = append(
repositories,
newRepositoryFromDynamoDBTable(table),
)
repo := newRepositoryFromDynamoDBTable(table)
repositories[repo.Id] = repo
}
return scanResponse{
repositories: repositories,
Expand All @@ -154,12 +146,10 @@ func scanS3Buckets(
),
)
}
repos := make([]scan.Repository, 0, len(buckets))
repos := make(map[string]scan.Repository, len(buckets))
for _, bucket := range buckets {
repos = append(
repos,
newRepositoryFromS3Bucket(bucket),
)
repo := newRepositoryFromS3Bucket(bucket)
repos[repo.Id] = repo
}
return scanResponse{
repositories: repos,
Expand Down
16 changes: 7 additions & 9 deletions aws/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"context"
"fmt"
"maps"
"sync"

"github.com/aws/aws-sdk-go-v2/aws"
Expand Down Expand Up @@ -57,9 +58,6 @@ func NewAWSScanner(
// results, containing a list of data repositories that includes: RDS clusters
// and instances, Redshift clusters and DynamoDB tables.
func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) {
repositories := []scan.Repository{}
var scanErrors []error

responseChan := make(chan scanResponse)
var wg sync.WaitGroup
wg.Add(len(s.scannerConfig.Regions))
Expand Down Expand Up @@ -96,6 +94,8 @@ func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) {
close(responseChan)
}()

repositories := make(map[string]scan.Repository)
var scanErrors []error
for {
select {
case <-ctx.Done():
Expand All @@ -116,7 +116,7 @@ func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) {
}, scanErr

}
repositories = append(repositories, response.repositories...)
maps.Copy(repositories, response.repositories)
scanErrors = append(scanErrors, response.scanErrors...)
}
}
Expand All @@ -129,9 +129,6 @@ func scanRegion(
newAWSClient awsClientConstructor,
scanFunctions []scanFunction,
) scanResponse {
repositories := []scan.Repository{}
var scanErrors []error

awsConfig.Region = region
awsClient := newAWSClient(awsConfig)

Expand Down Expand Up @@ -159,6 +156,8 @@ func scanRegion(
close(responseChan)
}()

repositories := make(map[string]scan.Repository)
var scanErrors []error
for {
select {
case <-ctx.Done():
Expand All @@ -176,8 +175,7 @@ func scanRegion(
scanErrors: scanErrors,
}
}

repositories = append(repositories, response.repositories...)
maps.Copy(repositories, response.repositories)
scanErrors = append(scanErrors, response.scanErrors...)
}
}
Expand Down
66 changes: 34 additions & 32 deletions aws/scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,64 +37,71 @@ type AWSScannerTestSuite struct {
func (s *AWSScannerTestSuite) SetupSuite() {
s.dummyRDSClusters = []rdsTypes.DBCluster{
{
DBClusterArn: aws.String("dummy-arn-1"),
DBClusterArn: aws.String("dummy-rds-cluster-arn-1"),
DBClusterIdentifier: aws.String("rds-cluster-1"),
},
{
DBClusterArn: aws.String("dummy-arn-2"),
DBClusterArn: aws.String("dummy-rds-cluster-arn-2"),
DBClusterIdentifier: aws.String("rds-cluster-2"),
},
{
DBClusterArn: aws.String("dummy-arn-3"),
DBClusterArn: aws.String("dummy-rds-cluster-arn-3"),
DBClusterIdentifier: aws.String("rds-cluster-3"),
},
}
s.dummyRDSInstances = []rdsTypes.DBInstance{
{
DBInstanceArn: aws.String("dummy-arn-1"),
DBInstanceArn: aws.String("dummy-rds-instance-arn-1"),
DBInstanceIdentifier: aws.String("rds-instance-1"),
},
{
DBInstanceArn: aws.String("dummy-arn-2"),
DBInstanceArn: aws.String("dummy-rds-instance-arn-2"),
DBInstanceIdentifier: aws.String("rds-instance-2"),
},
{
DBInstanceArn: aws.String("dummy-arn-3"),
DBInstanceArn: aws.String("dummy-rds-instance-arn-3"),
DBInstanceIdentifier: aws.String("rds-instance-3"),
},
}
s.dummyRedshiftClusters = []redshiftTypes.Cluster{
{
ClusterNamespaceArn: aws.String("dummy-arn-1"),
ClusterNamespaceArn: aws.String("dummy-redshift-cluster-arn-1"),
ClusterIdentifier: aws.String("redshift-cluster-1"),
},
{
ClusterNamespaceArn: aws.String("dummy-arn-2"),
ClusterNamespaceArn: aws.String("dummy-redshift-cluster-arn-2"),
ClusterIdentifier: aws.String("redshift-cluster-2"),
},
{
ClusterNamespaceArn: aws.String("dummy-arn-3"),
ClusterNamespaceArn: aws.String("dummy-redshift-cluster-arn-3"),
ClusterIdentifier: aws.String("redshift-cluster-3"),
},
}
s.dummyDynamoDBTableNames = []string{
"dynamodb-table-1",
"dynamodb-table-2",
"dynamodb-table-3",
"dynamodb-table-4",
}
s.dummyDynamoDBTable = map[string]*types.TableDescription{
s.dummyDynamoDBTableNames[0]: {
TableArn: aws.String("dummy-arn-1"),
TableArn: aws.String("dummy-dynamo-table-arn-1"),
TableName: aws.String(s.dummyDynamoDBTableNames[0]),
},
s.dummyDynamoDBTableNames[1]: {
TableArn: aws.String("dummy-arn-2"),
TableArn: aws.String("dummy-dynamo-table-arn-2"),
TableName: aws.String(s.dummyDynamoDBTableNames[1]),
},
s.dummyDynamoDBTableNames[2]: {
TableArn: aws.String("dummy-arn-3"),
TableArn: aws.String("dummy-dynamo-table-arn-3"),
TableName: aws.String(s.dummyDynamoDBTableNames[2]),
},
s.dummyDynamoDBTableNames[3]: {
// We add a duplicate ARN here to test that the scanner will not
// report the same table twice.
TableArn: aws.String("dummy-dynamo-table-arn-3"),
TableName: aws.String(s.dummyDynamoDBTableNames[3]),
},
}
s.dummyDynamoDBTags = []types.Tag{
{
Expand Down Expand Up @@ -175,71 +182,71 @@ func (s *AWSScannerTestSuite) TestScan() {
results, err := awsScanner.Scan(ctx)

expectedResults := &scan.ScanResults{
Repositories: []scan.Repository{
{
Repositories: map[string]scan.Repository{
*s.dummyRDSClusters[0].DBClusterArn: {
Id: *s.dummyRDSClusters[0].DBClusterArn,
Name: *s.dummyRDSClusters[0].DBClusterIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSClusters[0],
},
{
*s.dummyRDSClusters[1].DBClusterArn: {
Id: *s.dummyRDSClusters[1].DBClusterArn,
Name: *s.dummyRDSClusters[1].DBClusterIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSClusters[1],
},
{
*s.dummyRDSClusters[2].DBClusterArn: {
Id: *s.dummyRDSClusters[2].DBClusterArn,
Name: *s.dummyRDSClusters[2].DBClusterIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSClusters[2],
},
{
*s.dummyRDSInstances[0].DBInstanceArn: {
Id: *s.dummyRDSInstances[0].DBInstanceArn,
Name: *s.dummyRDSInstances[0].DBInstanceIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSInstances[0],
},
{
*s.dummyRDSInstances[1].DBInstanceArn: {
Id: *s.dummyRDSInstances[1].DBInstanceArn,
Name: *s.dummyRDSInstances[1].DBInstanceIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSInstances[1],
},
{
*s.dummyRDSInstances[2].DBInstanceArn: {
Id: *s.dummyRDSInstances[2].DBInstanceArn,
Name: *s.dummyRDSInstances[2].DBInstanceIdentifier,
Type: scan.RepoTypeRDS,
Tags: []string{},
Properties: s.dummyRDSInstances[2],
},
{
*s.dummyRedshiftClusters[0].ClusterNamespaceArn: {
Id: *s.dummyRedshiftClusters[0].ClusterNamespaceArn,
Name: *s.dummyRedshiftClusters[0].ClusterIdentifier,
Type: scan.RepoTypeRedshift,
Tags: []string{},
Properties: s.dummyRedshiftClusters[0],
},
{
*s.dummyRedshiftClusters[1].ClusterNamespaceArn: {
Id: *s.dummyRedshiftClusters[1].ClusterNamespaceArn,
Name: *s.dummyRedshiftClusters[1].ClusterIdentifier,
Type: scan.RepoTypeRedshift,
Tags: []string{},
Properties: s.dummyRedshiftClusters[1],
},
{
*s.dummyRedshiftClusters[2].ClusterNamespaceArn: {
Id: *s.dummyRedshiftClusters[2].ClusterNamespaceArn,
Name: *s.dummyRedshiftClusters[2].ClusterIdentifier,
Type: scan.RepoTypeRedshift,
Tags: []string{},
Properties: s.dummyRedshiftClusters[2],
},
{
*s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]].TableArn: {
Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]].TableArn,
Name: s.dummyDynamoDBTableNames[0],
Type: scan.RepoTypeDynamoDB,
Expand All @@ -259,7 +266,7 @@ func (s *AWSScannerTestSuite) TestScan() {
},
Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[0]],
},
{
*s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]].TableArn: {
Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]].TableArn,
Name: s.dummyDynamoDBTableNames[1],
Type: scan.RepoTypeDynamoDB,
Expand All @@ -279,7 +286,7 @@ func (s *AWSScannerTestSuite) TestScan() {
},
Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[1]],
},
{
*s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]].TableArn: {
Id: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]].TableArn,
Name: s.dummyDynamoDBTableNames[2],
Type: scan.RepoTypeDynamoDB,
Expand All @@ -302,12 +309,7 @@ func (s *AWSScannerTestSuite) TestScan() {
},
}

//l := len(expectedResults.Repositories)
require.ElementsMatch(
s.T(),
expectedResults.Repositories,
results.Repositories,
)
require.Equal(s.T(), expectedResults, results)
require.NoError(s.T(), err)
}

Expand Down Expand Up @@ -358,7 +360,7 @@ func (s *AWSScannerTestSuite) TestScan_WithErrors() {
results, err := awsScanner.Scan(ctx)

expectedResults := &scan.ScanResults{
Repositories: []scan.Repository{},
Repositories: nil,
}

require.ElementsMatch(
Expand Down
5 changes: 3 additions & 2 deletions scan/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ type Repository struct {
}

// ScanResults represents the results of a repository scan, including all the
// data repositories that were scanned.
// data repositories that were scanned. The map key is the repository ID and the
// value is the repository itself.
type ScanResults struct {
Repositories []Repository
Repositories map[string]Repository
}

// ScanError is an error type that represents a collection of errors that
Expand Down
Loading