diff --git a/aws/client.go b/aws/client.go index 81fadea..2594ee8 100644 --- a/aws/client.go +++ b/aws/client.go @@ -4,6 +4,8 @@ import ( "context" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/docdb" + docdbTypes "github.com/aws/aws-sdk-go-v2/service/docdb/types" "github.com/aws/aws-sdk-go-v2/service/dynamodb" ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/aws/aws-sdk-go-v2/service/rds" @@ -51,11 +53,26 @@ type dynamoDBClient interface { ) (*dynamodb.ListTagsOfResourceOutput, error) } +type documentDBClient interface { + DescribeDBClusters( + ctx context.Context, + params *docdb.DescribeDBClustersInput, + optFns ...func(*docdb.Options), + ) (*docdb.DescribeDBClustersOutput, error) + + ListTagsForResource( + ctx context.Context, + params *docdb.ListTagsForResourceInput, + optFns ...func(*docdb.Options), + ) (*docdb.ListTagsForResourceOutput, error) +} + type awsClient struct { config aws.Config rds rdsClient redshift redshiftClient dynamodb dynamoDBClient + docdb documentDBClient } type awsClientConstructor func(awsConfig aws.Config) *awsClient @@ -66,6 +83,7 @@ func newAWSClient(awsConfig aws.Config) *awsClient { rds: rds.NewFromConfig(awsConfig), redshift: redshift.NewFromConfig(awsConfig), dynamodb: dynamodb.NewFromConfig(awsConfig), + docdb: docdb.NewFromConfig(awsConfig), } } @@ -229,3 +247,83 @@ func (c *awsClient) getDynamoDBTables( } return tables, nil } + +type docdbCluster struct { + cluster docdbTypes.DBCluster + tags []string +} + +func (c *awsClient) getDocumentDBClusters( + ctx context.Context, +) ([]docdbCluster, error) { + // First we need to fetch all clusters. These have a bunch of information, but + // not all that we need. + clusters := []docdbTypes.DBCluster{} + var marker *string // Used for pagination + for { + output, err := c.docdb.DescribeDBClusters( + ctx, + &docdb.DescribeDBClustersInput{ + Filters: []docdbTypes.Filter{ + { + Name: aws.String("engine"), + Values: []string{"docdb"}, + }, + }, + Marker: marker, + }, + ) + if err != nil { + return nil, err + } + + clusters = append(clusters, output.DBClusters...) + + if output.Marker == nil { + break + } else { + marker = output.Marker + } + } + + // OK, we now have all the clusters. We can iterate through them, fetching + // all their tags + + // Map from cluster ARN to all the cluster and instance tags + tags := make(map[string][]string, len(clusters)) + for i := range clusters { + clusterARN := clusters[i].DBClusterArn + output, err := c.docdb.ListTagsForResource( + ctx, + &docdb.ListTagsForResourceInput{ + ResourceName: clusters[i].DBClusterArn, + }, + ) + if err != nil { + return nil, err + } + + formattedTags := make([]string, len(output.TagList)) + for i, tag := range output.TagList { + formattedTags[i] = formatTag(tag.Key, tag.Value) + } + + tags[*clusterARN] = formattedTags + } + + // Phew, that was a lot of work, but we have all that we wanted: + // All clusters in the variable + // A map from cluster ARN to tags, in the variable + ret := make([]docdbCluster, len(tags)) + for i := range clusters { + clusterARN := clusters[i].DBClusterArn + clusterTags := tags[*clusterARN] + + ret[i] = docdbCluster{ + cluster: clusters[i], + tags: clusterTags, + } + } + + return ret, nil +} diff --git a/aws/repository.go b/aws/repository.go index 77eac94..ce00090 100644 --- a/aws/repository.go +++ b/aws/repository.go @@ -89,3 +89,16 @@ func formatTag(key, value *string) string { aws.ToString(value), ) } + +func newRepositoryFromDocumentDBCluster( + cluster docdbCluster, +) scan.Repository { + return scan.Repository{ + Id: *cluster.cluster.DBClusterArn, + Name: *cluster.cluster.DBClusterIdentifier, + Type: scan.RepoTypeDocumentDB, + CreatedAt: *cluster.cluster.ClusterCreateTime, + Tags: cluster.tags, + Properties: cluster.cluster, + } +} diff --git a/aws/scan.go b/aws/scan.go index 64fcb83..494cf83 100644 --- a/aws/scan.go +++ b/aws/scan.go @@ -120,3 +120,29 @@ func scanDynamoDBRepositories( scanErrors: scanErrors, } } + +func scanDocumentDBRepositories( + ctx context.Context, + awsClient *awsClient, +) scanResponse { + repos := []scan.Repository{} + var scanErrors []error + + clusters, err := awsClient.getDocumentDBClusters(ctx) + if err != nil { + scanErrors = append( + scanErrors, + fmt.Errorf("error scanning DocumentDB clusters: %w", err), + ) + } + for _, cluster := range clusters { + repos = append( + repos, + newRepositoryFromDocumentDBCluster(cluster), + ) + } + return scanResponse{ + repositories: repos, + scanErrors: scanErrors, + } +} diff --git a/aws/scanner.go b/aws/scanner.go index 1a334f8..775b1ae 100644 --- a/aws/scanner.go +++ b/aws/scanner.go @@ -127,6 +127,7 @@ func scanRegion( scanRDSInstanceRepositories, scanRedshiftRepositories, scanDynamoDBRepositories, + scanDocumentDBRepositories, } responseChan := make(chan scanResponse) diff --git a/aws/scanner_integration_test.go b/aws/scanner_integration_test.go index 4a8e6d7..3c5cecb 100644 --- a/aws/scanner_integration_test.go +++ b/aws/scanner_integration_test.go @@ -40,5 +40,5 @@ func (s *AWSScannerIntegrationTestSuite) TestScan() { results, scanErrors := s.scanner.Scan(ctx) fmt.Printf("Num. Repositories: %v\n", len(results.Repositories)) fmt.Printf("Repositories: %v\n", results.Repositories) - fmt.Printf("Scan Erros: %v\n", scanErrors) + fmt.Printf("Scan Errors: %v\n", scanErrors) } diff --git a/aws/scanner_test.go b/aws/scanner_test.go index dd78a05..2047f83 100644 --- a/aws/scanner_test.go +++ b/aws/scanner_test.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" + docdbTypes "github.com/aws/aws-sdk-go-v2/service/docdb/types" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types" redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types" @@ -18,12 +20,18 @@ import ( type AWSScannerTestSuite struct { suite.Suite - dummyRDSClusters []rdsTypes.DBCluster - dummyRDSInstances []rdsTypes.DBInstance - dummyRedshiftClusters []redshiftTypes.Cluster + + dummyRDSClusters []rdsTypes.DBCluster + dummyRDSInstances []rdsTypes.DBInstance + + dummyRedshiftClusters []redshiftTypes.Cluster + dummyDynamoDBTableNames []string dummyDynamoDBTable map[string]*types.TableDescription dummyDynamoDBTags []types.Tag + + dummyDocumentDBClusters []docdbTypes.DBCluster + dummyDocumentDBTags []docdbTypes.Tag } func (s *AWSScannerTestSuite) SetupSuite() { @@ -102,6 +110,30 @@ func (s *AWSScannerTestSuite) SetupSuite() { Value: aws.String("value3"), }, } + + s.dummyDocumentDBClusters = []docdbTypes.DBCluster{ + { + DBClusterArn: aws.String("documentdb-arn-1"), + DBClusterIdentifier: aws.String("documentdb-cluster-1"), + ClusterCreateTime: &time.Time{}, + }, + { + DBClusterArn: aws.String("documentdb-arn-2"), + DBClusterIdentifier: aws.String("documentdb-cluster-2"), + ClusterCreateTime: &time.Time{}, + }, + { + DBClusterArn: aws.String("documentdb-arn-3"), + DBClusterIdentifier: aws.String("documentdb-cluster-3"), + ClusterCreateTime: &time.Time{}, + }, + } + s.dummyDocumentDBTags = []docdbTypes.Tag{ + { + Key: aws.String("docdbTag1"), + Value: aws.String("docdbValue1"), + }, + } } func TestAWSScanner(t *testing.T) { @@ -136,6 +168,10 @@ func (s *AWSScannerTestSuite) TestScan() { Table: s.dummyDynamoDBTable, Tags: s.dummyDynamoDBTags, }, + docdb: &mock.MockDocumentDBClient{ + Clusters: s.dummyDocumentDBClusters, + Tags: s.dummyDocumentDBTags, + }, } }, } @@ -267,9 +303,49 @@ func (s *AWSScannerTestSuite) TestScan() { }, Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]], }, + { + Id: *s.dummyDocumentDBClusters[0].DBClusterArn, + Name: *s.dummyDocumentDBClusters[0].DBClusterIdentifier, + Type: scan.RepoTypeDocumentDB, + CreatedAt: *s.dummyDocumentDBClusters[0].ClusterCreateTime, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value, + ), + }, + Properties: s.dummyDocumentDBClusters[0], + }, + { + Id: *s.dummyDocumentDBClusters[1].DBClusterArn, + Name: *s.dummyDocumentDBClusters[1].DBClusterIdentifier, + Type: scan.RepoTypeDocumentDB, + CreatedAt: *s.dummyDocumentDBClusters[1].ClusterCreateTime, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value, + ), + }, + Properties: s.dummyDocumentDBClusters[1], + }, + { + Id: *s.dummyDocumentDBClusters[2].DBClusterArn, + Name: *s.dummyDocumentDBClusters[2].DBClusterIdentifier, + Type: scan.RepoTypeDocumentDB, + CreatedAt: *s.dummyDocumentDBClusters[2].ClusterCreateTime, + Tags: []string{ + fmt.Sprintf( + "%s:%s", + *s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value, + ), + }, + Properties: s.dummyDocumentDBClusters[2], + }, }, } + //l := len(expectedResults.Repositories) require.ElementsMatch( s.T(), expectedResults.Repositories, @@ -311,6 +387,13 @@ func (s *AWSScannerTestSuite) TestScan_WithErrors() { "ListTables": dummyError, }, }, + docdb: &mock.MockDocumentDBClient{ + Errors: map[string]error{ + "DescribeDBClusters": dummyError, + "DescribeDBInstances": dummyError, + "ListTagsForResource": dummyError, + }, + }, } }, } diff --git a/go.mod b/go.mod index cc43451..979e902 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/config v1.26.6 github.com/aws/aws-sdk-go-v2/credentials v1.16.16 + github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 github.com/aws/aws-sdk-go-v2/service/rds v1.69.0 github.com/aws/aws-sdk-go-v2/service/redshift v1.40.0 diff --git a/go.sum b/go.sum index 315f9f0..fabe0db 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2m github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0 h1:urCBXhGZGBLuisF1f5zY49mWwOE1YDafCHiKtPT3YsY= +github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0/go.mod h1:ZA5arLPeTO0tAyLBfUhnP03ekhjGHtsUtxXVKujwkGI= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 h1:plNo3WtooT2fYnhdyuzzsIJ4QWzcF5AT9oFbnrYC5Dw= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= diff --git a/scan/scanner.go b/scan/scanner.go index 4ae3858..3ba51ee 100644 --- a/scan/scanner.go +++ b/scan/scanner.go @@ -20,9 +20,10 @@ type RepoType string const ( // Repo types - RepoTypeRDS RepoType = "REPO_TYPE_RDS" - RepoTypeRedshift RepoType = "REPO_TYPE_REDSHIFT" - RepoTypeDynamoDB RepoType = "REPO_TYPE_DYNAMODB" + RepoTypeRDS RepoType = "REPO_TYPE_RDS" + RepoTypeRedshift RepoType = "REPO_TYPE_REDSHIFT" + RepoTypeDynamoDB RepoType = "REPO_TYPE_DYNAMODB" + RepoTypeDocumentDB RepoType = "REPO_TYPE_DOCUMENTDB" ) // Repository represents a scanned data repository. diff --git a/testutil/mock/aws.go b/testutil/mock/aws.go index d840a79..6ab79e8 100644 --- a/testutil/mock/aws.go +++ b/testutil/mock/aws.go @@ -4,6 +4,8 @@ import ( "context" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/docdb" + docdbTypes "github.com/aws/aws-sdk-go-v2/service/docdb/types" "github.com/aws/aws-sdk-go-v2/service/dynamodb" dynamodbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/aws/aws-sdk-go-v2/service/rds" @@ -66,6 +68,53 @@ func (m *MockRDSClient) DescribeDBInstances( }, nil } +type MockDocumentDBClient struct { + Clusters []docdbTypes.DBCluster + Tags []docdbTypes.Tag + Errors map[string]error +} + +func (m *MockDocumentDBClient) DescribeDBClusters( + ctx context.Context, + params *docdb.DescribeDBClustersInput, + optFns ...func(*docdb.Options), +) (*docdb.DescribeDBClustersOutput, error) { + + if m.Errors["DescribeDBClusters"] != nil { + return nil, m.Errors["DescribeDBClusters"] + } + + if params.Marker == nil { + return &docdb.DescribeDBClustersOutput{ + DBClusters: []docdbTypes.DBCluster{ + m.Clusters[0], + m.Clusters[1], + }, + Marker: aws.String("2"), + }, nil + } + + return &docdb.DescribeDBClustersOutput{ + DBClusters: []docdbTypes.DBCluster{ + m.Clusters[2], + }, + }, nil +} + +func (m *MockDocumentDBClient) ListTagsForResource( + ctx context.Context, + params *docdb.ListTagsForResourceInput, + optFns ...func(*docdb.Options), +) (*docdb.ListTagsForResourceOutput, error) { + if m.Errors["ListTagsForResource"] != nil { + return nil, m.Errors["ListTagsForResource"] + } + + return &docdb.ListTagsForResourceOutput{ + TagList: m.Tags, + }, nil +} + type MockRedshiftClient struct { Clusters []redshiftTypes.Cluster Errors map[string]error