ENG-13477: scanner for DocumentDB (#4)

* ENG-13477: scanner for DocumentDB

Make sure unit and integration tests pass

Use aws.String() for string pointers

Address review comments

* Get rid of DescribeDBInstances for DocDB

* Remove outdated comment

aws/client.go

@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/docdb"
+	docdbTypes "github.com/aws/aws-sdk-go-v2/service/docdb/types"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
 	ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
 	"github.com/aws/aws-sdk-go-v2/service/rds"
@@ -51,11 +53,26 @@ type dynamoDBClient interface {
 	) (*dynamodb.ListTagsOfResourceOutput, error)
 }
 
+type documentDBClient interface {
+	DescribeDBClusters(
+		ctx context.Context,
+		params *docdb.DescribeDBClustersInput,
+		optFns ...func(*docdb.Options),
+	) (*docdb.DescribeDBClustersOutput, error)
+
+	ListTagsForResource(
+		ctx context.Context,
+		params *docdb.ListTagsForResourceInput,
+		optFns ...func(*docdb.Options),
+	) (*docdb.ListTagsForResourceOutput, error)
+}
+
 type awsClient struct {
 	config   aws.Config
 	rds      rdsClient
 	redshift redshiftClient
 	dynamodb dynamoDBClient
+	docdb    documentDBClient
 }
 
 type awsClientConstructor func(awsConfig aws.Config) *awsClient
@@ -66,6 +83,7 @@ func newAWSClient(awsConfig aws.Config) *awsClient {
 		rds:      rds.NewFromConfig(awsConfig),
 		redshift: redshift.NewFromConfig(awsConfig),
 		dynamodb: dynamodb.NewFromConfig(awsConfig),
+		docdb:    docdb.NewFromConfig(awsConfig),
 	}
 }
 
@@ -229,3 +247,83 @@ func (c *awsClient) getDynamoDBTables(
 	}
 	return tables, nil
 }
+
+type docdbCluster struct {
+	cluster docdbTypes.DBCluster
+	tags    []string
+}
+
+func (c *awsClient) getDocumentDBClusters(
+	ctx context.Context,
+) ([]docdbCluster, error) {
+	// First we need to fetch all clusters. These have a bunch of information, but
+	// not all that we need.
+	clusters := []docdbTypes.DBCluster{}
+	var marker *string // Used for pagination
+	for {
+		output, err := c.docdb.DescribeDBClusters(
+			ctx,
+			&docdb.DescribeDBClustersInput{
+				Filters: []docdbTypes.Filter{
+					{
+						Name:   aws.String("engine"),
+						Values: []string{"docdb"},
+					},
+				},
+				Marker: marker,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		clusters = append(clusters, output.DBClusters...)
+
+		if output.Marker == nil {
+			break
+		} else {
+			marker = output.Marker
+		}
+	}
+
+	// OK, we now have all the clusters. We can iterate through them, fetching
+	// all their tags
+
+	// Map from cluster ARN to all the cluster and instance tags
+	tags := make(map[string][]string, len(clusters))
+	for i := range clusters {
+		clusterARN := clusters[i].DBClusterArn
+		output, err := c.docdb.ListTagsForResource(
+			ctx,
+			&docdb.ListTagsForResourceInput{
+				ResourceName: clusters[i].DBClusterArn,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		formattedTags := make([]string, len(output.TagList))
+		for i, tag := range output.TagList {
+			formattedTags[i] = formatTag(tag.Key, tag.Value)
+		}
+
+		tags[*clusterARN] = formattedTags
+	}
+
+	// Phew, that was a lot of work, but we have all that we wanted:
+	// All clusters in the <clusters> variable
+	// A map from cluster ARN to tags, in the <tags> variable
+	ret := make([]docdbCluster, len(tags))
+	for i := range clusters {
+		clusterARN := clusters[i].DBClusterArn
+		clusterTags := tags[*clusterARN]
+
+		ret[i] = docdbCluster{
+			cluster: clusters[i],
+			tags:    clusterTags,
+		}
+	}
+
+	return ret, nil
+}

aws/repository.go

@@ -89,3 +89,16 @@ func formatTag(key, value *string) string {
 		aws.ToString(value),
 	)
 }
+
+func newRepositoryFromDocumentDBCluster(
+	cluster docdbCluster,
+) scan.Repository {
+	return scan.Repository{
+		Id:         *cluster.cluster.DBClusterArn,
+		Name:       *cluster.cluster.DBClusterIdentifier,
+		Type:       scan.RepoTypeDocumentDB,
+		CreatedAt:  *cluster.cluster.ClusterCreateTime,
+		Tags:       cluster.tags,
+		Properties: cluster.cluster,
+	}
+}

aws/scan.go

@@ -120,3 +120,29 @@ func scanDynamoDBRepositories(
 		scanErrors:   scanErrors,
 	}
 }
+
+func scanDocumentDBRepositories(
+	ctx context.Context,
+	awsClient *awsClient,
+) scanResponse {
+	repos := []scan.Repository{}
+	var scanErrors []error
+
+	clusters, err := awsClient.getDocumentDBClusters(ctx)
+	if err != nil {
+		scanErrors = append(
+			scanErrors,
+			fmt.Errorf("error scanning DocumentDB clusters: %w", err),
+		)
+	}
+	for _, cluster := range clusters {
+		repos = append(
+			repos,
+			newRepositoryFromDocumentDBCluster(cluster),
+		)
+	}
+	return scanResponse{
+		repositories: repos,
+		scanErrors:   scanErrors,
+	}
+}

aws/scanner.go

@@ -127,6 +127,7 @@ func scanRegion(
 		scanRDSInstanceRepositories,
 		scanRedshiftRepositories,
 		scanDynamoDBRepositories,
+		scanDocumentDBRepositories,
 	}
 
 	responseChan := make(chan scanResponse)

aws/scanner_integration_test.go

@@ -40,5 +40,5 @@ func (s *AWSScannerIntegrationTestSuite) TestScan() {
 	results, scanErrors := s.scanner.Scan(ctx)
 	fmt.Printf("Num. Repositories: %v\n", len(results.Repositories))
 	fmt.Printf("Repositories: %v\n", results.Repositories)
-	fmt.Printf("Scan Erros: %v\n", scanErrors)
+	fmt.Printf("Scan Errors: %v\n", scanErrors)
 }

aws/scanner_test.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	docdbTypes "github.com/aws/aws-sdk-go-v2/service/docdb/types"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
 	rdsTypes "github.com/aws/aws-sdk-go-v2/service/rds/types"
 	redshiftTypes "github.com/aws/aws-sdk-go-v2/service/redshift/types"
@@ -18,12 +20,18 @@ import (
 
 type AWSScannerTestSuite struct {
 	suite.Suite
-	dummyRDSClusters        []rdsTypes.DBCluster
-	dummyRDSInstances       []rdsTypes.DBInstance
-	dummyRedshiftClusters   []redshiftTypes.Cluster
+
+	dummyRDSClusters  []rdsTypes.DBCluster
+	dummyRDSInstances []rdsTypes.DBInstance
+
+	dummyRedshiftClusters []redshiftTypes.Cluster
+
 	dummyDynamoDBTableNames []string
 	dummyDynamoDBTable      map[string]*types.TableDescription
 	dummyDynamoDBTags       []types.Tag
+
+	dummyDocumentDBClusters []docdbTypes.DBCluster
+	dummyDocumentDBTags     []docdbTypes.Tag
 }
 
 func (s *AWSScannerTestSuite) SetupSuite() {
@@ -102,6 +110,30 @@ func (s *AWSScannerTestSuite) SetupSuite() {
 			Value: aws.String("value3"),
 		},
 	}
+
+	s.dummyDocumentDBClusters = []docdbTypes.DBCluster{
+		{
+			DBClusterArn:        aws.String("documentdb-arn-1"),
+			DBClusterIdentifier: aws.String("documentdb-cluster-1"),
+			ClusterCreateTime:   &time.Time{},
+		},
+		{
+			DBClusterArn:        aws.String("documentdb-arn-2"),
+			DBClusterIdentifier: aws.String("documentdb-cluster-2"),
+			ClusterCreateTime:   &time.Time{},
+		},
+		{
+			DBClusterArn:        aws.String("documentdb-arn-3"),
+			DBClusterIdentifier: aws.String("documentdb-cluster-3"),
+			ClusterCreateTime:   &time.Time{},
+		},
+	}
+	s.dummyDocumentDBTags = []docdbTypes.Tag{
+		{
+			Key:   aws.String("docdbTag1"),
+			Value: aws.String("docdbValue1"),
+		},
+	}
 }
 
 func TestAWSScanner(t *testing.T) {
@@ -136,6 +168,10 @@ func (s *AWSScannerTestSuite) TestScan() {
 					Table:      s.dummyDynamoDBTable,
 					Tags:       s.dummyDynamoDBTags,
 				},
+				docdb: &mock.MockDocumentDBClient{
+					Clusters: s.dummyDocumentDBClusters,
+					Tags:     s.dummyDocumentDBTags,
+				},
 			}
 		},
 	}
@@ -267,9 +303,49 @@ func (s *AWSScannerTestSuite) TestScan() {
 				},
 				Properties: *s.dummyDynamoDBTable[s.dummyDynamoDBTableNames[2]],
 			},
+			{
+				Id:        *s.dummyDocumentDBClusters[0].DBClusterArn,
+				Name:      *s.dummyDocumentDBClusters[0].DBClusterIdentifier,
+				Type:      scan.RepoTypeDocumentDB,
+				CreatedAt: *s.dummyDocumentDBClusters[0].ClusterCreateTime,
+				Tags: []string{
+					fmt.Sprintf(
+						"%s:%s",
+						*s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value,
+					),
+				},
+				Properties: s.dummyDocumentDBClusters[0],
+			},
+			{
+				Id:        *s.dummyDocumentDBClusters[1].DBClusterArn,
+				Name:      *s.dummyDocumentDBClusters[1].DBClusterIdentifier,
+				Type:      scan.RepoTypeDocumentDB,
+				CreatedAt: *s.dummyDocumentDBClusters[1].ClusterCreateTime,
+				Tags: []string{
+					fmt.Sprintf(
+						"%s:%s",
+						*s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value,
+					),
+				},
+				Properties: s.dummyDocumentDBClusters[1],
+			},
+			{
+				Id:        *s.dummyDocumentDBClusters[2].DBClusterArn,
+				Name:      *s.dummyDocumentDBClusters[2].DBClusterIdentifier,
+				Type:      scan.RepoTypeDocumentDB,
+				CreatedAt: *s.dummyDocumentDBClusters[2].ClusterCreateTime,
+				Tags: []string{
+					fmt.Sprintf(
+						"%s:%s",
+						*s.dummyDocumentDBTags[0].Key, *s.dummyDocumentDBTags[0].Value,
+					),
+				},
+				Properties: s.dummyDocumentDBClusters[2],
+			},
 		},
 	}
 
+	//l := len(expectedResults.Repositories)
 	require.ElementsMatch(
 		s.T(),
 		expectedResults.Repositories,
@@ -311,6 +387,13 @@ func (s *AWSScannerTestSuite) TestScan_WithErrors() {
 						"ListTables": dummyError,
 					},
 				},
+				docdb: &mock.MockDocumentDBClient{
+					Errors: map[string]error{
+						"DescribeDBClusters":  dummyError,
+						"DescribeDBInstances": dummyError,
+						"ListTagsForResource": dummyError,
+					},
+				},
 			}
 		},
 	}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.24.1
 	github.com/aws/aws-sdk-go-v2/config v1.26.6
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
+	github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1
 	github.com/aws/aws-sdk-go-v2/service/rds v1.69.0
 	github.com/aws/aws-sdk-go-v2/service/redshift v1.40.0

go.sum

@@ -12,6 +12,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2m
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0 h1:urCBXhGZGBLuisF1f5zY49mWwOE1YDafCHiKtPT3YsY=
+github.com/aws/aws-sdk-go-v2/service/docdb v1.30.0/go.mod h1:ZA5arLPeTO0tAyLBfUhnP03ekhjGHtsUtxXVKujwkGI=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 h1:plNo3WtooT2fYnhdyuzzsIJ4QWzcF5AT9oFbnrYC5Dw=
 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=

scan/scanner.go

@@ -20,9 +20,10 @@ type RepoType string
 
 const (
 	// Repo types
-	RepoTypeRDS      RepoType = "REPO_TYPE_RDS"
-	RepoTypeRedshift RepoType = "REPO_TYPE_REDSHIFT"
-	RepoTypeDynamoDB RepoType = "REPO_TYPE_DYNAMODB"
+	RepoTypeRDS        RepoType = "REPO_TYPE_RDS"
+	RepoTypeRedshift   RepoType = "REPO_TYPE_REDSHIFT"
+	RepoTypeDynamoDB   RepoType = "REPO_TYPE_DYNAMODB"
+	RepoTypeDocumentDB RepoType = "REPO_TYPE_DOCUMENTDB"
 )
 
 // Repository represents a scanned data repository.