From e1d0af75e14c44e3ab52633b72ee0bb07c5736c1 Mon Sep 17 00:00:00 2001 From: Chris Campo Date: Tue, 2 Apr 2024 11:58:37 -0400 Subject: [PATCH] More refactoring --- aws/scanner.go | 16 +- aws/scanner_test.go | 4 +- classification/classification.go | 40 --- classification/classification_test.go | 220 ---------------- classification/gen.go | 5 - classification/label.go | 2 +- classification/label_classifier_test.go | 2 +- classification/mock_classifier_test.go | 95 ------- classification/publisher.go | 16 ++ classification/publisher/publisher.go | 18 -- classification/{publisher => }/stdout.go | 20 +- cmd/repo_scan.go | 7 +- discovery/{config => }/config.go | 19 +- discovery/{config => }/config_test.go | 2 +- discovery/{sql => }/denodo.go | 22 +- discovery/doc.go | 19 +- discovery/{sql => }/gen.go | 2 +- discovery/{sql => }/generic.go | 14 +- discovery/{sql => }/generic_test.go | 2 +- discovery/{sql => }/metadata.go | 2 +- discovery/{sql => }/metadata_test.go | 2 +- discovery/{sql => }/mock_repository_test.go | 95 +++++-- discovery/{sql => }/mysql.go | 22 +- discovery/{sql => }/mysql_test.go | 2 +- discovery/{sql => }/oracle.go | 24 +- discovery/{sql => }/postgres.go | 30 +-- discovery/{sql => }/postgres_test.go | 2 +- discovery/{sql => }/redshift.go | 20 +- discovery/{sql => }/redshift_test.go | 2 +- discovery/{sql => }/registry.go | 28 +- discovery/registry_test.go | 113 ++++++++ .../{sql => }/sample_all_databases_test.go | 20 +- discovery/{sql => }/sample_repository_test.go | 2 +- discovery/{sql => }/sampling.go | 13 +- discovery/{sql => }/sampling_test.go | 2 +- discovery/scanner.go | 156 ----------- discovery/{sql => }/snowflake.go | 30 +-- discovery/{sql => }/snowflake_test.go | 2 +- discovery/sql/doc.go | 8 - discovery/sql/repository_test.go | 134 ---------- .../{sql/repository.go => sqlrepository.go} | 8 +- discovery/{sql => }/sqlserver.go | 22 +- discovery/{sql => }/sqlserver_test.go | 2 +- go.mod | 58 ++-- go.sum | 158 +++++------ scan/doc.go | 4 + scan/env_scanner.go | 66 +++++ scan/gen.go | 5 + scan/mocks/mock_classifier.go | 97 +++++++ scan/repo_scanner.go | 198 ++++++++++++++ scan/repo_scanner_test.go | 247 ++++++++++++++++++ scan/scanner.go | 66 ----- 52 files changed, 1085 insertions(+), 1080 deletions(-) delete mode 100644 classification/gen.go delete mode 100644 classification/mock_classifier_test.go create mode 100644 classification/publisher.go delete mode 100644 classification/publisher/publisher.go rename classification/{publisher => }/stdout.go (55%) rename discovery/{config => }/config.go (90%) rename discovery/{config => }/config_test.go (99%) rename discovery/{sql => }/denodo.go (82%) rename discovery/{sql => }/gen.go (90%) rename discovery/{sql => }/generic.go (96%) rename discovery/{sql => }/generic_test.go (99%) rename discovery/{sql => }/metadata.go (99%) rename discovery/{sql => }/metadata_test.go (98%) rename discovery/{sql => }/mock_repository_test.go (82%) rename discovery/{sql => }/mysql.go (80%) rename discovery/{sql => }/mysql_test.go (97%) rename discovery/{sql => }/oracle.go (84%) rename discovery/{sql => }/postgres.go (78%) rename discovery/{sql => }/postgres_test.go (97%) rename discovery/{sql => }/redshift.go (81%) rename discovery/{sql => }/redshift_test.go (97%) rename discovery/{sql => }/registry.go (79%) create mode 100644 discovery/registry_test.go rename discovery/{sql => }/sample_all_databases_test.go (93%) rename discovery/{sql => }/sample_repository_test.go (99%) rename discovery/{sql => }/sampling.go (96%) rename discovery/{sql => }/sampling_test.go (98%) delete mode 100644 discovery/scanner.go rename discovery/{sql => }/snowflake.go (77%) rename discovery/{sql => }/snowflake_test.go (97%) delete mode 100644 discovery/sql/doc.go delete mode 100644 discovery/sql/repository_test.go rename discovery/{sql/repository.go => sqlrepository.go} (88%) rename discovery/{sql => }/sqlserver.go (81%) rename discovery/{sql => }/sqlserver_test.go (97%) create mode 100644 scan/doc.go create mode 100644 scan/env_scanner.go create mode 100644 scan/gen.go create mode 100644 scan/mocks/mock_classifier.go create mode 100644 scan/repo_scanner.go create mode 100644 scan/repo_scanner_test.go delete mode 100644 scan/scanner.go diff --git a/aws/scanner.go b/aws/scanner.go index 15c1343..2e9eccf 100644 --- a/aws/scanner.go +++ b/aws/scanner.go @@ -14,7 +14,7 @@ import ( "github.com/cyralinc/dmap/scan" ) -// AWSScanner is an implementation of the Scanner interface for the AWS cloud +// AWSScanner is an implementation of the EnvironmentScanner interface for the AWS cloud // provider. It supports scanning data repositories from multiple AWS regions, // including RDS clusters and instances, Redshift clusters and DynamoDB tables. type AWSScanner struct { @@ -23,8 +23,8 @@ type AWSScanner struct { awsClientConstructor awsClientConstructor } -// AWSScanner implements scan.Scanner -var _ scan.Scanner = (*AWSScanner)(nil) +// AWSScanner implements scan.EnvironmentScanner +var _ scan.EnvironmentScanner = (*AWSScanner)(nil) // NewAWSScanner creates a new instance of AWSScanner based on the ScannerConfig. // If AssumeRoleConfig is specified, the AWSScanner will assume this IAM Role @@ -57,7 +57,7 @@ func NewAWSScanner( // Scan performs a scan across all the AWS regions configured and return a scan // results, containing a list of data repositories that includes: RDS clusters // and instances, Redshift clusters and DynamoDB tables. -func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) { +func (s *AWSScanner) Scan(ctx context.Context) (*scan.EnvironmentScanResults, error) { responseChan := make(chan scanResponse) var wg sync.WaitGroup wg.Add(len(s.scannerConfig.Regions)) @@ -100,18 +100,18 @@ func (s *AWSScanner) Scan(ctx context.Context) (*scan.ScanResults, error) { select { case <-ctx.Done(): scanErrors = append(scanErrors, ctx.Err()) - return &scan.ScanResults{ + return &scan.EnvironmentScanResults{ Repositories: repositories, - }, &scan.ScanError{Errs: scanErrors} + }, &scan.EnvironmentScanError{Errs: scanErrors} case response, ok := <-responseChan: if !ok { // Channel closed, all scans finished. var scanErr error if len(scanErrors) > 0 { - scanErr = &scan.ScanError{Errs: scanErrors} + scanErr = &scan.EnvironmentScanError{Errs: scanErrors} } - return &scan.ScanResults{ + return &scan.EnvironmentScanResults{ Repositories: repositories, }, scanErr diff --git a/aws/scanner_test.go b/aws/scanner_test.go index af77ff0..aa54bc6 100644 --- a/aws/scanner_test.go +++ b/aws/scanner_test.go @@ -181,7 +181,7 @@ func (s *AWSScannerTestSuite) TestScan() { ctx := context.Background() results, err := awsScanner.Scan(ctx) - expectedResults := &scan.ScanResults{ + expectedResults := &scan.EnvironmentScanResults{ Repositories: map[string]scan.Repository{ *s.dummyRDSClusters[0].DBClusterArn: { Id: *s.dummyRDSClusters[0].DBClusterArn, @@ -359,7 +359,7 @@ func (s *AWSScannerTestSuite) TestScan_WithErrors() { ctx := context.Background() results, err := awsScanner.Scan(ctx) - expectedResults := &scan.ScanResults{ + expectedResults := &scan.EnvironmentScanResults{ Repositories: nil, } diff --git a/classification/classification.go b/classification/classification.go index 1e481dd..53d8fe8 100644 --- a/classification/classification.go +++ b/classification/classification.go @@ -8,10 +8,7 @@ package classification import ( "context" - "fmt" "maps" - - "github.com/cyralinc/dmap/discovery/repository" ) // Classifier is an interface that represents a data classifier. A classifier @@ -57,40 +54,3 @@ func (c Result) Merge(other Result) { maps.Copy(c[attr], labelSet) } } - -// ClassifySamples uses the provided classifiers to classify the sample data -// passed via the "samples" parameter. It is mostly a helper function which -// loops through each repository.Sample, retrieves the attribute names and -// values of that sample, passes them to Classifier.Classify, and then -// aggregates the results. Please see the documentation for Classifier and its -// Classify method for more details. The returned slice represents all the -// unique classification results for a given sample set. -func ClassifySamples( - ctx context.Context, - samples []sql.Sample, - classifier Classifier, -) ([]ClassifiedTable, error) { - tables := make([]ClassifiedTable, 0, len(samples)) - for _, sample := range samples { - // Classify each sampled row and combine the results. - result := make(Result) - for _, sampleResult := range sample.Results { - res, err := classifier.Classify(ctx, sampleResult) - if err != nil { - return nil, fmt.Errorf("error classifying sample: %w", err) - } - result.Merge(res) - } - if len(result) > 0 { - table := ClassifiedTable{ - Repo: sample.Metadata.Repo, - Database: sample.Metadata.Database, - Schema: sample.Metadata.Schema, - Table: sample.Metadata.Table, - Classifications: result, - } - tables = append(tables, table) - } - } - return tables, nil -} diff --git a/classification/classification_test.go b/classification/classification_test.go index f709be7..d6dde27 100644 --- a/classification/classification_test.go +++ b/classification/classification_test.go @@ -1,12 +1,9 @@ package classification import ( - "context" "testing" "github.com/stretchr/testify/require" - - "github.com/cyralinc/dmap/discovery/repository" ) func TestMerge_WhenCalledOnNilReceiver_ShouldNotPanic(t *testing.T) { @@ -60,220 +57,3 @@ func TestMerge_WhenCalledWithExistingAttributes_ShouldOverwrite(t *testing.T) { result.Merge(other) require.Equal(t, expected, result) } - -func TestClassifySamples_SingleTable(t *testing.T) { - ctx := context.Background() - meta := sql.SampleMetadata{ - Repo: "repo", - Database: "db", - Schema: "schema", - Table: "table", - } - - sample := sql.Sample{ - Metadata: meta, - Results: []sql.SampleResult{ - { - "age": "52", - "social_sec_num": "512-23-4258", - "credit_card_num": "4111111111111111", - }, - { - "age": "101", - "social_sec_num": "foobarbaz", - "credit_card_num": "4111111111111111", - }, - }, - } - - classifier := NewMockClassifier(t) - // Need to explicitly convert it to a map because Mockery isn't smart enough - // to infer the type. - classifier.EXPECT().Classify(ctx, map[string]any(sample.Results[0])).Return( - Result{ - "age": { - "AGE": {Name: "AGE"}, - }, - "social_sec_num": { - "SSN": {Name: "SSN"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - nil, - ) - classifier.EXPECT().Classify(ctx, map[string]any(sample.Results[1])).Return( - Result{ - "age": { - "AGE": {Name: "AGE"}, - "CVV": {Name: "CVV"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - nil, - ) - - expected := []ClassifiedTable{ - { - Repo: meta.Repo, - Database: meta.Database, - Schema: meta.Schema, - Table: meta.Table, - Classifications: Result{ - "age": { - "AGE": {Name: "AGE"}, - "CVV": {Name: "CVV"}, - }, - "social_sec_num": { - "SSN": {Name: "SSN"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - }, - } - actual, err := ClassifySamples(ctx, []sql.Sample{sample}, classifier) - require.NoError(t, err) - require.Len(t, actual, len(expected)) - for i := range actual { - requireClassifiedTableEqual(t, expected[i], actual[i]) - } -} - -func TestClassifySamples_MultipleTables(t *testing.T) { - ctx := context.Background() - meta1 := sql.SampleMetadata{ - Repo: "repo1", - Database: "db1", - Schema: "schema1", - Table: "table1", - } - meta2 := sql.SampleMetadata{ - Repo: "repo2", - Database: "db2", - Schema: "schema2", - Table: "table2", - } - - samples := []sql.Sample{ - { - Metadata: meta1, - Results: []sql.SampleResult{ - { - "age": "52", - "social_sec_num": "512-23-4258", - "credit_card_num": "4111111111111111", - }, - { - "age": "101", - "social_sec_num": "foobarbaz", - "credit_card_num": "4111111111111111", - }, - }, - }, - { - Metadata: meta2, - Results: []sql.SampleResult{ - { - "fullname": "John Doe", - "dob": "2000-01-01", - "random": "foobarbaz", - }, - }, - }, - } - - classifier := NewMockClassifier(t) - // Need to explicitly convert it to a map because Mockery isn't smart enough - // to infer the type. - classifier.EXPECT().Classify(ctx, map[string]any(samples[0].Results[0])).Return( - Result{ - "age": { - "AGE": {Name: "AGE"}, - }, - "social_sec_num": { - "SSN": {Name: "SSN"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - nil, - ) - classifier.EXPECT().Classify(ctx, map[string]any(samples[0].Results[1])).Return( - Result{ - "age": { - "AGE": {Name: "AGE"}, - "CVV": {Name: "CVV"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - nil, - ) - classifier.EXPECT().Classify(ctx, map[string]any(samples[1].Results[0])).Return( - Result{ - "fullname": { - "FULL_NAME": {Name: "FULL_NAME"}, - }, - "dob": { - "DOB": {Name: "DOB"}, - }, - }, - nil, - ) - - expected := []ClassifiedTable{ - { - Repo: meta1.Repo, - Database: meta1.Database, - Schema: meta1.Schema, - Table: meta1.Table, - Classifications: Result{ - "age": { - "AGE": {Name: "AGE"}, - "CVV": {Name: "CVV"}, - }, - "social_sec_num": { - "SSN": {Name: "SSN"}, - }, - "credit_card_num": { - "CCN": {Name: "CCN"}, - }, - }, - }, - { - Repo: meta2.Repo, - Database: meta2.Database, - Schema: meta2.Schema, - Table: meta2.Table, - Classifications: Result{ - "fullname": { - "FULL_NAME": {Name: "FULL_NAME"}, - }, - "dob": { - "DOB": {Name: "DOB"}, - }, - }, - }, - } - actual, err := ClassifySamples(ctx, samples, classifier) - require.NoError(t, err) - require.Len(t, actual, len(expected)) - for i := range actual { - requireClassifiedTableEqual(t, expected[i], actual[i]) - } -} - -func requireClassifiedTableEqual(t *testing.T, expected, actual ClassifiedTable) { - require.Equal(t, expected.Repo, actual.Repo) - require.Equal(t, expected.Database, actual.Database) - require.Equal(t, expected.Schema, actual.Schema) - require.Equal(t, expected.Table, actual.Table) - requireResultEqual(t, expected.Classifications, actual.Classifications) -} diff --git a/classification/gen.go b/classification/gen.go deleted file mode 100644 index 0d86176..0000000 --- a/classification/gen.go +++ /dev/null @@ -1,5 +0,0 @@ -package classification - -// Mock generation - see https://vektra.github.io/mockery/ - -//go:generate mockery --testonly --inpackage --with-expecter --name=Classifier --filename=mock_classifier_test.go diff --git a/classification/label.go b/classification/label.go index 8d6bd0f..88702e6 100644 --- a/classification/label.go +++ b/classification/label.go @@ -67,7 +67,7 @@ func GetEmbeddedLabels() (LabelSet, error) { return nil, fmt.Errorf("error unmarshalling labels.yaml: %w", err) } for name, lbl := range labels.Labels { - fname := "rego/" + strings.ReplaceAll(strings.ToLower(name), " ", "_") + ".rego" + fname := "labels/" + strings.ReplaceAll(strings.ToLower(name), " ", "_") + ".rego" b, err := regoFs.ReadFile(fname) if err != nil { return nil, fmt.Errorf("error reading rego file %s: %w", fname, err) diff --git a/classification/label_classifier_test.go b/classification/label_classifier_test.go index 698035c..abea713 100644 --- a/classification/label_classifier_test.go +++ b/classification/label_classifier_test.go @@ -145,7 +145,7 @@ func newTestLabelClassifier(t *testing.T, lblNames ...string) *LabelClassifier { } func newTestLabel(t *testing.T, lblName string) Label { - fname := "rego/" + strings.ReplaceAll(strings.ToLower(lblName), " ", "_") + ".rego" + fname := "labels/" + strings.ReplaceAll(strings.ToLower(lblName), " ", "_") + ".rego" fin, err := regoFs.ReadFile(fname) require.NoError(t, err) classifierCode := string(fin) diff --git a/classification/mock_classifier_test.go b/classification/mock_classifier_test.go deleted file mode 100644 index 82ad1a7..0000000 --- a/classification/mock_classifier_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Code generated by mockery v2.42.1. DO NOT EDIT. - -package classification - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" -) - -// MockClassifier is an autogenerated mock type for the Classifier type -type MockClassifier struct { - mock.Mock -} - -type MockClassifier_Expecter struct { - mock *mock.Mock -} - -func (_m *MockClassifier) EXPECT() *MockClassifier_Expecter { - return &MockClassifier_Expecter{mock: &_m.Mock} -} - -// Classify provides a mock function with given fields: ctx, input -func (_m *MockClassifier) Classify(ctx context.Context, input map[string]interface{}) (Result, error) { - ret := _m.Called(ctx, input) - - if len(ret) == 0 { - panic("no return value specified for Classify") - } - - var r0 Result - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) (Result, error)); ok { - return rf(ctx, input) - } - if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) Result); ok { - r0 = rf(ctx, input) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(Result) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, map[string]interface{}) error); ok { - r1 = rf(ctx, input) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MockClassifier_Classify_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Classify' -type MockClassifier_Classify_Call struct { - *mock.Call -} - -// Classify is a helper method to define mock.On call -// - ctx context.Context -// - input map[string]interface{} -func (_e *MockClassifier_Expecter) Classify(ctx interface{}, input interface{}) *MockClassifier_Classify_Call { - return &MockClassifier_Classify_Call{Call: _e.mock.On("Classify", ctx, input)} -} - -func (_c *MockClassifier_Classify_Call) Run(run func(ctx context.Context, input map[string]interface{})) *MockClassifier_Classify_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(map[string]interface{})) - }) - return _c -} - -func (_c *MockClassifier_Classify_Call) Return(_a0 Result, _a1 error) *MockClassifier_Classify_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *MockClassifier_Classify_Call) RunAndReturn(run func(context.Context, map[string]interface{}) (Result, error)) *MockClassifier_Classify_Call { - _c.Call.Return(run) - return _c -} - -// NewMockClassifier creates a new instance of MockClassifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewMockClassifier(t interface { - mock.TestingT - Cleanup(func()) -}) *MockClassifier { - mock := &MockClassifier{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/classification/publisher.go b/classification/publisher.go new file mode 100644 index 0000000..c367428 --- /dev/null +++ b/classification/publisher.go @@ -0,0 +1,16 @@ +package classification + +import ( + "context" +) + +// Publisher publishes classification and discovery results to some destination, +// which is left up to the implementer. +// TODO: add doc about labels -ccampo 2024-04-02 +type Publisher interface { + // TODO: doc -ccampo 2024-04-02 + PublishLabels(ctx context.Context, repoId string, labels []Label) error + // PublishClassifications publishes a slice of ClassifiedTable to some + // destination. Any error(s) during publication should be returned. + PublishClassifications(ctx context.Context, repoId string, results []ClassifiedTable) error +} diff --git a/classification/publisher/publisher.go b/classification/publisher/publisher.go deleted file mode 100644 index 3c3f02f..0000000 --- a/classification/publisher/publisher.go +++ /dev/null @@ -1,18 +0,0 @@ -// Package publisher provides types for publishing data classifications results -// to an external source (Publisher), such as to stdout (StdOutPublisher). -package publisher - -import ( - "context" - - "github.com/cyralinc/dmap/classification" -) - -// Publisher publishes classification and discovery results to some destination, -// which is left up to the implementer. -type Publisher interface { - // PublishClassifications publishes a slice of - // classification.ClassifiedTable to some destination. Any error(s) during - // publication should be returned. - PublishClassifications(ctx context.Context, repoId string, results []classification.ClassifiedTable) error -} diff --git a/classification/publisher/stdout.go b/classification/stdout.go similarity index 55% rename from classification/publisher/stdout.go rename to classification/stdout.go index 3e0fd17..a77865b 100644 --- a/classification/publisher/stdout.go +++ b/classification/stdout.go @@ -1,11 +1,9 @@ -package publisher +package classification import ( "context" "encoding/json" "fmt" - - "github.com/cyralinc/dmap/classification" ) // StdOutPublisher "publishes" classification results to stdout in JSON format. @@ -14,17 +12,21 @@ type StdOutPublisher struct{} // StdOutPublisher implements Publisher var _ Publisher = (*StdOutPublisher)(nil) +// TODO: godoc -ccampo 2024-04-02 func NewStdOutPublisher() *StdOutPublisher { return &StdOutPublisher{} } -func (c *StdOutPublisher) PublishClassifications( - _ context.Context, - _ string, - tables []classification.ClassifiedTable, -) error { +// TODO: godoc -ccampo 2024-04-02 +func (c *StdOutPublisher) PublishLabels(ctx context.Context, repoId string, labels []Label) error { + //TODO implement me + panic("not implemented") +} + +// TODO: godoc -ccampo 2024-04-02 +func (c *StdOutPublisher) PublishClassifications(_ context.Context, _ string, tables []ClassifiedTable) error { results := struct { - Results []classification.ClassifiedTable `json:"results"` + Results []ClassifiedTable `json:"results"` }{ Results: tables, } diff --git a/cmd/repo_scan.go b/cmd/repo_scan.go index a4e97a8..b5830d9 100644 --- a/cmd/repo_scan.go +++ b/cmd/repo_scan.go @@ -4,17 +4,16 @@ import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery" - "github.com/cyralinc/dmap/discovery/config" + "github.com/cyralinc/dmap/scan" ) type RepoScanCmd struct { - config.Config + scan.RepoScannerConfig } func (cmd *RepoScanCmd) Run(_ *Globals) error { ctx := context.Background() - scanner, err := discovery.NewScanner(ctx, &cmd.Config) + scanner, err := scan.NewRepoScanner(ctx, cmd.RepoScannerConfig) if err != nil { return fmt.Errorf("error creating new scanner: %w", err) } diff --git a/discovery/config/config.go b/discovery/config.go similarity index 90% rename from discovery/config/config.go rename to discovery/config.go index d974b2f..925c31b 100644 --- a/discovery/config/config.go +++ b/discovery/config.go @@ -1,4 +1,4 @@ -package config +package discovery import ( "fmt" @@ -11,19 +11,6 @@ import ( const configConnOpts = "connection-string-args" -// Config is the configuration for the application. -type Config struct { - Dmap DmapConfig `embed:""` - Repo RepoConfig `embed:""` -} - -// DmapConfig is the necessary configuration to connect to the Dmap API. -type DmapConfig struct { - ApiBaseUrl string `help:"Base URL of the Dmap API." default:"https://api.dmap.cyral.io"` - ClientID string `help:"API client ID to access the Dmap API."` - ClientSecret string `help:"API client secret to access the Dmap API."` //#nosec G101 -- false positive -} - // RepoConfig is the necessary configuration to connect to a data sql. type RepoConfig struct { Type string `help:"Type of repository to connect to (postgres|mysql|oracle|sqlserver|snowflake|redshift|denodo)." enum:"postgres,mysql,oracle,sqlserver,snowflake,redshift,denodo" required:""` @@ -64,7 +51,7 @@ func (g GlobFlag) Decode(ctx *kong.DecodeContext) error { // BuildConnOptsStr parses the repo config to produce a string in the format // "?option=value&option2=value2". Example: // -// BuildConnOptsStr(config.RepoConfig{ +// BuildConnOptsStr(RepoConfig{ // Advanced: map[string]any{ // "connection-string-args": []any{"sslmode=disable"}, // }, @@ -143,7 +130,7 @@ func FetchAdvancedConfigString( } // mapFromConnOpts builds a map from the list of connection options given in the -// config. Each option has the format 'option=value'. Err only if the config is +// Each option has the format 'option=value'. Err only if the config is // malformed, to inform user. func mapFromConnOpts(cfg RepoConfig) (map[string]string, error) { m := make(map[string]string) diff --git a/discovery/config/config_test.go b/discovery/config_test.go similarity index 99% rename from discovery/config/config_test.go rename to discovery/config_test.go index 63a5007..c54ea99 100644 --- a/discovery/config/config_test.go +++ b/discovery/config_test.go @@ -1,4 +1,4 @@ -package config +package discovery import ( "testing" diff --git a/discovery/sql/denodo.go b/discovery/denodo.go similarity index 82% rename from discovery/sql/denodo.go rename to discovery/denodo.go index 0706d34..2075db5 100644 --- a/discovery/sql/denodo.go +++ b/discovery/denodo.go @@ -1,12 +1,10 @@ -package sql +package discovery import ( "context" "errors" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // Use PostgreSQL driver for Denodo _ "github.com/lib/pq" ) @@ -26,18 +24,18 @@ const ( "CATALOG_VDP_METADATA_VIEWS()" ) -// DenodoRepository is a sql.Repository implementation for Denodo. +// DenodoRepository is a sql.SQLRepository implementation for Denodo. type DenodoRepository struct { - // The majority of the sql.Repository functionality is delegated to + // The majority of the sql.SQLRepository functionality is delegated to // a generic SQL repository instance. genericSqlRepo *GenericRepository } -// DenodoRepository implements sql.Repository -var _ Repository = (*DenodoRepository)(nil) +// DenodoRepository implements sql.SQLRepository +var _ SQLRepository = (*DenodoRepository)(nil) // NewDenodoRepository is the constructor for sql. -func NewDenodoRepository(cfg config.RepoConfig) (*DenodoRepository, error) { +func NewDenodoRepository(cfg RepoConfig) (*DenodoRepository, error) { pgCfg, err := ParsePostgresConfig(cfg) if err != nil { return nil, fmt.Errorf("unable to parse postgres config: %w", err) @@ -76,14 +74,14 @@ func (r *DenodoRepository) ListDatabases(_ context.Context) ([]string, error) { } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *DenodoRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.IntrospectWithQuery(ctx, DenodoIntrospectQuery) } // SampleTable delegates sampling to GenericRepository, using a Denodo-specific -// table sample query. See Repository.SampleTable and +// table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *DenodoRepository) SampleTable( ctx context.Context, @@ -103,13 +101,13 @@ func (r *DenodoRepository) SampleTable( return r.genericSqlRepo.SampleTableWithQuery(ctx, meta, query) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *DenodoRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *DenodoRepository) Close() error { return r.genericSqlRepo.Close() diff --git a/discovery/doc.go b/discovery/doc.go index 03bebfd..999e7e2 100644 --- a/discovery/doc.go +++ b/discovery/doc.go @@ -1,9 +1,16 @@ // Package discovery provides mechanisms to perform database introspection and -// data discovery on various data repositories. It provides a Scanner type that +// data discovery on various data repositories. It provides a RepoScanner type that // can be used to scan a data repository for sensitive data, classify the data, -// and publish the results to external sources. Additionally, the sql subpackage -// provides various SQL repository implementations that can be used to -// introspect and sample SQL-based data repositories. Support for additional -// data repository types, such as NoSQL-based repos, is intended to be added in -// the future. +// and publish the results to external sources. +// +// Additionally, the SQLRepository interface provides an API for performing +// database introspection and data discovery on SQL databases. It encapsulates +// the concept of a Dmap data SQL repository. All out-of-the-box SQLRepository +// implementations are included in their own files named after the repository +// type, e.g. mysql.go, postgres.go, etc. +// +// Registry provides an API for registering and constructing SQLRepository +// implementations within an application. There is a global DefaultRegistry +// which has all-out-of-the-box SQLRepository implementations registered to it +// by default. package discovery diff --git a/discovery/sql/gen.go b/discovery/gen.go similarity index 90% rename from discovery/sql/gen.go rename to discovery/gen.go index fd4cde9..b295816 100644 --- a/discovery/sql/gen.go +++ b/discovery/gen.go @@ -1,4 +1,4 @@ -package sql +package discovery // Mock generation - see https://vektra.github.io/mockery/ diff --git a/discovery/sql/generic.go b/discovery/generic.go similarity index 96% rename from discovery/sql/generic.go rename to discovery/generic.go index 2d22d7f..92db79c 100644 --- a/discovery/sql/generic.go +++ b/discovery/generic.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" @@ -34,13 +34,13 @@ const ( ) // GenericRepository implements generic SQL functionalities that work for a -// subset of ANSI SQL compatible databases. Many Repository implementations may +// subset of ANSI SQL compatible databases. Many SQLRepository implementations may // partially or fully delegate to this implementation. In that respect, it acts // somewhat as a base implementation which can be used by SQL-compatible // repositories. Note that while GenericRepository is an implementation of -// the Repository interface, GenericRepository is meant to be used as a building -// block for other Repository implementations, rather than as a standalone -// implementation. Specifically, the Repository.ListDatabases method is left +// the SQLRepository interface, GenericRepository is meant to be used as a building +// block for other SQLRepository implementations, rather than as a standalone +// implementation. Specifically, the SQLRepository.ListDatabases method is left // un-implemented, since there is no standard way to list databases across // different SQL database platforms. It does however provide the // ListDatabasesWithQuery method, which dependent implementations can use to @@ -54,7 +54,7 @@ type GenericRepository struct { excludePaths []glob.Glob } -var _ Repository = (*GenericRepository)(nil) +var _ SQLRepository = (*GenericRepository)(nil) // NewGenericRepository is a constructor for the GenericRepository type. It // opens a database handle for a given repoType and returns a pointer to a new @@ -177,7 +177,7 @@ func (r *GenericRepository) IntrospectWithQuery( // SampleTable samples the table referenced by the TableMetadata meta parameter // by issuing a standard, ANSI-compatible SELECT query to the database. All // attributes of the table are selected, and are quoted using double quotes. See -// Repository.SampleTable for more details. +// SQLRepository.SampleTable for more details. func (r *GenericRepository) SampleTable( ctx context.Context, meta *TableMetadata, diff --git a/discovery/sql/generic_test.go b/discovery/generic_test.go similarity index 99% rename from discovery/sql/generic_test.go rename to discovery/generic_test.go index a947dee..ac13df5 100644 --- a/discovery/sql/generic_test.go +++ b/discovery/generic_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/metadata.go b/discovery/metadata.go similarity index 99% rename from discovery/sql/metadata.go rename to discovery/metadata.go index dfd3c3c..ca71935 100644 --- a/discovery/sql/metadata.go +++ b/discovery/metadata.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "strings" diff --git a/discovery/sql/metadata_test.go b/discovery/metadata_test.go similarity index 98% rename from discovery/sql/metadata_test.go rename to discovery/metadata_test.go index f26b33c..c787e03 100644 --- a/discovery/sql/metadata_test.go +++ b/discovery/metadata_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "testing" diff --git a/discovery/sql/mock_repository_test.go b/discovery/mock_repository_test.go similarity index 82% rename from discovery/sql/mock_repository_test.go rename to discovery/mock_repository_test.go index a9bdf12..492bcb9 100644 --- a/discovery/sql/mock_repository_test.go +++ b/discovery/mock_repository_test.go @@ -1,6 +1,6 @@ // Code generated by mockery v2.42.1. DO NOT EDIT. -package sql +package discovery import ( context "context" @@ -8,7 +8,7 @@ import ( mock "github.com/stretchr/testify/mock" ) -// MockRepository is an autogenerated mock type for the Repository type +// MockRepository is an autogenerated mock type for the SQLRepository type type MockRepository struct { mock.Mock } @@ -50,9 +50,11 @@ func (_e *MockRepository_Expecter) Close() *MockRepository_Close_Call { } func (_c *MockRepository_Close_Call) Run(run func()) *MockRepository_Close_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) + _c.Call.Run( + func(args mock.Arguments) { + run() + }, + ) return _c } @@ -108,9 +110,11 @@ func (_e *MockRepository_Expecter) Introspect(ctx interface{}) *MockRepository_I } func (_c *MockRepository_Introspect_Call) Run(run func(ctx context.Context)) *MockRepository_Introspect_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) + _c.Call.Run( + func(args mock.Arguments) { + run(args[0].(context.Context)) + }, + ) return _c } @@ -119,7 +123,12 @@ func (_c *MockRepository_Introspect_Call) Return(_a0 *Metadata, _a1 error) *Mock return _c } -func (_c *MockRepository_Introspect_Call) RunAndReturn(run func(context.Context) (*Metadata, error)) *MockRepository_Introspect_Call { +func (_c *MockRepository_Introspect_Call) RunAndReturn( + run func(context.Context) ( + *Metadata, + error, + ), +) *MockRepository_Introspect_Call { _c.Call.Return(run) return _c } @@ -166,9 +175,11 @@ func (_e *MockRepository_Expecter) ListDatabases(ctx interface{}) *MockRepositor } func (_c *MockRepository_ListDatabases_Call) Run(run func(ctx context.Context)) *MockRepository_ListDatabases_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) + _c.Call.Run( + func(args mock.Arguments) { + run(args[0].(context.Context)) + }, + ) return _c } @@ -177,7 +188,12 @@ func (_c *MockRepository_ListDatabases_Call) Return(_a0 []string, _a1 error) *Mo return _c } -func (_c *MockRepository_ListDatabases_Call) RunAndReturn(run func(context.Context) ([]string, error)) *MockRepository_ListDatabases_Call { +func (_c *MockRepository_ListDatabases_Call) RunAndReturn( + run func(context.Context) ( + []string, + error, + ), +) *MockRepository_ListDatabases_Call { _c.Call.Return(run) return _c } @@ -212,9 +228,11 @@ func (_e *MockRepository_Expecter) Ping(ctx interface{}) *MockRepository_Ping_Ca } func (_c *MockRepository_Ping_Call) Run(run func(ctx context.Context)) *MockRepository_Ping_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) + _c.Call.Run( + func(args mock.Arguments) { + run(args[0].(context.Context)) + }, + ) return _c } @@ -229,7 +247,10 @@ func (_c *MockRepository_Ping_Call) RunAndReturn(run func(context.Context) error } // SampleTable provides a mock function with given fields: ctx, meta, params -func (_m *MockRepository) SampleTable(ctx context.Context, meta *TableMetadata, params SampleParameters) (Sample, error) { +func (_m *MockRepository) SampleTable(ctx context.Context, meta *TableMetadata, params SampleParameters) ( + Sample, + error, +) { ret := _m.Called(ctx, meta, params) if len(ret) == 0 { @@ -265,14 +286,26 @@ type MockRepository_SampleTable_Call struct { // - ctx context.Context // - meta *TableMetadata // - params SampleParameters -func (_e *MockRepository_Expecter) SampleTable(ctx interface{}, meta interface{}, params interface{}) *MockRepository_SampleTable_Call { +func (_e *MockRepository_Expecter) SampleTable( + ctx interface{}, + meta interface{}, + params interface{}, +) *MockRepository_SampleTable_Call { return &MockRepository_SampleTable_Call{Call: _e.mock.On("SampleTable", ctx, meta, params)} } -func (_c *MockRepository_SampleTable_Call) Run(run func(ctx context.Context, meta *TableMetadata, params SampleParameters)) *MockRepository_SampleTable_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*TableMetadata), args[2].(SampleParameters)) - }) +func (_c *MockRepository_SampleTable_Call) Run( + run func( + ctx context.Context, + meta *TableMetadata, + params SampleParameters, + ), +) *MockRepository_SampleTable_Call { + _c.Call.Run( + func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*TableMetadata), args[2].(SampleParameters)) + }, + ) return _c } @@ -281,17 +314,25 @@ func (_c *MockRepository_SampleTable_Call) Return(_a0 Sample, _a1 error) *MockRe return _c } -func (_c *MockRepository_SampleTable_Call) RunAndReturn(run func(context.Context, *TableMetadata, SampleParameters) (Sample, error)) *MockRepository_SampleTable_Call { +func (_c *MockRepository_SampleTable_Call) RunAndReturn( + run func( + context.Context, + *TableMetadata, + SampleParameters, + ) (Sample, error), +) *MockRepository_SampleTable_Call { _c.Call.Return(run) return _c } // NewMockRepository creates a new instance of Mocksql. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewMockRepository(t interface { - mock.TestingT - Cleanup(func()) -}) *MockRepository { +func NewMockRepository( + t interface { + mock.TestingT + Cleanup(func()) + }, +) *MockRepository { mock := &MockRepository{} mock.Mock.Test(t) diff --git a/discovery/sql/mysql.go b/discovery/mysql.go similarity index 80% rename from discovery/sql/mysql.go rename to discovery/mysql.go index f3e9421..7a7afc8 100644 --- a/discovery/sql/mysql.go +++ b/discovery/mysql.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // MySQL DB driver _ "github.com/go-sql-driver/mysql" ) @@ -24,18 +22,18 @@ WHERE ` ) -// MySqlRepository is a Repository implementation for MySQL databases. +// MySqlRepository is a SQLRepository implementation for MySQL databases. type MySqlRepository struct { - // The majority of the Repository functionality is delegated to + // The majority of the SQLRepository functionality is delegated to // a generic SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// MySqlRepository implements Repository -var _ Repository = (*MySqlRepository)(nil) +// MySqlRepository implements SQLRepository +var _ SQLRepository = (*MySqlRepository)(nil) // NewMySqlRepository creates a new MySQL sql. -func NewMySqlRepository(cfg config.RepoConfig) (*MySqlRepository, error) { +func NewMySqlRepository(cfg RepoConfig) (*MySqlRepository, error) { connStr := fmt.Sprintf( "%s:%s@tcp(%s:%d)/%s", cfg.User, @@ -69,14 +67,14 @@ func (r *MySqlRepository) ListDatabases(ctx context.Context) ([]string, error) { } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *MySqlRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.Introspect(ctx) } // SampleTable delegates sampling to GenericRepository, using a MySQL-specific -// table sample query. See Repository.SampleTable and +// table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *MySqlRepository) SampleTable( ctx context.Context, @@ -91,13 +89,13 @@ func (r *MySqlRepository) SampleTable( return r.genericSqlRepo.SampleTableWithQuery(ctx, meta, query, params.SampleSize, params.Offset) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *MySqlRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *MySqlRepository) Close() error { return r.genericSqlRepo.Close() diff --git a/discovery/sql/mysql_test.go b/discovery/mysql_test.go similarity index 97% rename from discovery/sql/mysql_test.go rename to discovery/mysql_test.go index 1ee978c..9361a5f 100644 --- a/discovery/sql/mysql_test.go +++ b/discovery/mysql_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/oracle.go b/discovery/oracle.go similarity index 84% rename from discovery/sql/oracle.go rename to discovery/oracle.go index 5547ef2..a3dfcb1 100644 --- a/discovery/sql/oracle.go +++ b/discovery/oracle.go @@ -1,12 +1,10 @@ -package sql +package discovery import ( "context" "errors" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // Oracle DB driver _ "github.com/sijms/go-ora/v2" ) @@ -38,18 +36,18 @@ ON configServiceName = "service-name" ) -// OracleRepository is a Repository implementation for Oracle databases. +// OracleRepository is a SQLRepository implementation for Oracle databases. type OracleRepository struct { // The majority of the OracleRepository functionality is delegated to // a generic SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// OracleRepository implements Repository -var _ Repository = (*OracleRepository)(nil) +// OracleRepository implements SQLRepository +var _ SQLRepository = (*OracleRepository)(nil) // NewOracleRepository creates a new Oracle repository. -func NewOracleRepository(cfg config.RepoConfig) (*OracleRepository, error) { +func NewOracleRepository(cfg RepoConfig) (*OracleRepository, error) { oracleCfg, err := ParseOracleConfig(cfg) if err != nil { return nil, fmt.Errorf("unable to parse oracle config: %w", err) @@ -85,14 +83,14 @@ func (r *OracleRepository) ListDatabases(_ context.Context) ([]string, error) { } // Introspect delegates introspection to GenericRepository, using an -// Oracle-specific introspection query. See Repository.Introspect and +// Oracle-specific introspection query. See SQLRepository.Introspect and // GenericRepository.IntrospectWithQuery for more details. func (r *OracleRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.IntrospectWithQuery(ctx, OracleIntrospectQuery) } // SampleTable delegates sampling to GenericRepository, using an Oracle-specific -// table sample query. See Repository.SampleTable and +// table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *OracleRepository) SampleTable( ctx context.Context, @@ -118,7 +116,7 @@ func (r *OracleRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.GetDb().PingContext(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *OracleRepository) Close() error { return r.genericSqlRepo.Close() @@ -131,10 +129,10 @@ type OracleConfig struct { } // ParseOracleConfig parses the Oracle-specific configuration from the -// given config. The Oracle configuration is expected to be in the +// given The Oracle configuration is expected to be in the // config's "advanced config" property. -func ParseOracleConfig(cfg config.RepoConfig) (*OracleConfig, error) { - oracleCfg, err := config.FetchAdvancedConfigString( +func ParseOracleConfig(cfg RepoConfig) (*OracleConfig, error) { + oracleCfg, err := FetchAdvancedConfigString( cfg, RepoTypeOracle, []string{configServiceName}, diff --git a/discovery/sql/postgres.go b/discovery/postgres.go similarity index 78% rename from discovery/sql/postgres.go rename to discovery/postgres.go index 04c5732..df9603a 100644 --- a/discovery/sql/postgres.go +++ b/discovery/postgres.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // Postgresql DB driver _ "github.com/lib/pq" ) @@ -25,18 +23,18 @@ WHERE ` ) -// PostgresRepository is a Repository implementation for Postgres databases. +// PostgresRepository is a SQLRepository implementation for Postgres databases. type PostgresRepository struct { - // The majority of the Repository functionality is delegated to + // The majority of the SQLRepository functionality is delegated to // a generic SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// PostgresRepository implements Repository -var _ Repository = (*PostgresRepository)(nil) +// PostgresRepository implements SQLRepository +var _ SQLRepository = (*PostgresRepository)(nil) // NewPostgresRepository creates a new PostgresRepository. -func NewPostgresRepository(cfg config.RepoConfig) (*PostgresRepository, error) { +func NewPostgresRepository(cfg RepoConfig) (*PostgresRepository, error) { pgCfg, err := ParsePostgresConfig(cfg) if err != nil { return nil, fmt.Errorf("error parsing postgres config: %w", err) @@ -78,14 +76,14 @@ func (r *PostgresRepository) ListDatabases(ctx context.Context) ([]string, error } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *PostgresRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.Introspect(ctx) } // SampleTable delegates sampling to GenericRepository, using a -// Postgres-specific table sample query. See Repository.SampleTable and +// Postgres-specific table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *PostgresRepository) SampleTable( ctx context.Context, @@ -99,13 +97,13 @@ func (r *PostgresRepository) SampleTable( return r.genericSqlRepo.SampleTableWithQuery(ctx, meta, query, params.SampleSize, params.Offset) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *PostgresRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *PostgresRepository) Close() error { return r.genericSqlRepo.Close() @@ -118,10 +116,10 @@ type PostgresConfig struct { } // ParsePostgresConfig parses the Postgres-specific configuration parameters -// from the given config. The Postgres connection options are built from the -// config and stored in the ConnOptsStr field of the returned PostgresConfig. -func ParsePostgresConfig(cfg config.RepoConfig) (*PostgresConfig, error) { - connOptsStr, err := config.BuildConnOptsStr(cfg) +// from the given The Postgres connection options are built from the +// config and stored in the ConnOptsStr field of the returned Postgres +func ParsePostgresConfig(cfg RepoConfig) (*PostgresConfig, error) { + connOptsStr, err := BuildConnOptsStr(cfg) if err != nil { return nil, fmt.Errorf("error building connection options string: %w", err) } diff --git a/discovery/sql/postgres_test.go b/discovery/postgres_test.go similarity index 97% rename from discovery/sql/postgres_test.go rename to discovery/postgres_test.go index b586a63..1794a17 100644 --- a/discovery/sql/postgres_test.go +++ b/discovery/postgres_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/redshift.go b/discovery/redshift.go similarity index 81% rename from discovery/sql/redshift.go rename to discovery/redshift.go index 11dc5a2..57027d8 100644 --- a/discovery/sql/redshift.go +++ b/discovery/redshift.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // Use PostgreSQL DB driver for Redshift _ "github.com/lib/pq" ) @@ -14,18 +12,18 @@ const ( RepoTypeRedshift = "redshift" ) -// RedshiftRepository is a Repository implementation for Redshift databases. +// RedshiftRepository is a SQLRepository implementation for Redshift databases. type RedshiftRepository struct { // The majority of the RedshiftRepository functionality is delegated to // a generic SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// RedshiftRepository implements Repository -var _ Repository = (*RedshiftRepository)(nil) +// RedshiftRepository implements SQLRepository +var _ SQLRepository = (*RedshiftRepository)(nil) // NewRedshiftRepository creates a new RedshiftRepository. -func NewRedshiftRepository(cfg config.RepoConfig) (*RedshiftRepository, error) { +func NewRedshiftRepository(cfg RepoConfig) (*RedshiftRepository, error) { pgCfg, err := ParsePostgresConfig(cfg) if err != nil { return nil, fmt.Errorf("unable to parse postgres config: %w", err) @@ -68,14 +66,14 @@ func (r *RedshiftRepository) ListDatabases(ctx context.Context) ([]string, error } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *RedshiftRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.Introspect(ctx) } // SampleTable delegates sampling to GenericRepository, using a -// Redshift-specific table sample query. See Repository.SampleTable and +// Redshift-specific table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *RedshiftRepository) SampleTable( ctx context.Context, @@ -89,13 +87,13 @@ func (r *RedshiftRepository) SampleTable( return r.genericSqlRepo.SampleTableWithQuery(ctx, meta, query, params.SampleSize, params.Offset) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *RedshiftRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *RedshiftRepository) Close() error { return r.genericSqlRepo.Close() diff --git a/discovery/sql/redshift_test.go b/discovery/redshift_test.go similarity index 97% rename from discovery/sql/redshift_test.go rename to discovery/redshift_test.go index 5b28bfd..07df14c 100644 --- a/discovery/sql/redshift_test.go +++ b/discovery/redshift_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/registry.go b/discovery/registry.go similarity index 79% rename from discovery/sql/registry.go rename to discovery/registry.go index f9c2e4e..da18fc2 100644 --- a/discovery/sql/registry.go +++ b/discovery/registry.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "errors" "fmt" - - "github.com/cyralinc/dmap/discovery/config" ) var ( @@ -13,7 +11,7 @@ var ( // package of which a number of convenience functions in this package act // on. All currently out-of-the-box repository types are registered to this // registry by this package's init function. Users who want to use custom - // Repository implementations, or just avoid global state altogether, should + // SQLRepository implementations, or just avoid global state altogether, should // use their own instance of Registry, instead of using DefaultRegistry and // the corresponding convenience functions. DefaultRegistry = NewRegistry() @@ -28,7 +26,7 @@ type Registry struct { // RepoConstructor represents the function signature that all repository // implementations should use for their constructor functions. -type RepoConstructor func(ctx context.Context, cfg config.RepoConfig) (Repository, error) +type RepoConstructor func(ctx context.Context, cfg RepoConfig) (SQLRepository, error) // NewRegistry creates a new Registry instance. func NewRegistry() *Registry { @@ -59,12 +57,12 @@ func (r *Registry) MustRegister(repoType string, constructor RepoConstructor) { } } -// NewRepository is a factory method to return a concrete Repository +// NewRepository is a factory method to return a concrete SQLRepository // implementation based on the specified type, e.g. MySQL, Postgres, SQL Server, // etc., which must be registered with the registry. If the repository type is // not registered, an error is returned. A new instance of the repository is // returned each time this method is called. -func (r *Registry) NewRepository(ctx context.Context, cfg config.RepoConfig) (Repository, error) { +func (r *Registry) NewRepository(ctx context.Context, cfg RepoConfig) (SQLRepository, error) { constructor, ok := r.constructors[cfg.Type] if !ok { return nil, errors.New("unsupported repo type " + cfg.Type) @@ -90,7 +88,7 @@ func MustRegister(repoType string, constructor RepoConstructor) { // NewRepository is a convenience function that delegates to DefaultRegistry. // See Registry.NewRepository for more details. -func NewRepository(ctx context.Context, cfg config.RepoConfig) (Repository, error) { +func NewRepository(ctx context.Context, cfg RepoConfig) (SQLRepository, error) { return DefaultRegistry.NewRepository(ctx, cfg) } @@ -99,43 +97,43 @@ func NewRepository(ctx context.Context, cfg config.RepoConfig) (Repository, erro func init() { MustRegister( RepoTypeDenodo, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewDenodoRepository(cfg) }, ) MustRegister( RepoTypeMysql, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewMySqlRepository(cfg) }, ) MustRegister( RepoTypeOracle, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewOracleRepository(cfg) }, ) MustRegister( RepoTypePostgres, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewPostgresRepository(cfg) }, ) MustRegister( RepoTypeRedshift, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewRedshiftRepository(cfg) }, ) MustRegister( RepoTypeSnowflake, - func(ctx context.Context, cfg config.RepoConfig) (Repository, error) { + func(ctx context.Context, cfg RepoConfig) (SQLRepository, error) { return NewSnowflakeRepository(cfg) }, ) MustRegister( RepoTypeSqlServer, - func(_ context.Context, cfg config.RepoConfig) (Repository, error) { + func(_ context.Context, cfg RepoConfig) (SQLRepository, error) { return NewSqlServerRepository(cfg) }, ) diff --git a/discovery/registry_test.go b/discovery/registry_test.go new file mode 100644 index 0000000..d459e36 --- /dev/null +++ b/discovery/registry_test.go @@ -0,0 +1,113 @@ +package discovery + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TODO: refactor tests to use registry instance, not default registry -ccampo 2024-04-02 + +func TestRegistry_Register_Successful(t *testing.T) { + repoType := "repoType" + constructor := func(context.Context, RepoConfig) (SQLRepository, error) { + return nil, nil + } + reg := NewRegistry() + err := reg.Register(repoType, constructor) + require.NoError(t, err) + assert.Contains(t, reg.constructors, repoType) +} + +func TestRegistry_MustRegister_NilConstructor(t *testing.T) { + reg := NewRegistry() + assert.Panics(t, func() { reg.MustRegister("repoType", nil) }) +} + +func TestRegistry_MustRegister_TwoCalls_Panics(t *testing.T) { + repoType := "repoType" + constructor := func(context.Context, RepoConfig) (SQLRepository, error) { + return nil, nil + } + reg := NewRegistry() + reg.MustRegister(repoType, constructor) + assert.Contains(t, reg.constructors, repoType) + assert.Panics(t, func() { reg.MustRegister(repoType, constructor) }) +} + +func TestRegistry_NewRepository_IsSuccessful(t *testing.T) { + repoType := "repoType" + called := false + expectedRepo := dummyRepo{} + constructor := func(context.Context, RepoConfig) (SQLRepository, error) { + called = true + return expectedRepo, nil + } + reg := NewRegistry() + err := reg.Register(repoType, constructor) + require.NoError(t, err) + assert.Contains(t, reg.constructors, repoType) + + cfg := RepoConfig{Type: repoType} + repo, err := reg.NewRepository(context.Background(), cfg) + assert.NoError(t, err) + assert.Equal(t, expectedRepo, repo) + assert.True(t, called, "Constructor was not called") +} + +func TestRegistry_NewRepository_ConstructorError(t *testing.T) { + repoType := "repoType" + called := false + expectedErr := errors.New("dummy error") + constructor := func(context.Context, RepoConfig) (SQLRepository, error) { + called = true + return nil, expectedErr + } + reg := NewRegistry() + err := reg.Register(repoType, constructor) + require.NoError(t, err) + assert.Contains(t, reg.constructors, repoType) + + cfg := RepoConfig{Type: repoType} + repo, err := reg.NewRepository(context.Background(), cfg) + assert.ErrorIs(t, err, expectedErr) + assert.Nil(t, repo) + assert.True(t, called, "Constructor was not called") +} + +func TestRegistry_NewRepository_UnsupportedRepoType(t *testing.T) { + repoType := "repoType" + cfg := RepoConfig{Type: repoType} + reg := NewRegistry() + repo, err := reg.NewRepository(context.Background(), cfg) + assert.Error(t, err) + assert.Nil(t, repo) +} + +type dummyRepo struct{} + +func (d dummyRepo) SampleTable(context.Context, *TableMetadata, SampleParameters) ( + Sample, + error, +) { + panic("not implemented") +} + +func (d dummyRepo) ListDatabases(context.Context) ([]string, error) { + panic("not implemented") +} + +func (d dummyRepo) Introspect(context.Context) (*Metadata, error) { + panic("not implemented") +} + +func (d dummyRepo) Ping(context.Context) error { + panic("not implemented") +} + +func (d dummyRepo) Close() error { + panic("not implemented") +} diff --git a/discovery/sql/sample_all_databases_test.go b/discovery/sample_all_databases_test.go similarity index 93% rename from discovery/sql/sample_all_databases_test.go rename to discovery/sample_all_databases_test.go index fad87eb..0539d05 100644 --- a/discovery/sql/sample_all_databases_test.go +++ b/discovery/sample_all_databases_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" @@ -7,17 +7,15 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - - "github.com/cyralinc/dmap/discovery/config" ) const mockRepoType = "mockRepo" func setup(t *testing.T) *MockRepository { repo := NewMockRepository(t) - Register( + MustRegister( mockRepoType, - func(ctx context.Context, cfg config.RepoConfig) (Repository, error) { + func(ctx context.Context, cfg RepoConfig) (SQLRepository, error) { return repo, nil }, ) @@ -25,7 +23,7 @@ func setup(t *testing.T) *MockRepository { } func cleanup() { - delete(registry, mockRepoType) + delete(DefaultRegistry.constructors, mockRepoType) } func TestSampleAllDatabases_Error(t *testing.T) { @@ -35,7 +33,7 @@ func TestSampleAllDatabases_Error(t *testing.T) { ctx := context.Background() listDbErr := errors.New("error listing databases") repo.On("ListDatabases", ctx).Return(nil, listDbErr) - cfg := config.RepoConfig{Type: mockRepoType} + cfg := RepoConfig{Type: mockRepoType} sampleParams := SampleParameters{SampleSize: 5} samples, err := SampleAllDatabases(ctx, repo, cfg, sampleParams) require.Nil(t, samples) @@ -92,7 +90,7 @@ func TestSampleAllDatabases_Successful_TwoDatabases(t *testing.T) { Return(sample, nil) repo.On("Close").Return(nil) - cfg := config.RepoConfig{Type: mockRepoType} + cfg := RepoConfig{Type: mockRepoType} sampleParams := SampleParameters{SampleSize: 5} samples, err := SampleAllDatabases(ctx, repo, cfg, sampleParams) require.NoError(t, err) @@ -113,7 +111,7 @@ func TestSampleAllDatabases_IntrospectError(t *testing.T) { repo.On("Introspect", mock.Anything).Return(nil, introspectErr) repo.On("Close").Return(nil) - cfg := config.RepoConfig{Type: mockRepoType} + cfg := RepoConfig{Type: mockRepoType} sampleParams := SampleParameters{SampleSize: 5} samples, err := SampleAllDatabases(ctx, repo, cfg, sampleParams) require.Empty(t, samples) @@ -158,7 +156,7 @@ func TestSampleAllDatabases_SampleError(t *testing.T) { Return(Sample{}, sampleErr) repo.On("Close").Return(nil) - cfg := config.RepoConfig{Type: mockRepoType} + cfg := RepoConfig{Type: mockRepoType} sampleParams := SampleParameters{SampleSize: 5} samples, err := SampleAllDatabases(ctx, repo, cfg, sampleParams) require.NoError(t, err) @@ -218,7 +216,7 @@ func TestSampleAllDatabases_TwoDatabases_OneSampleError(t *testing.T) { Return(Sample{}, sampleErr).Once() repo.On("Close").Return(nil) - cfg := config.RepoConfig{Type: mockRepoType} + cfg := RepoConfig{Type: mockRepoType} sampleParams := SampleParameters{SampleSize: 5} samples, err := SampleAllDatabases(ctx, repo, cfg, sampleParams) require.NoError(t, err) diff --git a/discovery/sql/sample_repository_test.go b/discovery/sample_repository_test.go similarity index 99% rename from discovery/sql/sample_repository_test.go rename to discovery/sample_repository_test.go index ecc2449..e1323e3 100644 --- a/discovery/sql/sample_repository_test.go +++ b/discovery/sample_repository_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/sampling.go b/discovery/sampling.go similarity index 96% rename from discovery/sql/sampling.go rename to discovery/sampling.go index ec4d5dc..24b3a14 100644 --- a/discovery/sql/sampling.go +++ b/discovery/sampling.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" @@ -8,8 +8,6 @@ import ( "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" - - "github.com/cyralinc/dmap/discovery/config" ) // SampleParameters contains all parameters necessary to sample a table. @@ -84,7 +82,7 @@ func (result SampleResult) GetAttributeNamesAndValues() ([]string, []string) { // metadata, it calls sql.SampleTable in a new goroutine. Once all the // sampling goroutines are finished, their results are collected and returned // as a slice of Sample. -func SampleRepository(ctx context.Context, repo Repository, params SampleParameters) ( +func SampleRepository(ctx context.Context, repo SQLRepository, params SampleParameters) ( []Sample, error, ) { @@ -138,8 +136,8 @@ func SampleRepository(ctx context.Context, repo Repository, params SampleParamet // sampled. func SampleAllDatabases( ctx context.Context, - repo Repository, - repoCfg config.RepoConfig, + repo SQLRepository, + repoCfg RepoConfig, sampleParams SampleParameters, ) ( []Sample, @@ -166,7 +164,7 @@ func SampleAllDatabases( sema = semaphore.NewWeighted(int64(repoCfg.MaxOpenConns)) } for _, db := range dbs { - go func(db string, cfg config.RepoConfig) { + go func(db string, cfg RepoConfig) { defer wg.Done() if sema != nil { _ = sema.Acquire(ctx, 1) @@ -175,6 +173,7 @@ func SampleAllDatabases( cfg.Database = db // Create a repository instance for this database. It will be used // to connect and sample the database. + // TODO: this is ugly - there's gotta be a better way to do this -ccampo 2024-04-02 repo, err := NewRepository(ctx, cfg) if err != nil { log.WithError(err).Errorf("error creating repository instance for database %s", db) diff --git a/discovery/sql/sampling_test.go b/discovery/sampling_test.go similarity index 98% rename from discovery/sql/sampling_test.go rename to discovery/sampling_test.go index 8bdceb7..f964a4f 100644 --- a/discovery/sql/sampling_test.go +++ b/discovery/sampling_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "testing" diff --git a/discovery/scanner.go b/discovery/scanner.go deleted file mode 100644 index f01a776..0000000 --- a/discovery/scanner.go +++ /dev/null @@ -1,156 +0,0 @@ -package discovery - -import ( - "context" - "errors" - "fmt" - - log "github.com/sirupsen/logrus" - - "github.com/cyralinc/dmap/classification" - "github.com/cyralinc/dmap/classification/publisher" - "github.com/cyralinc/dmap/discovery/config" - "github.com/cyralinc/dmap/discovery/sql" -) - -// Scanner is a data discovery scanner that scans a data repository for -// sensitive data. It also classifies the data and publishes the results to -// the configured external sources. It currently only supports SQL-based -// repositories. -type Scanner struct { - config *config.Config - repository sql.Repository - classifier classification.Classifier - publisher publisher.Publisher -} - -// ScannerOption is a functional option type for the Scanner type. -type ScannerOption func(*Scanner) - -// Repository is a functional option that sets the repository for the Scanner. -func Repository(r sql.Repository) ScannerOption { - return func(s *Scanner) { s.repository = r } -} - -// Classifier is a functional option that sets the classifier for the Scanner. -func Classifier(c classification.Classifier) ScannerOption { - return func(s *Scanner) { s.classifier = c } -} - -// Publisher is a functional option that sets the publisher for the Scanner. -func Publisher(p publisher.Publisher) ScannerOption { - return func(s *Scanner) { s.publisher = p } -} - -// NewScanner creates a new Scanner instance with the provided configuration. -func NewScanner( - ctx context.Context, - config *config.Config, - opts ...ScannerOption, -) (*Scanner, error) { - if config == nil { - return nil, errors.New("config can't be nil") - } - s := &Scanner{config: config} - // Apply options. - for _, opt := range opts { - opt(s) - } - if s.publisher == nil { - // Default to stdout publisher. - s.publisher = publisher.NewStdOutPublisher() - } - if s.classifier == nil { - // Create a new label classifier with the embedded labels. - lbls, err := classification.GetEmbeddedLabels() - if err != nil { - return nil, fmt.Errorf("error getting embedded labels: %w", err) - } - c, err := classification.NewLabelClassifier(lbls.ToSlice()...) - if err != nil { - return nil, fmt.Errorf("error creating new label classifier: %w", err) - } - s.classifier = c - } - if s.repository == nil { - // Get a repository instance from the default registry. - repo, err := sql.NewRepository(ctx, s.config.Repo) - if err != nil { - return nil, fmt.Errorf("error connecting to database: %w", err) - } - s.repository = repo - } - return s, nil -} - -// Scan performs the data repository scan. It introspects and samples the -// repository, classifies the sampled data, and publishes the results to the -// configured publisher. -func (s *Scanner) Scan(ctx context.Context) error { - sampleParams := sql.SampleParameters{SampleSize: s.config.Repo.SampleSize} - var samples []sql.Sample - // The name of the database to connect to has been left unspecified by - // the user, so we try to connect and sample all databases instead. Note - // that Oracle doesn't really have the concept of "databases", and thus - // the Scanner always scans the entire database, so only the single - // (default) repository instance is required in that case. - if s.config.Repo.Database == "" && s.config.Repo.Type != sql.RepoTypeOracle { - var err error - samples, err = sql.SampleAllDatabases( - ctx, - s.repository, - s.config.Repo, - sampleParams, - ) - if err != nil { - err = fmt.Errorf("error sampling databases: %w", err) - // If we didn't get any samples, just return the error. - if len(samples) == 0 { - return err - } - // There were error(s) during sampling, but we still got some - // samples. Just warn and continue. - log.WithError(err).Warn("error sampling databases") - } - } else { - // User specified a database (or this is an Oracle DB), therefore - // we already have a repository instance for it. Just use it to - // sample that database only. - var err error - samples, err = sql.SampleRepository(ctx, s.repository, sampleParams) - if err != nil { - err = fmt.Errorf("error gathering repository data samples: %w", err) - // If we didn't get any samples, just return the error. - if len(samples) == 0 { - return err - } - // There were error(s) during sampling, but we still got some - // samples. Just warn and continue. - log.WithError(err).Warn("error gathering repository data samples") - } - } - - // Classify sampled data - classifications, err := classification.ClassifySamples(ctx, samples, s.classifier) - if err != nil { - return fmt.Errorf("error classifying samples: %w", err) - } - - // Publish classifications if necessary - if len(classifications) == 0 { - log.Info("No discovered classifications") - } else if err := s.publisher.PublishClassifications(ctx, s.config.Repo.Host, classifications); err != nil { - return fmt.Errorf("error publishing classifications: %w", err) - } - - // Done! - return nil -} - -// Cleanup performs cleanup operations for the Scanner. -func (s *Scanner) Cleanup() { - // Nil checks are prevent panics if deps are not yet initialized. - if s.repository != nil { - _ = s.repository.Close() - } -} diff --git a/discovery/sql/snowflake.go b/discovery/snowflake.go similarity index 77% rename from discovery/sql/snowflake.go rename to discovery/snowflake.go index 6e97ba8..cf6d3a0 100644 --- a/discovery/sql/snowflake.go +++ b/discovery/snowflake.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // Snowflake DB driver _ "github.com/snowflakedb/gosnowflake" ) @@ -25,18 +23,18 @@ WHERE configWarehouse = "warehouse" ) -// SnowflakeRepository is a Repository implementation for Snowflake databases. +// SnowflakeRepository is a SQLRepository implementation for Snowflake databases. type SnowflakeRepository struct { - // The majority of the Repository functionality is delegated to + // The majority of the SQLRepository functionality is delegated to // a generic SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// SnowflakeRepository implements Repository -var _ Repository = (*SnowflakeRepository)(nil) +// SnowflakeRepository implements SQLRepository +var _ SQLRepository = (*SnowflakeRepository)(nil) // NewSnowflakeRepository creates a new SnowflakeRepository. -func NewSnowflakeRepository(cfg config.RepoConfig) (*SnowflakeRepository, error) { +func NewSnowflakeRepository(cfg RepoConfig) (*SnowflakeRepository, error) { snowflakeCfg, err := ParseSnowflakeConfig(cfg) if err != nil { return nil, fmt.Errorf("error parsing snowflake config: %w", err) @@ -78,14 +76,14 @@ func (r *SnowflakeRepository) ListDatabases(ctx context.Context) ([]string, erro } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *SnowflakeRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.Introspect(ctx) } // SampleTable delegates sampling to GenericRepository. See -// Repository.SampleTable and GenericRepository.SampleTable for more details. +// SQLRepository.SampleTable and GenericRepository.SampleTable for more details. func (r *SnowflakeRepository) SampleTable( ctx context.Context, meta *TableMetadata, @@ -94,13 +92,13 @@ func (r *SnowflakeRepository) SampleTable( return r.genericSqlRepo.SampleTable(ctx, meta, params) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *SnowflakeRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *SnowflakeRepository) Close() error { return r.genericSqlRepo.Close() @@ -117,10 +115,10 @@ type SnowflakeConfig struct { } // ParseSnowflakeConfig produces a config structure with Snowflake-specific -// parameters found in the repo config. The Snowflake account, role, and -// warehouse are required in the advanced config. -func ParseSnowflakeConfig(cfg config.RepoConfig) (*SnowflakeConfig, error) { - snowflakeCfg, err := config.FetchAdvancedConfigString( +// parameters found in the repo The Snowflake account, role, and +// warehouse are required in the advanced +func ParseSnowflakeConfig(cfg RepoConfig) (*SnowflakeConfig, error) { + snowflakeCfg, err := FetchAdvancedConfigString( cfg, RepoTypeSnowflake, []string{configAccount, configRole, configWarehouse}, diff --git a/discovery/sql/snowflake_test.go b/discovery/snowflake_test.go similarity index 97% rename from discovery/sql/snowflake_test.go rename to discovery/snowflake_test.go index ebb7988..6499a11 100644 --- a/discovery/sql/snowflake_test.go +++ b/discovery/snowflake_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/discovery/sql/doc.go b/discovery/sql/doc.go deleted file mode 100644 index 1aeffed..0000000 --- a/discovery/sql/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Package sql provides an API for performing database introspection and data -// discovery on SQL databases. The Repository type encapsulates the concept of a -// Dmap data SQL repository. The package provides a Registry for all supported -// repository implementations and a factory function to create new instances of -// a repository from the registry. All out-of-the-box Repository implementations -// are included in their own files named after the repository type, e.g. -// mysql.go, postgres.go, etc. -package sql diff --git a/discovery/sql/repository_test.go b/discovery/sql/repository_test.go deleted file mode 100644 index 049440f..0000000 --- a/discovery/sql/repository_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package sql - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cyralinc/dmap/discovery/config" -) - -func TestRegister_Successful(t *testing.T) { - require.Empty(t, registry) - - repoType := "repoType" - constructor := func(context.Context, config.RepoConfig) (Repository, error) { - return nil, nil - } - - Register(repoType, constructor) - assert.Contains(t, registry, repoType) - - t.Cleanup(func() { delete(registry, repoType) }) -} - -func TestRegister_NilConstructor(t *testing.T) { - require.Empty(t, registry) - assert.Panics(t, func() { Register("repoType", nil) }) - assert.Empty(t, registry) -} - -func TestRegister_TwoCalls_Panics(t *testing.T) { - require.Empty(t, registry) - - repoType := "repoType" - constructor := func(context.Context, config.RepoConfig) (Repository, error) { - return nil, nil - } - - Register(repoType, constructor) - assert.Contains(t, registry, repoType) - - assert.Panics(t, func() { Register(repoType, constructor) }) - - t.Cleanup(func() { delete(registry, repoType) }) -} - -func TestNewRepository_IsSuccessful(t *testing.T) { - require.Empty(t, registry) - - repoType := "repoType" - - called := false - expectedRepo := dummyRepo{} - constructor := func(context.Context, config.RepoConfig) (Repository, error) { - called = true - return expectedRepo, nil - } - - Register(repoType, constructor) - assert.Contains(t, registry, repoType) - - cfg := config.RepoConfig{Type: repoType} - repo, err := NewRepository(context.Background(), cfg) - assert.NoError(t, err) - assert.Equal(t, expectedRepo, repo) - assert.True(t, called, "Constructor was not called") - - t.Cleanup(func() { delete(registry, repoType) }) -} - -func TestNewRepository_ConstructorError(t *testing.T) { - require.Empty(t, registry) - - repoType := "repoType" - - called := false - expectedErr := errors.New("dummy error") - constructor := func(context.Context, config.RepoConfig) (Repository, error) { - called = true - return nil, expectedErr - } - - Register(repoType, constructor) - assert.Contains(t, registry, repoType) - - cfg := config.RepoConfig{Type: repoType} - repo, err := NewRepository(context.Background(), cfg) - assert.ErrorIs(t, err, expectedErr) - assert.Nil(t, repo) - assert.True(t, called, "Constructor was not called") - - t.Cleanup(func() { delete(registry, repoType) }) -} - -func TestNewRepository_UnsupportedRepoType(t *testing.T) { - require.Empty(t, registry) - - repoType := "repoType" - - cfg := config.RepoConfig{Type: repoType} - repo, err := NewRepository(context.Background(), cfg) - assert.Error(t, err) - assert.Nil(t, repo) - - t.Cleanup(func() { delete(registry, repoType) }) -} - -type dummyRepo struct{} - -func (d dummyRepo) SampleTable(context.Context, *TableMetadata, SampleParameters) ( - Sample, - error, -) { - panic("not implemented") -} - -func (d dummyRepo) ListDatabases(context.Context) ([]string, error) { - panic("not implemented") -} - -func (d dummyRepo) Introspect(context.Context) (*Metadata, error) { - panic("not implemented") -} - -func (d dummyRepo) Ping(context.Context) error { - panic("not implemented") -} - -func (d dummyRepo) Close() error { - panic("not implemented") -} diff --git a/discovery/sql/repository.go b/discovery/sqlrepository.go similarity index 88% rename from discovery/sql/repository.go rename to discovery/sqlrepository.go index 174053c..8f0eeb8 100644 --- a/discovery/sql/repository.go +++ b/discovery/sqlrepository.go @@ -1,12 +1,12 @@ -package sql +package discovery import ( "context" ) -// Repository represents a Dmap data SQL repository, and provides functionality +// SQLRepository represents a Dmap data SQL repository, and provides functionality // to introspect its corresponding schema. -type Repository interface { +type SQLRepository interface { // ListDatabases returns a list of the names of all databases on the server. ListDatabases(ctx context.Context) ([]string, error) // Introspect will read and analyze the basic properties of the repository @@ -27,6 +27,6 @@ type Repository interface { // should be invoked e.g. in the dry-run mode. Ping(ctx context.Context) error // Close is meant to be used as a general purpose cleanup. It should be - // invoked when the Repository is no longer used. + // invoked when the SQLRepository is no longer used. Close() error } diff --git a/discovery/sql/sqlserver.go b/discovery/sqlserver.go similarity index 81% rename from discovery/sql/sqlserver.go rename to discovery/sqlserver.go index 65acebf..15320e7 100644 --- a/discovery/sql/sqlserver.go +++ b/discovery/sqlserver.go @@ -1,11 +1,9 @@ -package sql +package discovery import ( "context" "fmt" - "github.com/cyralinc/dmap/discovery/config" - // SQL Server DB driver _ "github.com/denisenkom/go-mssqldb" ) @@ -23,19 +21,19 @@ const ( SqlServerDatabaseQuery = "SELECT name FROM sys.databases WHERE name != 'model' AND name != 'tempdb'" ) -// SqlServerRepository is a Repository implementation for MS SQL Server +// SqlServerRepository is a SQLRepository implementation for MS SQL Server // databases. type SqlServerRepository struct { - // The majority of the Repository functionality is delegated to a generic + // The majority of the SQLRepository functionality is delegated to a generic // SQL repository instance (genericSqlRepo). genericSqlRepo *GenericRepository } -// SqlServerRepository implements Repository -var _ Repository = (*SqlServerRepository)(nil) +// SqlServerRepository implements SQLRepository +var _ SQLRepository = (*SqlServerRepository)(nil) // NewSqlServerRepository creates a new MS SQL Server sql. -func NewSqlServerRepository(cfg config.RepoConfig) (*SqlServerRepository, error) { +func NewSqlServerRepository(cfg RepoConfig) (*SqlServerRepository, error) { connStr := fmt.Sprintf( "sqlserver://%s:%s@%s:%d", cfg.User, @@ -70,14 +68,14 @@ func (r *SqlServerRepository) ListDatabases(ctx context.Context) ([]string, erro } // Introspect delegates introspection to GenericRepository. See -// Repository.Introspect and GenericRepository.IntrospectWithQuery for more +// SQLRepository.Introspect and GenericRepository.IntrospectWithQuery for more // details. func (r *SqlServerRepository) Introspect(ctx context.Context) (*Metadata, error) { return r.genericSqlRepo.Introspect(ctx) } // SampleTable delegates sampling to GenericRepository, using a -// SQL Server-specific table sample query. See Repository.SampleTable and +// SQL Server-specific table sample query. See SQLRepository.SampleTable and // GenericRepository.SampleTableWithQuery for more details. func (r *SqlServerRepository) SampleTable( ctx context.Context, @@ -90,13 +88,13 @@ func (r *SqlServerRepository) SampleTable( return r.genericSqlRepo.SampleTableWithQuery(ctx, meta, query, params.SampleSize) } -// Ping delegates the ping to GenericRepository. See Repository.Ping and +// Ping delegates the ping to GenericRepository. See SQLRepository.Ping and // GenericRepository.Ping for more details. func (r *SqlServerRepository) Ping(ctx context.Context) error { return r.genericSqlRepo.Ping(ctx) } -// Close delegates the close to GenericRepository. See Repository.Close and +// Close delegates the close to GenericRepository. See SQLRepository.Close and // GenericRepository.Close for more details. func (r *SqlServerRepository) Close() error { return r.genericSqlRepo.Close() diff --git a/discovery/sql/sqlserver_test.go b/discovery/sqlserver_test.go similarity index 97% rename from discovery/sql/sqlserver_test.go rename to discovery/sqlserver_test.go index e45c75a..16f2ae5 100644 --- a/discovery/sql/sqlserver_test.go +++ b/discovery/sqlserver_test.go @@ -1,4 +1,4 @@ -package sql +package discovery import ( "context" diff --git a/go.mod b/go.mod index 518e7ff..82a0a44 100644 --- a/go.mod +++ b/go.mod @@ -18,10 +18,10 @@ require ( github.com/gobwas/glob v0.2.3 github.com/hashicorp/go-multierror v1.1.1 github.com/lib/pq v1.10.9 - github.com/open-policy-agent/opa v0.62.1 + github.com/open-policy-agent/opa v0.63.0 github.com/sijms/go-ora/v2 v2.8.10 github.com/sirupsen/logrus v1.9.3 - github.com/snowflakedb/gosnowflake v1.8.0 + github.com/snowflakedb/gosnowflake v1.9.0 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.6.0 gopkg.in/yaml.v3 v3.0.1 @@ -31,16 +31,16 @@ require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect - github.com/apache/arrow/go/v14 v14.0.2 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect @@ -55,55 +55,55 @@ require ( github.com/aws/smithy-go v1.20.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/danieljoos/wincred v1.1.2 // indirect + github.com/danieljoos/wincred v1.2.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.48.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.51.1 // indirect + github.com/prometheus/procfs v0.13.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.15.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/protobuf v1.32.0 // indirect + golang.org/x/tools v0.19.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 2571f7f..f9b98da 100644 --- a/go.sum +++ b/go.sum @@ -5,18 +5,20 @@ github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.0 h1:U/kwEXj0Y+1REAkV4kV8VO1CsEp8tSaQDG/7qC5XuqQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.0/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= @@ -31,72 +33,54 @@ github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= -github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= +github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= github.com/aws/aws-sdk-go-v2/config v1.27.10 h1:PS+65jThT0T/snC5WjyfHHyUgG+eBoupSDV+f838cro= github.com/aws/aws-sdk-go-v2/config v1.27.10/go.mod h1:BePM7Vo4OBpHreKRUMuDXX+/+JWP38FLkzl5m27/Jjs= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= github.com/aws/aws-sdk-go-v2/credentials v1.17.10 h1:qDZ3EA2lv1KangvQB6y258OssCHD0xvaGiEDkG4X/10= github.com/aws/aws-sdk-go-v2/credentials v1.17.10/go.mod h1:6t3sucOaYDwDssHQa0ojH1RpmVmF5/jArkye1b2FKMI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 h1:E3Y+OfzOK1+rmRo/K2G0ml8Vs+Xqk0kOnf4nS0kUtBc= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59/go.mod h1:1M4PLSBUVfBI0aP+C9XI7SM6kZPCGYyI6izWz0TGprE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.14 h1:Nhcq+ODoD9FRQYI3lATy6iADS5maER3ZXSfE8v3FMh8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.14/go.mod h1:VlBbwTpgCj3rKWMVkEAYiAR3FKs7Mi3jALTMGfbfuns= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23/go.mod h1:uIiFgURZbACBEQJfqTZPb/jxO7R+9LeoHUFudtIdeQI= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1 h1:dZXY07Dm59TxAjJcUfNMJHLDI/gLMxTRZefn2jFAVsw= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1/go.mod h1:lVLqEtX+ezgtfalyJs7Peb0uv9dEpAQP5yuq2O26R44= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26/go.mod h1:2UqAAwMUXKeRkAHIlDJqvMVgOWkUi/AUXPk/YIe+Dg4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 h1:6tayEze2Y+hiL3kdnEUxSPsP+pJsUfwLSFspFl1ru9Q= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6/go.mod h1:qVNb/9IOVsLCZh0x2lnagrBwQ9fxajUpXS7OZfIsKn0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0/go.mod h1:bh2E0CXKZsQN+faiKVqC40vfNMAWheoULBCnEgO9K+8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= github.com/aws/aws-sdk-go-v2/service/rds v1.76.1 h1:6NeDO4UyHun2N5Lmkv2yW1sNblRLRjq3j6Azv7kUyuM= github.com/aws/aws-sdk-go-v2/service/rds v1.76.1/go.mod h1:Rw15qGaGWu3jO0dOz7JyvdOEjgae//YrJxVWLYGynvg= github.com/aws/aws-sdk-go-v2/service/redshift v1.43.5 h1:qiwrbGdDKuD0OD2hxNezyiu17AXJtC3UyLpwx0FA3Ds= github.com/aws/aws-sdk-go-v2/service/redshift v1.43.5/go.mod h1:8ldsMsikORLj0GZUozSvbQdctrumAPYhizmj/3AAATI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0/go.mod h1:ncltU6n4Nof5uJttDtcNQ537uNuwYqsZZQcpkd2/GUQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 h1:WzFol5Cd+yDxPAdnzTA5LmpHYSWinhmSj4rQChV0ee8= github.com/aws/aws-sdk-go-v2/service/sso v1.20.4/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -109,8 +93,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= +github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -137,8 +121,8 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -156,23 +140,23 @@ github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+ github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -184,8 +168,9 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -195,10 +180,10 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -215,35 +200,35 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/open-policy-agent/opa v0.62.1 h1:UcxBQ0fe6NEjkYc775j4PWoUFFhx4f6yXKIKSTAuTVk= -github.com/open-policy-agent/opa v0.62.1/go.mod h1:YqiSIIuvKwyomtnnXkJvy0E3KtVKbavjPJ/hNMuOmeM= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/open-policy-agent/opa v0.63.0 h1:ztNNste1v8kH0/vJMJNquE45lRvqwrM5mY9Ctr9xIXw= +github.com/open-policy-agent/opa v0.63.0/go.mod h1:9VQPqEfoB2N//AToTxzZ1pVTVPUoF2Mhd64szzjWPpU= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.51.1 h1:eIjN50Bwglz6a/c3hAgSMcofL3nD+nFQkV6Dd4DsQCw= +github.com/prometheus/common v0.51.1/go.mod h1:lrWtQx+iDfn2mbH5GUzlH9TSHyfZpHkSiG1W7y3sF2Q= +github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= +github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sijms/go-ora/v2 v2.8.10 h1:Ekhx0I+A9qVBy1eOLa2eIhHWWYwVTa0MM78KS6h+5fg= github.com/sijms/go-ora/v2 v2.8.10/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/snowflakedb/gosnowflake v1.8.0 h1:4bQj8eAYGMkou/nICiIEb9jSbBLDDp5cB6JaKx9WwiA= -github.com/snowflakedb/gosnowflake v1.8.0/go.mod h1:7yyY2MxtDti2eXgtvlZ8QxzCN6KV2B4qb1HuygMI+0U= +github.com/snowflakedb/gosnowflake v1.9.0 h1:s2ZdwFxFfpqwa5CqlhnzRESnLmwU3fED6zyNOJHFBQA= +github.com/snowflakedb/gosnowflake v1.9.0/go.mod h1:4ZgHxVf2OKwecx07WjfyAMr0gn8Qj4yvwAo68Og8wsU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -266,18 +251,18 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -285,10 +270,10 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -301,9 +286,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -315,20 +299,20 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= -golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/scan/doc.go b/scan/doc.go new file mode 100644 index 0000000..f0557aa --- /dev/null +++ b/scan/doc.go @@ -0,0 +1,4 @@ +// Package scan provides a EnvironmentScanner interface which can be used for scanning +// cloud environments and performing data repository discovery. +// TODO: fix doc -ccampo 2024-04-02 +package scan diff --git a/scan/env_scanner.go b/scan/env_scanner.go new file mode 100644 index 0000000..fdb90eb --- /dev/null +++ b/scan/env_scanner.go @@ -0,0 +1,66 @@ +package scan + +import ( + "context" + "errors" + "time" +) + +// EnvironmentScanner provides an API to scan cloud environments. It should be +// implemented for a specific cloud provider (e.g. AWS, GCP, etc.). It defines +// the Scan method responsible for scanning the existing data repositories of +// the corresponding cloud provider environment. +type EnvironmentScanner interface { + Scan(ctx context.Context) (*EnvironmentScanResults, error) +} + +// RepoType defines the AWS data repository types supported (e.g. RDS, Redshift, +// DynamoDB, etc). +type RepoType string + +const ( + RepoTypeRDS RepoType = "TYPE_RDS" + RepoTypeRedshift RepoType = "TYPE_REDSHIFT" + RepoTypeDynamoDB RepoType = "TYPE_DYNAMODB" + RepoTypeS3 RepoType = "TYPE_S3" + RepoTypeDocumentDB RepoType = "TYPE_DOCUMENTDB" +) + +// Repository represents a scanned data repository. +type Repository struct { + Id string + Name string + Type RepoType + CreatedAt time.Time + Tags []string + Properties any +} + +// EnvironmentScanResults represents the results of a repository scan, including +// all the data repositories that were scanned. The map key is the repository ID +// and the value is the repository itself. +type EnvironmentScanResults struct { + Repositories map[string]Repository +} + +// EnvironmentScanError is an error type that represents a collection of errors +// that occurred during the scanning process. +type EnvironmentScanError struct { + Errs []error +} + +// Error returns a string representation of the error. +func (e *EnvironmentScanError) Error() string { + if e == nil { + return "" + } + return errors.Join(e.Errs...).Error() +} + +// Unwrap returns the list of errors that occurred during the scanning process. +func (e *EnvironmentScanError) Unwrap() []error { + if e == nil { + return nil + } + return e.Errs +} diff --git a/scan/gen.go b/scan/gen.go new file mode 100644 index 0000000..a6b6284 --- /dev/null +++ b/scan/gen.go @@ -0,0 +1,5 @@ +package scan + +// Mock generation - see https://vektra.github.io/mockery/ + +//go:generate mockery --with-expecter --srcpkg=github.com/cyralinc/dmap/classification --name=Classifier --filename=mock_classifier.go diff --git a/scan/mocks/mock_classifier.go b/scan/mocks/mock_classifier.go new file mode 100644 index 0000000..7738d64 --- /dev/null +++ b/scan/mocks/mock_classifier.go @@ -0,0 +1,97 @@ +// Code generated by mockery v2.42.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + classification "github.com/cyralinc/dmap/classification" + + mock "github.com/stretchr/testify/mock" +) + +// Classifier is an autogenerated mock type for the Classifier type +type Classifier struct { + mock.Mock +} + +type Classifier_Expecter struct { + mock *mock.Mock +} + +func (_m *Classifier) EXPECT() *Classifier_Expecter { + return &Classifier_Expecter{mock: &_m.Mock} +} + +// Classify provides a mock function with given fields: ctx, input +func (_m *Classifier) Classify(ctx context.Context, input map[string]interface{}) (classification.Result, error) { + ret := _m.Called(ctx, input) + + if len(ret) == 0 { + panic("no return value specified for Classify") + } + + var r0 classification.Result + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) (classification.Result, error)); ok { + return rf(ctx, input) + } + if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) classification.Result); ok { + r0 = rf(ctx, input) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(classification.Result) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, map[string]interface{}) error); ok { + r1 = rf(ctx, input) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Classifier_Classify_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Classify' +type Classifier_Classify_Call struct { + *mock.Call +} + +// Classify is a helper method to define mock.On call +// - ctx context.Context +// - input map[string]interface{} +func (_e *Classifier_Expecter) Classify(ctx interface{}, input interface{}) *Classifier_Classify_Call { + return &Classifier_Classify_Call{Call: _e.mock.On("Classify", ctx, input)} +} + +func (_c *Classifier_Classify_Call) Run(run func(ctx context.Context, input map[string]interface{})) *Classifier_Classify_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(map[string]interface{})) + }) + return _c +} + +func (_c *Classifier_Classify_Call) Return(_a0 classification.Result, _a1 error) *Classifier_Classify_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Classifier_Classify_Call) RunAndReturn(run func(context.Context, map[string]interface{}) (classification.Result, error)) *Classifier_Classify_Call { + _c.Call.Return(run) + return _c +} + +// NewClassifier creates a new instance of Classifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClassifier(t interface { + mock.TestingT + Cleanup(func()) +}) *Classifier { + mock := &Classifier{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/scan/repo_scanner.go b/scan/repo_scanner.go new file mode 100644 index 0000000..87bcf83 --- /dev/null +++ b/scan/repo_scanner.go @@ -0,0 +1,198 @@ +package scan + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "github.com/cyralinc/dmap/classification" + "github.com/cyralinc/dmap/discovery" +) + +// RepoScannerConfig is the configuration for the RepoScanner. +type RepoScannerConfig struct { + Dmap DmapConfig `embed:""` + Repo discovery.RepoConfig `embed:""` +} + +// DmapConfig is the necessary configuration to connect to the Dmap API. +type DmapConfig struct { + ApiBaseUrl string `help:"Base URL of the Dmap API." default:"https://api.dmap.cyral.io"` + ClientID string `help:"API client ID to access the Dmap API."` + ClientSecret string `help:"API client secret to access the Dmap API."` //#nosec G101 -- false positive +} + +// RepoScanner is a data discovery scanner that scans a data repository for +// sensitive data. It also classifies the data and publishes the results to +// the configured external sources. It currently only supports SQL-based +// repositories. +type RepoScanner struct { + config RepoScannerConfig + repository discovery.SQLRepository + classifier classification.Classifier + publisher classification.Publisher +} + +// RepoScannerOption is a functional option type for the RepoScanner type. +type RepoScannerOption func(*RepoScanner) + +// WithSQLRepository is a functional option that sets the SQLRepository for the +// RepoScanner. +func WithSQLRepository(r discovery.SQLRepository) RepoScannerOption { + return func(s *RepoScanner) { s.repository = r } +} + +// WithClassifier is a functional option that sets the classifier for the +// RepoScanner. +func WithClassifier(c classification.Classifier) RepoScannerOption { + return func(s *RepoScanner) { s.classifier = c } +} + +// WithPublisher is a functional option that sets the publisher for the RepoScanner. +func WithPublisher(p classification.Publisher) RepoScannerOption { + return func(s *RepoScanner) { s.publisher = p } +} + +// NewRepoScanner creates a new RepoScanner instance with the provided configuration. +func NewRepoScanner(ctx context.Context, cfg RepoScannerConfig, opts ...RepoScannerOption) (*RepoScanner, error) { + s := &RepoScanner{config: cfg} + // Apply options. + for _, opt := range opts { + opt(s) + } + if s.publisher == nil { + // Default to stdout publisher. + s.publisher = classification.NewStdOutPublisher() + } + if s.classifier == nil { + // Create a new label classifier with the embedded labels. + lbls, err := classification.GetEmbeddedLabels() + if err != nil { + return nil, fmt.Errorf("error getting embedded labels: %w", err) + } + c, err := classification.NewLabelClassifier(lbls.ToSlice()...) + if err != nil { + return nil, fmt.Errorf("error creating new label classifier: %w", err) + } + s.classifier = c + } + if s.repository == nil { + // Get a repository instance from the default registry. + repo, err := discovery.NewRepository(ctx, s.config.Repo) + if err != nil { + return nil, fmt.Errorf("error connecting to database: %w", err) + } + s.repository = repo + } + return s, nil +} + +// Scan performs the data repository scan. It introspects and samples the +// repository, classifies the sampled data, and publishes the results to the +// configured classification publisher. +func (s *RepoScanner) Scan(ctx context.Context) error { + sampleParams := discovery.SampleParameters{SampleSize: s.config.Repo.SampleSize} + var samples []discovery.Sample + // The name of the database to connect to has been left unspecified by + // the user, so we try to connect and sample all databases instead. Note + // that Oracle doesn't really have the concept of "databases", and thus + // the RepoScanner always scans the entire database, so only the single + // (default) repository instance is required in that case. + if s.config.Repo.Database == "" && s.config.Repo.Type != discovery.RepoTypeOracle { + var err error + samples, err = discovery.SampleAllDatabases( + ctx, + s.repository, + s.config.Repo, + sampleParams, + ) + if err != nil { + err = fmt.Errorf("error sampling databases: %w", err) + // If we didn't get any samples, just return the error. + if len(samples) == 0 { + return err + } + // There were error(s) during sampling, but we still got some + // samples. Just warn and continue. + log.WithError(err).Warn("error sampling databases") + } + } else { + // User specified a database (or this is an Oracle DB), therefore + // we already have a repository instance for it. Just use it to + // sample that database only. + var err error + samples, err = discovery.SampleRepository(ctx, s.repository, sampleParams) + if err != nil { + err = fmt.Errorf("error gathering repository data samples: %w", err) + // If we didn't get any samples, just return the error. + if len(samples) == 0 { + return err + } + // There were error(s) during sampling, but we still got some + // samples. Just warn and continue. + log.WithError(err).Warn("error gathering repository data samples") + } + } + + // Classify sampled data + classifications, err := classifySamples(ctx, samples, s.classifier) + if err != nil { + return fmt.Errorf("error classifying samples: %w", err) + } + + // Publish classifications if necessary + if len(classifications) == 0 { + log.Info("No discovered classifications") + } else if err := s.publisher.PublishClassifications(ctx, s.config.Repo.Host, classifications); err != nil { + return fmt.Errorf("error publishing classifications: %w", err) + } + + // Done! + return nil +} + +// Cleanup performs cleanup operations for the RepoScanner. +func (s *RepoScanner) Cleanup() { + // Nil checks are prevent panics if deps are not yet initialized. + if s.repository != nil { + _ = s.repository.Close() + } +} + +// classifySamples uses the provided classifiers to classify the sample data +// passed via the "samples" parameter. It is mostly a helper function which +// loops through each repository.Sample, retrieves the attribute names and +// values of that sample, passes them to Classifier.Classify, and then +// aggregates the results. Please see the documentation for Classifier and its +// Classify method for more details. The returned slice represents all the +// unique classification results for a given sample set. +func classifySamples( + ctx context.Context, + samples []discovery.Sample, + classifier classification.Classifier, +) ([]classification.ClassifiedTable, error) { + tables := make([]classification.ClassifiedTable, 0, len(samples)) + for _, sample := range samples { + // Classify each sampled row and combine the results. + result := make(classification.Result) + for _, sampleResult := range sample.Results { + res, err := classifier.Classify(ctx, sampleResult) + if err != nil { + return nil, fmt.Errorf("error classifying sample: %w", err) + } + result.Merge(res) + } + if len(result) > 0 { + table := classification.ClassifiedTable{ + Repo: sample.Metadata.Repo, + Database: sample.Metadata.Database, + Schema: sample.Metadata.Schema, + Table: sample.Metadata.Table, + Classifications: result, + } + tables = append(tables, table) + } + } + return tables, nil +} diff --git a/scan/repo_scanner_test.go b/scan/repo_scanner_test.go new file mode 100644 index 0000000..95c4e50 --- /dev/null +++ b/scan/repo_scanner_test.go @@ -0,0 +1,247 @@ +package scan + +import ( + "context" + "testing" + + "github.com/cyralinc/dmap/classification" + "github.com/cyralinc/dmap/discovery" + "github.com/cyralinc/dmap/scan/mocks" + + "github.com/stretchr/testify/require" +) + +func Test_classifySamples_SingleTable(t *testing.T) { + ctx := context.Background() + meta := discovery.SampleMetadata{ + Repo: "repo", + Database: "db", + Schema: "schema", + Table: "table", + } + + sample := discovery.Sample{ + Metadata: meta, + Results: []discovery.SampleResult{ + { + "age": "52", + "social_sec_num": "512-23-4258", + "credit_card_num": "4111111111111111", + }, + { + "age": "101", + "social_sec_num": "foobarbaz", + "credit_card_num": "4111111111111111", + }, + }, + } + + classifier := mocks.NewClassifier(t) + // Need to explicitly convert it to a map because Mockery isn't smart enough + // to infer the type. + classifier.EXPECT().Classify(ctx, map[string]any(sample.Results[0])).Return( + classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + }, + "social_sec_num": { + "SSN": {Name: "SSN"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + nil, + ) + classifier.EXPECT().Classify(ctx, map[string]any(sample.Results[1])).Return( + classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + "CVV": {Name: "CVV"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + nil, + ) + + expected := []classification.ClassifiedTable{ + { + Repo: meta.Repo, + Database: meta.Database, + Schema: meta.Schema, + Table: meta.Table, + Classifications: classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + "CVV": {Name: "CVV"}, + }, + "social_sec_num": { + "SSN": {Name: "SSN"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + }, + } + actual, err := classifySamples(ctx, []discovery.Sample{sample}, classifier) + require.NoError(t, err) + require.Len(t, actual, len(expected)) + for i := range actual { + requireClassifiedTableEqual(t, expected[i], actual[i]) + } +} + +func Test_classifySamples_MultipleTables(t *testing.T) { + ctx := context.Background() + meta1 := discovery.SampleMetadata{ + Repo: "repo1", + Database: "db1", + Schema: "schema1", + Table: "table1", + } + meta2 := discovery.SampleMetadata{ + Repo: "repo2", + Database: "db2", + Schema: "schema2", + Table: "table2", + } + + samples := []discovery.Sample{ + { + Metadata: meta1, + Results: []discovery.SampleResult{ + { + "age": "52", + "social_sec_num": "512-23-4258", + "credit_card_num": "4111111111111111", + }, + { + "age": "101", + "social_sec_num": "foobarbaz", + "credit_card_num": "4111111111111111", + }, + }, + }, + { + Metadata: meta2, + Results: []discovery.SampleResult{ + { + "fullname": "John Doe", + "dob": "2000-01-01", + "random": "foobarbaz", + }, + }, + }, + } + + classifier := mocks.NewClassifier(t) + // Need to explicitly convert it to a map because Mockery isn't smart enough + // to infer the type. + classifier.EXPECT().Classify(ctx, map[string]any(samples[0].Results[0])).Return( + classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + }, + "social_sec_num": { + "SSN": {Name: "SSN"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + nil, + ) + classifier.EXPECT().Classify(ctx, map[string]any(samples[0].Results[1])).Return( + classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + "CVV": {Name: "CVV"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + nil, + ) + classifier.EXPECT().Classify(ctx, map[string]any(samples[1].Results[0])).Return( + classification.Result{ + "fullname": { + "FULL_NAME": {Name: "FULL_NAME"}, + }, + "dob": { + "DOB": {Name: "DOB"}, + }, + }, + nil, + ) + + expected := []classification.ClassifiedTable{ + { + Repo: meta1.Repo, + Database: meta1.Database, + Schema: meta1.Schema, + Table: meta1.Table, + Classifications: classification.Result{ + "age": { + "AGE": {Name: "AGE"}, + "CVV": {Name: "CVV"}, + }, + "social_sec_num": { + "SSN": {Name: "SSN"}, + }, + "credit_card_num": { + "CCN": {Name: "CCN"}, + }, + }, + }, + { + Repo: meta2.Repo, + Database: meta2.Database, + Schema: meta2.Schema, + Table: meta2.Table, + Classifications: classification.Result{ + "fullname": { + "FULL_NAME": {Name: "FULL_NAME"}, + }, + "dob": { + "DOB": {Name: "DOB"}, + }, + }, + }, + } + actual, err := classifySamples(ctx, samples, classifier) + require.NoError(t, err) + require.Len(t, actual, len(expected)) + for i := range actual { + requireClassifiedTableEqual(t, expected[i], actual[i]) + } +} + +func requireClassifiedTableEqual(t *testing.T, expected, actual classification.ClassifiedTable) { + require.Equal(t, expected.Repo, actual.Repo) + require.Equal(t, expected.Database, actual.Database) + require.Equal(t, expected.Schema, actual.Schema) + require.Equal(t, expected.Table, actual.Table) + requireResultEqual(t, expected.Classifications, actual.Classifications) +} + +func requireResultEqual(t *testing.T, want, got classification.Result) { + require.Len(t, got, len(want)) + for k, v := range want { + gotSet, ok := got[k] + require.Truef(t, ok, "missing attribute %s", k) + requireLabelSetEqual(t, v, gotSet) + } +} + +func requireLabelSetEqual(t *testing.T, want, got classification.LabelSet) { + require.Len(t, got, len(want)) + for k, v := range want { + gotLbl, ok := got[k] + require.Truef(t, ok, "missing label %s", k) + require.Equal(t, v.Name, gotLbl.Name) + } +} diff --git a/scan/scanner.go b/scan/scanner.go deleted file mode 100644 index ddece19..0000000 --- a/scan/scanner.go +++ /dev/null @@ -1,66 +0,0 @@ -// Package scan provides a Scanner interface which can be used for scanning -// cloud environments and performing data repository discovery. -package scan - -import ( - "context" - "errors" - "time" -) - -// Scanner is an interface that should be implemented for a specific cloud -// provider (e.g. AWS, GCP, etc.). It defines the Scan method responsible for -// scanning the existing data repositories of the corresponding cloud provider -// environment. -type Scanner interface { - Scan(ctx context.Context) (*ScanResults, error) -} - -// RepoType defines the AWS data repository types supported (e.g. RDS, Redshift, -// DynamoDB, etc). -type RepoType string - -const ( - RepoTypeRDS RepoType = "TYPE_RDS" - RepoTypeRedshift RepoType = "TYPE_REDSHIFT" - RepoTypeDynamoDB RepoType = "TYPE_DYNAMODB" - RepoTypeS3 RepoType = "TYPE_S3" - RepoTypeDocumentDB RepoType = "TYPE_DOCUMENTDB" -) - -// Repository represents a scanned data repository. -type Repository struct { - Id string - Name string - Type RepoType - CreatedAt time.Time - Tags []string - Properties any -} - -// ScanResults represents the results of a repository scan, including all the -// data repositories that were scanned. The map key is the repository ID and the -// value is the repository itself. -type ScanResults struct { - Repositories map[string]Repository -} - -// ScanError is an error type that represents a collection of errors that -// occurred during the scanning process. -type ScanError struct { - Errs []error -} - -func (e *ScanError) Error() string { - if e == nil { - return "" - } - return errors.Join(e.Errs...).Error() -} - -func (e *ScanError) Unwrap() []error { - if e == nil { - return nil - } - return e.Errs -}