From 18a7e7e16cf00b9167a49d318a618b8281304811 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Fri, 31 May 2024 15:26:13 -0400 Subject: [PATCH 1/8] GODRIVER-3140 Update spec tests. --- .../legacy/timeoutMS.json | 200 ++++++++++++++++++ .../legacy/timeoutMS.yml | 67 ++++++ 2 files changed, 267 insertions(+) create mode 100644 testdata/client-side-encryption/legacy/timeoutMS.json create mode 100644 testdata/client-side-encryption/legacy/timeoutMS.yml diff --git a/testdata/client-side-encryption/legacy/timeoutMS.json b/testdata/client-side-encryption/legacy/timeoutMS.json new file mode 100644 index 0000000000..9eb0974d40 --- /dev/null +++ b/testdata/client-side-encryption/legacy/timeoutMS.json @@ -0,0 +1,200 @@ +{ + "runOn": [ + { + "minServerVersion": "4.4" + } + ], + "database_name": "cse-timeouts-db", + "collection_name": "cse-timeouts-coll", + "data": [], + "json_schema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + }, + "key_vault_data": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ], + "tests": [ + { + "description": "timeoutMS applied to listCollections to get collection schema", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "blockConnection": true, + "blockTimeMS": 60 + } + }, + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + } + }, + "timeoutMS": 50 + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0", + "random": "abc" + } + }, + "result": { + "isTimeoutError": true + } + } + ], + "expectations": [ + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "cse-timeouts-coll" + }, + "maxTimeMS": { + "$$type": [ + "int", + "long" + ] + } + }, + "command_name": "listCollections" + } + } + ] + }, + { + "description": "remaining timeoutMS applied to find to get keyvault data", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "listCollections", + "find" + ], + "blockConnection": true, + "blockTimeMS": 30 + } + }, + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + } + }, + "timeoutMS": 50 + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0", + "random": "abc" + } + }, + "result": { + "isTimeoutError": true + } + } + ] + } + ] + } \ No newline at end of file diff --git a/testdata/client-side-encryption/legacy/timeoutMS.yml b/testdata/client-side-encryption/legacy/timeoutMS.yml new file mode 100644 index 0000000000..bb71d67650 --- /dev/null +++ b/testdata/client-side-encryption/legacy/timeoutMS.yml @@ -0,0 +1,67 @@ +runOn: + - minServerVersion: "4.4" +database_name: &database_name "cse-timeouts-db" +collection_name: &collection_name "cse-timeouts-coll" + +data: [] +json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} +key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}] + +tests: + - description: "timeoutMS applied to listCollections to get collection schema" + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["listCollections"] + blockConnection: true + blockTimeMS: 60 + clientOptions: + autoEncryptOpts: + kmsProviders: + aws: {} # Credentials filled in from environment. + timeoutMS: 50 + operations: + - name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0", random: "abc" } + result: + isTimeoutError: true + expectations: + # Auto encryption will request the collection info. + - command_started_event: + command: + listCollections: 1 + filter: + name: *collection_name + maxTimeMS: { $$type: ["int", "long"] } + command_name: listCollections + + # Test that timeoutMS applies to the sum of all operations done for client-side encryption. This is done by blocking + # listCollections and find for 30ms each and running an insertOne with timeoutMS=50. There should be one + # listCollections command and one "find" command, so the sum should take more than timeoutMS. A second listCollections + # event doesn't occur due to the internal MongoClient lacking configured auto encryption, plus libmongocrypt holds the + # collection schema in cache for a minute. + # + # This test does not include command monitoring expectations because the exact command sequence is dependent on the + # amount of time taken by mongocryptd communication. In slow runs, mongocryptd communication can breach the timeout + # and result in the final "find" not being sent. + - description: "remaining timeoutMS applied to find to get keyvault data" + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: ["listCollections", "find"] + blockConnection: true + blockTimeMS: 30 + clientOptions: + autoEncryptOpts: + kmsProviders: + aws: {} # Credentials filled in from environment. + timeoutMS: 50 + operations: + - name: insertOne + arguments: + document: *doc0 + result: + isTimeoutError: true \ No newline at end of file From 5ab8f1574e069eedfbc908eac4972e45b0129aad Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Fri, 31 May 2024 18:32:43 -0400 Subject: [PATCH 2/8] GODRIVER-3140 Update help function. --- mongo/integration/json_helpers_test.go | 14 ++++++++++++-- mongo/integration/unified_spec_test.go | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mongo/integration/json_helpers_test.go b/mongo/integration/json_helpers_test.go index 463d1e54dc..51559debca 100644 --- a/mongo/integration/json_helpers_test.go +++ b/mongo/integration/json_helpers_test.go @@ -114,6 +114,8 @@ func createClientOptions(t testing.TB, opts bson.Raw) *options.ClientOptions { case "socketTimeoutMS": st := convertValueToMilliseconds(t, opt) clientOpts.SetSocketTimeout(st) + case "timeoutMS": + clientOpts.SetTimeout(time.Duration(opt.Int32()) * time.Millisecond) case "minPoolSize": clientOpts.SetMinPoolSize(uint64(opt.AsInt64())) case "maxPoolSize": @@ -470,8 +472,9 @@ func errorFromResult(t testing.TB, result interface{}) *operationError { if err != nil { return nil } - if expected.ErrorCodeName == nil && expected.ErrorContains == nil && len(expected.ErrorLabelsOmit) == 0 && - len(expected.ErrorLabelsContain) == 0 { + if expected.ErrorCodeName == nil && expected.ErrorContains == nil && + len(expected.ErrorLabelsOmit) == 0 && len(expected.ErrorLabelsContain) == 0 && + expected.IsTimeoutError == nil { return nil } @@ -563,6 +566,13 @@ func verifyError(expected *operationError, actual error) error { return fmt.Errorf("expected error %w to not contain label %q", actual, label) } } + if expected.IsTimeoutError != nil { + isTimeoutError := mongo.IsTimeout(actual) + if *expected.IsTimeoutError != isTimeoutError { + return fmt.Errorf("expected error %w to be a timeout error: %v, is timeout error: %v", + actual, *expected.IsTimeoutError, isTimeoutError) + } + } return nil } diff --git a/mongo/integration/unified_spec_test.go b/mongo/integration/unified_spec_test.go index a28e9f6cf0..ac13577c60 100644 --- a/mongo/integration/unified_spec_test.go +++ b/mongo/integration/unified_spec_test.go @@ -175,6 +175,7 @@ type operationError struct { ErrorCodeName *string `bson:"errorCodeName"` ErrorLabelsContain []string `bson:"errorLabelsContain"` ErrorLabelsOmit []string `bson:"errorLabelsOmit"` + IsTimeoutError *bool `bson:"isTimeoutError"` } const dataPath string = "../../testdata/" From b0455bf04c1ef423e9caef631acb8575d7083b61 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Sat, 1 Jun 2024 00:44:10 -0400 Subject: [PATCH 3/8] minor fix for configureFailPoint --- .../cmd_monitoring_helpers_test.go | 19 ++++++++++++++++++- mongo/integration/mtest/mongotest.go | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongo/integration/cmd_monitoring_helpers_test.go b/mongo/integration/cmd_monitoring_helpers_test.go index 6c4931d26c..0d41ad94d1 100644 --- a/mongo/integration/cmd_monitoring_helpers_test.go +++ b/mongo/integration/cmd_monitoring_helpers_test.go @@ -82,7 +82,24 @@ func compareValues(mt *mtest.T, key string, expected, actual bson.RawValue) erro if typeVal, err := e.LookupErr("$$type"); err == nil { // $$type represents a type assertion // for example {field: {$$type: "binData"}} should assert that "field" is an element with a binary value - return checkValueType(mt, key, actual.Type, typeVal.StringValue()) + switch t := typeVal.Type; t { + case bson.TypeString: + return checkValueType(mt, key, actual.Type, typeVal.StringValue()) + case bson.TypeArray: + array := typeVal.Array() + elems, err := array.Values() + if err != nil { + return err + } + for _, elem := range elems { + if checkValueType(mt, key, actual.Type, elem.StringValue()) == nil { + return nil + } + } + return fmt.Errorf("BSON type mismatch for key %s; expected %s, got %s", key, array.String(), actual) + default: + return fmt.Errorf("unsupported $$type: %s", t.String()) + } } a := actual.Document() diff --git a/mongo/integration/mtest/mongotest.go b/mongo/integration/mtest/mongotest.go index f92b5c583f..b14f65eb6c 100644 --- a/mongo/integration/mtest/mongotest.go +++ b/mongo/integration/mtest/mongotest.go @@ -586,6 +586,11 @@ func (t *T) TrackFailPoint(fpName string) { // ClearFailPoints disables all previously set failpoints for this test. func (t *T) ClearFailPoints() { + if t.clientOpts != nil && t.clientOpts.AutoEncryptionOptions != nil && len(t.failPointNames) > 0 { + t.Logf("configureFailPoint is not supported for auto encryption, skipping ClearFailPoints()") + t.failPointNames = t.failPointNames[:0] + return + } db := t.Client.Database("admin") for _, fp := range t.failPointNames { cmd := bson.D{ From 47abc06e71aed1606a8023c286f92f97934c45cf Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Tue, 11 Jun 2024 13:14:49 -0400 Subject: [PATCH 4/8] updates --- mongo/integration/cmd_monitoring_helpers_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongo/integration/cmd_monitoring_helpers_test.go b/mongo/integration/cmd_monitoring_helpers_test.go index 0d41ad94d1..ef821d9e9a 100644 --- a/mongo/integration/cmd_monitoring_helpers_test.go +++ b/mongo/integration/cmd_monitoring_helpers_test.go @@ -82,7 +82,7 @@ func compareValues(mt *mtest.T, key string, expected, actual bson.RawValue) erro if typeVal, err := e.LookupErr("$$type"); err == nil { // $$type represents a type assertion // for example {field: {$$type: "binData"}} should assert that "field" is an element with a binary value - switch t := typeVal.Type; t { + switch typ := typeVal.Type; typ { case bson.TypeString: return checkValueType(mt, key, actual.Type, typeVal.StringValue()) case bson.TypeArray: @@ -96,9 +96,9 @@ func compareValues(mt *mtest.T, key string, expected, actual bson.RawValue) erro return nil } } - return fmt.Errorf("BSON type mismatch for key %s; expected %s, got %s", key, array.String(), actual) + return fmt.Errorf("BSON type mismatch for key %q; expected %s, got %q", key, array, actual.Type) default: - return fmt.Errorf("unsupported $$type: %s", t.String()) + return fmt.Errorf("unsupported $$type: %q", typ) } } From 3ddf46acc8ffdf461bd38fa5aaea4e84422fe8b9 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Mon, 17 Jun 2024 18:14:13 -0400 Subject: [PATCH 5/8] clear logic --- internal/test/compilecheck/go.mod | 8 ++++---- internal/test/compilecheck/go.sum | 19 +++++++++---------- mongo/integration/mtest/mongotest.go | 5 ----- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/internal/test/compilecheck/go.mod b/internal/test/compilecheck/go.mod index 69d192022a..cc09124838 100644 --- a/internal/test/compilecheck/go.mod +++ b/internal/test/compilecheck/go.mod @@ -9,14 +9,14 @@ replace go.mongodb.org/mongo-driver => ../../../ require go.mongodb.org/mongo-driver v1.11.7 require ( - github.com/golang/snappy v0.0.1 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.13.6 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/text v0.14.0 // indirect ) diff --git a/internal/test/compilecheck/go.sum b/internal/test/compilecheck/go.sum index fe79e66209..802402a881 100644 --- a/internal/test/compilecheck/go.sum +++ b/internal/test/compilecheck/go.sum @@ -1,11 +1,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -17,16 +17,16 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -44,4 +44,3 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/mongo/integration/mtest/mongotest.go b/mongo/integration/mtest/mongotest.go index b14f65eb6c..f92b5c583f 100644 --- a/mongo/integration/mtest/mongotest.go +++ b/mongo/integration/mtest/mongotest.go @@ -586,11 +586,6 @@ func (t *T) TrackFailPoint(fpName string) { // ClearFailPoints disables all previously set failpoints for this test. func (t *T) ClearFailPoints() { - if t.clientOpts != nil && t.clientOpts.AutoEncryptionOptions != nil && len(t.failPointNames) > 0 { - t.Logf("configureFailPoint is not supported for auto encryption, skipping ClearFailPoints()") - t.failPointNames = t.failPointNames[:0] - return - } db := t.Client.Database("admin") for _, fp := range t.failPointNames { cmd := bson.D{ From 928555a6fafecbb6c43cd476b7335814ccb883ca Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Thu, 8 Aug 2024 18:12:53 -0400 Subject: [PATCH 6/8] updates --- mongo/integration/mtest/mongotest.go | 75 +++++++++++++++++--------- mongo/integration/unified_spec_test.go | 12 ++--- 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/mongo/integration/mtest/mongotest.go b/mongo/integration/mtest/mongotest.go index 25f30849b0..7fd40890d3 100644 --- a/mongo/integration/mtest/mongotest.go +++ b/mongo/integration/mtest/mongotest.go @@ -84,6 +84,11 @@ type WriteConcernErrorData struct { ErrInfo bson.Raw `bson:"errInfo,omitempty"` } +type failPoint struct { + name string + client *mongo.Client +} + // T is a wrapper around testing.T. type T struct { // connsCheckedOut is the net number of connections checked out during test execution. @@ -103,7 +108,7 @@ type T struct { createdColls []*Collection // collections created in this test proxyDialer *proxyDialer dbName, collName string - failPointNames []string + failPoints []failPoint minServerVersion string maxServerVersion string validTopologies []TopologyKind @@ -128,14 +133,16 @@ type T struct { succeeded []*event.CommandSucceededEvent failed []*event.CommandFailedEvent - Client *mongo.Client - DB *mongo.Database - Coll *mongo.Collection + Client *mongo.Client + fpClients map[*mongo.Client]bool + DB *mongo.Database + Coll *mongo.Collection } func newT(wrapped *testing.T, opts ...*Options) *T { t := &T{ - T: wrapped, + T: wrapped, + fpClients: make(map[*mongo.Client]bool), } for _, opt := range opts { for _, optFn := range opt.optFuncs { @@ -202,6 +209,12 @@ func (t *T) cleanup() { // always disconnect the client regardless of clientType because Client.Disconnect will work against // all deployments _ = t.Client.Disconnect(context.Background()) + for client, v := range t.fpClients { + if v { + client.Disconnect(context.Background()) + } + } + t.fpClients = nil } // Run creates a new T instance for a sub-test and runs the given callback. It also creates a new collection using the @@ -254,9 +267,11 @@ func (t *T) RunOpts(name string, opts *Options, callback func(mt *T)) { sub.ClearFailPoints() sub.ClearCollections() } - // only disconnect client if it's not being shared + // only disconnect client if it's not being shared and not used by fail points. if sub.shareClient == nil || !*sub.shareClient { - _ = sub.Client.Disconnect(context.Background()) + if _, ok := sub.fpClients[sub.Client]; !ok { + _ = sub.Client.Disconnect(context.Background()) + } } assert.Equal(sub, 0, sessions, "%v sessions checked out", sessions) assert.Equal(sub, 0, conns, "%v connections checked out", conns) @@ -405,7 +420,10 @@ func (t *T) ResetClient(opts *options.ClientOptions) { t.clientOpts = opts } - _ = t.Client.Disconnect(context.Background()) + // Disconnect client if it is not being used by fail points. + if _, ok := t.fpClients[t.Client]; !ok { + _ = t.Client.Disconnect(context.Background()) + } t.createTestClient() t.DB = t.Client.Database(t.dbName) t.Coll = t.DB.Collection(t.collName, t.collOpts) @@ -562,7 +580,8 @@ func (t *T) SetFailPoint(fp FailPoint) { if err := SetFailPoint(fp, t.Client); err != nil { t.Fatal(err) } - t.failPointNames = append(t.failPointNames, fp.ConfigureFailPoint) + t.fpClients[t.Client] = true + t.failPoints = append(t.failPoints, failPoint{fp.ConfigureFailPoint, t.Client}) } // SetFailPointFromDocument sets the fail point represented by the given document for the client associated with T. This @@ -574,30 +593,35 @@ func (t *T) SetFailPointFromDocument(fp bson.Raw) { t.Fatal(err) } + t.fpClients[t.Client] = true name := fp.Index(0).Value().StringValue() - t.failPointNames = append(t.failPointNames, name) + t.failPoints = append(t.failPoints, failPoint{name, t.Client}) } // TrackFailPoint adds the given fail point to the list of fail points to be disabled when the current test finishes. // This function does not create a fail point on the server. -func (t *T) TrackFailPoint(fpName string) { - t.failPointNames = append(t.failPointNames, fpName) +func (t *T) TrackFailPoint(fpName string, client *mongo.Client) { + t.fpClients[client] = true + t.failPoints = append(t.failPoints, failPoint{fpName, client}) } // ClearFailPoints disables all previously set failpoints for this test. func (t *T) ClearFailPoints() { - db := t.Client.Database("admin") - for _, fp := range t.failPointNames { + for _, fp := range t.failPoints { cmd := bson.D{ - {"configureFailPoint", fp}, + {"configureFailPoint", fp.name}, {"mode", "off"}, } - err := db.RunCommand(context.Background(), cmd).Err() + err := fp.client.Database("admin").RunCommand(context.Background(), cmd).Err() if err != nil { - t.Fatalf("error clearing fail point %s: %v", fp, err) + t.Fatalf("error clearing fail point %s: %v", fp.name, err) + } + if fp.client != t.Client { + _ = fp.client.Disconnect(context.Background()) + t.fpClients[fp.client] = false } } - t.failPointNames = t.failPointNames[:0] + t.failPoints = t.failPoints[:0] } // CloneDatabase modifies the default database for this test to match the given options. @@ -684,19 +708,17 @@ func (t *T) createTestClient() { }) } - var err error + var uriOpts *options.ClientOptions switch t.clientType { case Pinned: // pin to first mongos pinnedHostList := []string{testContext.connString.Hosts[0]} - uriOpts := options.Client().ApplyURI(testContext.connString.Original).SetHosts(pinnedHostList) - t.Client, err = mongo.NewClient(uriOpts, clientOpts) + uriOpts = options.Client().ApplyURI(testContext.connString.Original).SetHosts(pinnedHostList) case Mock: // clear pool monitor to avoid configuration error clientOpts.PoolMonitor = nil t.mockDeployment = newMockDeployment() clientOpts.Deployment = t.mockDeployment - t.Client, err = mongo.NewClient(clientOpts) case Proxy: t.proxyDialer = newProxyDialer() clientOpts.SetDialer(t.proxyDialer) @@ -706,16 +728,17 @@ func (t *T) createTestClient() { case Default: // Use a different set of options to specify the URI because clientOpts may already have a URI or host seedlist // specified. - var uriOpts *options.ClientOptions if clientOpts.Deployment == nil { // Only specify URI if the deployment is not set to avoid setting topology/server options along with the // deployment. uriOpts = options.Client().ApplyURI(testContext.connString.Original) } - - // Pass in uriOpts first so clientOpts wins if there are any conflicting settings. - t.Client, err = mongo.NewClient(uriOpts, clientOpts) } + t.clientOpts = options.MergeClientOptions(uriOpts, clientOpts) + + var err error + // Pass in uriOpts first so clientOpts wins if there are any conflicting settings. + t.Client, err = mongo.NewClient(t.clientOpts) if err != nil { t.Fatalf("error creating client: %v", err) } diff --git a/mongo/integration/unified_spec_test.go b/mongo/integration/unified_spec_test.go index ac13577c60..df2d705132 100644 --- a/mongo/integration/unified_spec_test.go +++ b/mongo/integration/unified_spec_test.go @@ -42,7 +42,7 @@ const ( gridFSFiles = "fs.files" gridFSChunks = "fs.chunks" spec1403SkipReason = "servers less than 4.2 do not have mongocryptd; see SPEC-1403" - godriver2123SkipReason = "failpoints and timeouts together cause failures; see GODRIVER-2123" + godriver2466SkipReason = "test has not been updated; see GODRIVER-2466" godriver2413SkipReason = "encryptedFields argument is not supported on Collection.Drop; see GODRIVER-2413" ) @@ -53,11 +53,8 @@ var ( // Currently, the test will fail because a server < 4.2 wouldn't have mongocryptd, so Client construction // would fail with a mongocryptd spawn error. "operation fails with maxWireVersion < 8": spec1403SkipReason, - // GODRIVER-2123: The two tests below use a failpoint and a socket or server selection timeout. - // The timeout causes the eventual clearing of the failpoint in the test runner to fail with an - // i/o timeout. - "Ignore network timeout error on find": godriver2123SkipReason, - "Network error on minPoolSize background creation": godriver2123SkipReason, + // GODRIVER-2466: The test below has not been updated as required. + "Network error on minPoolSize background creation": godriver2466SkipReason, "CreateCollection from encryptedFields.": godriver2413SkipReason, "DropCollection from encryptedFields": godriver2413SkipReason, "DropCollection from remote encryptedFields": godriver2413SkipReason, @@ -476,12 +473,11 @@ func executeTestRunnerOperation(mt *mtest.T, testCase *testCase, op *operation, if err != nil { return fmt.Errorf("Connect error for targeted client: %w", err) } - defer func() { _ = client.Disconnect(context.Background()) }() if err = client.Database("admin").RunCommand(context.Background(), fp).Err(); err != nil { return fmt.Errorf("error setting targeted fail point: %w", err) } - mt.TrackFailPoint(fp.ConfigureFailPoint) + mt.TrackFailPoint(fp.ConfigureFailPoint, client) case "configureFailPoint": fp, err := op.Arguments.LookupErr("failPoint") if err != nil { From 12fb07c3c2d3362fc90e158ad0fac840c78ae8d5 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Mon, 12 Aug 2024 16:57:17 -0400 Subject: [PATCH 7/8] minor fix --- mongo/integration/mtest/mongotest.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mongo/integration/mtest/mongotest.go b/mongo/integration/mtest/mongotest.go index 7fd40890d3..70c0f0333d 100644 --- a/mongo/integration/mtest/mongotest.go +++ b/mongo/integration/mtest/mongotest.go @@ -616,9 +616,11 @@ func (t *T) ClearFailPoints() { if err != nil { t.Fatalf("error clearing fail point %s: %v", fp.name, err) } - if fp.client != t.Client { - _ = fp.client.Disconnect(context.Background()) - t.fpClients[fp.client] = false + t.fpClients[fp.client] = false + } + for client, active := range t.fpClients { + if !active && client != t.Client { + _ = client.Disconnect(context.Background()) } } t.failPoints = t.failPoints[:0] From 163b83ec326d70fdb5a370ac70fd085990dae218 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Mon, 23 Sep 2024 18:23:03 -0400 Subject: [PATCH 8/8] minor update --- mongo/integration/mtest/mongotest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongo/integration/mtest/mongotest.go b/mongo/integration/mtest/mongotest.go index e16b9e21b4..fb0833ba32 100644 --- a/mongo/integration/mtest/mongotest.go +++ b/mongo/integration/mtest/mongotest.go @@ -211,7 +211,7 @@ func (t *T) cleanup() { _ = t.Client.Disconnect(context.Background()) for client, v := range t.fpClients { if v { - client.Disconnect(context.Background()) + _ = client.Disconnect(context.Background()) } } t.fpClients = nil