CSHARP-4255: Automatically create Queryable Encryption keys.

src/MongoDB.Driver.Core/Core/Encryption/EncryptedCollectionHelper.cs

@@ -76,6 +76,30 @@ public static BsonDocument GetEffectiveEncryptedFields(CollectionNamespace colle
             }
         }
 
+        public static IEnumerable<BsonDocument> IterateEmptyKeyIds(CollectionNamespace collectionNamespace, BsonDocument encryptedFields)
+        {
+            if (!EncryptedCollectionHelper.TryGetEffectiveEncryptedFields(collectionNamespace, encryptedFields, encryptedFieldsMap: null, out var storedEncryptedFields))
+            {
+                throw new InvalidOperationException("There are no encrypted fields defined for the collection.");
+            }
+
+            if (storedEncryptedFields.TryGetValue("fields", out var fields) && fields is BsonArray fieldsArray)
+            {
+                foreach (var field in fieldsArray.OfType<BsonDocument>()) // If `F` is not a document element, skip it.
+                {
+                    if (field.TryGetElement("keyId", out var keyId) && keyId.Value == BsonNull.Value)
+                    {
+                        yield return field;
+                    }
+                }
+            }
+        }
+
+        public static void ModifyEncryptedFields(BsonDocument fieldDocument, Guid dataKey)
+        {
+            fieldDocument["keyId"] = new BsonBinaryData(dataKey, GuidRepresentation.Standard);
+        }
+
         public enum HelperCollectionForEncryption
         {
             Esc,

src/MongoDB.Driver/Encryption/ClientEncryption.cs

@@ -20,6 +20,7 @@
 using MongoDB.Bson;
 using MongoDB.Driver.Core.Clusters;
 using MongoDB.Driver.Core.Configuration;
+using MongoDB.Driver.Core.Misc;
 using MongoDB.Libmongocrypt;
 
 namespace MongoDB.Driver.Encryption
@@ -78,6 +79,64 @@ public BsonDocument AddAlternateKeyName(Guid id, string alternateKeyName, Cancel
         public Task<BsonDocument> AddAlternateKeyNameAsync(Guid id, string alternateKeyName, CancellationToken cancellationToken = default) =>
             _libMongoCryptController.AddAlternateKeyNameAsync(id, alternateKeyName, cancellationToken);
 
+        /// <summary>
+        /// Create encrypted collection.
+        /// </summary>
+        /// <param name="collectionNamespace">The collection namespace.</param>
+        /// <param name="createCollectionOptions">The create collection options.</param>
+        /// <param name="kmsProvider">The kms provider.</param>
+        /// <param name="dataKeyOptions">The datakey options.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <remarks>
+        /// if EncryptionFields contains a keyId with a null value, a data key will be automatically generated and assigned to keyId value.
+        /// </remarks>
+        public void CreateEncryptedCollection<TCollection>(CollectionNamespace collectionNamespace, CreateCollectionOptions createCollectionOptions, string kmsProvider, DataKeyOptions dataKeyOptions, CancellationToken cancellationToken = default)
+        {
+            Ensure.IsNotNull(collectionNamespace, nameof(collectionNamespace));
+            Ensure.IsNotNull(createCollectionOptions, nameof(createCollectionOptions));
+            Ensure.IsNotNull(dataKeyOptions, nameof(dataKeyOptions));
+            Ensure.IsNotNull(kmsProvider, nameof(kmsProvider));
+
+            foreach (var fieldDocument in EncryptedCollectionHelper.IterateEmptyKeyIds(collectionNamespace, createCollectionOptions.EncryptedFields))
+            {
+                var dataKey = CreateDataKey(kmsProvider, dataKeyOptions, cancellationToken);
+                EncryptedCollectionHelper.ModifyEncryptedFields(fieldDocument, dataKey);
+            }
+
+            var database = _libMongoCryptController.KeyVaultClient.GetDatabase(collectionNamespace.DatabaseNamespace.DatabaseName);
+
+            database.CreateCollection(collectionNamespace.CollectionName, createCollectionOptions, cancellationToken);
+        }
+
+        /// <summary>
+        /// Create encrypted collection.
+        /// </summary>
+        /// <param name="collectionNamespace">The collection namespace.</param>
+        /// <param name="createCollectionOptions">The create collection options.</param>
+        /// <param name="kmsProvider">The kms provider.</param>
+        /// <param name="dataKeyOptions">The datakey options.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <remarks>
+        /// if EncryptionFields contains a keyId with a null value, a data key will be automatically generated and assigned to keyId value.
+        /// </remarks>
+        public async Task CreateEncryptedCollectionAsync<TCollection>(CollectionNamespace collectionNamespace, CreateCollectionOptions createCollectionOptions, string kmsProvider, DataKeyOptions dataKeyOptions, CancellationToken cancellationToken = default)
+        {
+            Ensure.IsNotNull(collectionNamespace, nameof(collectionNamespace));
+            Ensure.IsNotNull(createCollectionOptions, nameof(createCollectionOptions));
+            Ensure.IsNotNull(dataKeyOptions, nameof(dataKeyOptions));
+            Ensure.IsNotNull(kmsProvider, nameof(kmsProvider));
+
+            foreach (var fieldDocument in EncryptedCollectionHelper.IterateEmptyKeyIds(collectionNamespace, createCollectionOptions.EncryptedFields))
+            {
+                var dataKey = await CreateDataKeyAsync(kmsProvider, dataKeyOptions, cancellationToken).ConfigureAwait(false);
+                EncryptedCollectionHelper.ModifyEncryptedFields(fieldDocument, dataKey);
+            }
+
+            var database = _libMongoCryptController.KeyVaultClient.GetDatabase(collectionNamespace.DatabaseNamespace.DatabaseName);
+
+            await database.CreateCollectionAsync(collectionNamespace.CollectionName, createCollectionOptions, cancellationToken).ConfigureAwait(false);
+        }
+
         /// <summary>
         /// An alias function equivalent to createKey.
         /// </summary>

src/MongoDB.Driver/Encryption/LibMongoCryptControllerBase.cs

@@ -61,6 +61,9 @@ protected LibMongoCryptControllerBase(
             _tlsOptions = Ensure.IsNotNull(encryptionOptions.TlsOptions, nameof(encryptionOptions.TlsOptions));
         }
 
+        // public properties
+        public IMongoClient KeyVaultClient => _keyVaultClient;
+
         // protected methods
         protected void FeedResult(CryptContext context, BsonDocument document)
         {

tests/MongoDB.Driver.Tests/Specifications/client-side-encryption/prose-tests/ClientEncryptionProseTests.cs

@@ -33,7 +33,6 @@
 using MongoDB.Driver.Core.Authentication.External;
 using MongoDB.Driver.Core.Bindings;
 using MongoDB.Driver.Core.Clusters;
-using MongoDB.Driver.Core.Configuration;
 using MongoDB.Driver.Core.Events;
 using MongoDB.Driver.Core.Misc;
 using MongoDB.Driver.Core.Operations;
@@ -84,6 +83,76 @@ public ClientEncryptionProseTests(ITestOutputHelper testOutputHelper)
         }
 
         // public methods
+        [SkippableTheory]
+        [ParameterAttributeData]
+        public void AutomaticDataEncryptionKeysTest(
+            [Range(1, 4)] int testCase,
+            [Values(false, true)] bool async)
+        {
+            RequireServer.Check().Supports(Feature.Csfle2).ClusterTypes(ClusterType.ReplicaSet, ClusterType.Sharded, ClusterType.LoadBalanced);
+
+            var kmsProvider = "local";
+            using (var client = ConfigureClient())
+            using (var clientEncryption = ConfigureClientEncryption(client, kmsProviderFilter: kmsProvider))
+            {
+                var encryptedFields = BsonDocument.Parse($@"
+                {{
+                    fields:
+                    [
+                    {{
+                        path: ""ssn"",
+                        bsonType: ""string"",
+                        keyId: null
+                    }}
+                    ]
+                }}");
+
+                DropCollection(__collCollectionNamespace, encryptedFields);
+
+                RunTestCase(testCase);
+
+                void RunTestCase(int testCase)
+                {
+                    switch (testCase)
+                    {
+                        case 1: // Case 1: Simple Creation and Validation
+                            {
+                                var collection = CreateEncryptedCollection(client, clientEncryption, __collCollectionNamespace, encryptedFields, kmsProvider, async);
+
+                                var exception = Record.Exception(() => Insert(collection, async, new BsonDocument("ssn", "123-45-6789")));
+                                exception.Should().BeOfType<MongoBulkWriteException<BsonDocument>>().Which.Message.Should().Contain("Document failed validation");
+                            }
+                            break;
+                        case 2: // Case 2: Missing ``encryptedFields``
+                            {
+                                var exception = Record.Exception(() => CreateEncryptedCollection(client, clientEncryption, __collCollectionNamespace, encryptedFields: null, kmsProvider, async));
+
+                                exception.Should().BeOfType<InvalidOperationException>().Which.Message.Should().Contain("There are no encrypted fields defined for the collection.") ;
+                            }
+                            break;
+                        case 3: // Case 3: Invalid ``keyId``
+                            {
+                                var effectiveEncryptedFields = encryptedFields.DeepClone();
+                                effectiveEncryptedFields["fields"].AsBsonArray[0].AsBsonDocument["keyId"] = false;
+                                var exception = Record.Exception(() => CreateEncryptedCollection(client, clientEncryption, __collCollectionNamespace, effectiveEncryptedFields.AsBsonDocument, kmsProvider, async));
+                                exception.Should().BeOfType<MongoCommandException>().Which.Message.Should().Contain("BSON field 'create.encryptedFields.fields.keyId' is the wrong type 'bool', expected type 'binData'");
+                            }
+                            break;
+                       case 4: // Case 4: Insert encrypted value
+                            {
+                                var createCollectionOptions = new CreateCollectionOptions { EncryptedFields = encryptedFields };
+                                var collection = CreateEncryptedCollection(client, clientEncryption, __collCollectionNamespace, createCollectionOptions, kmsProvider, async);
+                                var dataKey = createCollectionOptions.EncryptedFields["fields"].AsBsonArray[0].AsBsonDocument["keyId"].AsGuid; // get generated datakey
+                                var encryptedValue = ExplicitEncrypt(clientEncryption, new EncryptOptions(algorithm: EncryptionAlgorithm.Unindexed, keyId: dataKey), "123-45-6789", async); // use explicit encryption to encrypt data before inserting
+                                Insert(collection, async, new BsonDocument("ssn", encryptedValue));
+                            }
+                            break;
+                        default: throw new Exception($"Unexpected test case {testCase}.");
+                    }
+                }
+            }
+        }
+
         [SkippableTheory]
         [ParameterAttributeData]
         public void BsonSizeLimitAndBatchSizeSplittingTest(
@@ -1025,6 +1094,7 @@ void RunTestCase(IMongoCollection<BsonDocument> decryptionEventsCollection, int
                             reply["cursor"]["firstBatch"].AsBsonArray.Single()["encrypted"].AsBsonBinaryData.SubType.Should().Be(BsonBinarySubType.Encrypted);
                         }
                         break;
+                    default: throw new Exception($"Unexpected test case {testCase}.");
                 }
             }
 
@@ -1819,6 +1889,8 @@ HttpClientWrapperWithModifiedRequest CreateHttpClientWrapperWithModifiedRequest(
             }
         }
 
+        [SkippableTheory]
+        [ParameterAttributeData]
         public void RewrapTest(
             [Values("local", "aws", "azure", "gcp", "kmip")] string srcProvider,
             [Values("local", "aws", "azure", "gcp", "kmip")] string dstProvider,
@@ -1873,7 +1945,7 @@ public void ViewAreProhibitedTest([Values(false, true)] bool async)
             using (var client = ConfigureClient(false))
             using (var clientEncrypted = ConfigureClientEncrypted(kmsProviderFilter: "local"))
             {
-                DropView(viewName);
+                DropCollection(viewName);
                 client
                     .GetDatabase(viewName.DatabaseNamespace.DatabaseName)
                     .CreateView(
@@ -2201,6 +2273,28 @@ private void CreateCollection(IMongoClient client, CollectionNamespace collectio
                     });
         }
 
+        private IMongoCollection<BsonDocument> CreateEncryptedCollection(IMongoClient client, ClientEncryption clientEncryption, CollectionNamespace collectionNamespace, BsonDocument encryptedFields, string kmsProvider, bool async)
+        {
+            var createCollectionOptions = new CreateCollectionOptions { EncryptedFields = encryptedFields };
+            return CreateEncryptedCollection(client, clientEncryption, collectionNamespace, createCollectionOptions, kmsProvider, async);
+        }
+
+        private IMongoCollection<BsonDocument> CreateEncryptedCollection(IMongoClient client, ClientEncryption clientEncryption, CollectionNamespace collectionNamespace, CreateCollectionOptions createCollectionOptions, string kmsProvider, bool async)
+        {
+            var datakeyOptions = CreateDataKeyOptions(kmsProvider);
+
+            if (async)
+            {
+                clientEncryption.CreateEncryptedCollectionAsync<BsonDocument>(collectionNamespace, createCollectionOptions, kmsProvider, datakeyOptions, cancellationToken: default).GetAwaiter().GetResult();
+            }
+            else
+            { 
+                clientEncryption.CreateEncryptedCollection<BsonDocument>(collectionNamespace, createCollectionOptions, kmsProvider, datakeyOptions, cancellationToken: default);
+            }
+
+            return client.GetDatabase(collectionNamespace.DatabaseNamespace.DatabaseName).GetCollection<BsonDocument>(collectionNamespace.CollectionName);
+        }
+
         private Guid CreateDataKey(
             ClientEncryption clientEncryption,
             string kmsProvider,
@@ -2351,9 +2445,9 @@ private MongoClientSettings CreateMongoClientSettings(
             return mongoClientSettings;
         }
 
-        private void DropView(CollectionNamespace viewNamespace)
+        private void DropCollection(CollectionNamespace collectionNamespace, BsonDocument encryptedFields = null)
         {
-            var operation = new DropCollectionOperation(viewNamespace, CoreTestConfiguration.MessageEncoderSettings);
+            var operation = DropCollectionOperation.CreateEncryptedDropCollectionOperationIfConfigured(collectionNamespace, encryptedFields, CoreTestConfiguration.MessageEncoderSettings, configureDropCollectionConfigurator: null);
             using (var session = CoreTestConfiguration.StartSession(_cluster))
             using (var binding = new WritableServerBinding(_cluster, session.Fork()))
             using (var bindingHandle = new ReadWriteBindingHandle(binding))