feat: Add option to use CMK and set the resources retention (#573)

.gitignore

@@ -442,6 +442,7 @@ venv
 lib/user-interface/react-app/public/aws-exports.json
 out.tmp
 bin/config.json
+bin/config*.json
 
 # Docs
 docs/.vitepress/cache

cli/magic-config.ts

@@ -145,6 +145,8 @@ const embeddingModels = [
         fs.readFileSync("./bin/config.json").toString("utf8")
       );
       options.prefix = config.prefix;
+      options.createCMKs = config.createCMKs;
+      options.retainOnDelete = config.retainOnDelete;
       options.vpcId = config.vpc?.vpcId;
       options.bedrockEnable = config.bedrock?.enabled;
       options.bedrockRegion = config.bedrock?.region;
@@ -287,6 +289,22 @@ async function processCreateOptions(options: any): Promise<void> {
         return !(this as any).state.answers.existingVpc;
       },
     },
+    {
+      type: "confirm",
+      name: "createCMKs",
+      message:
+        "Do you want to create KMS Customer Managed Keys (CMKs)? (It will be used to encrypt the data at rest.)",
+      initial: true,
+      hint: "It is recommended but enabling it on an existing environment will cause the re-creation of some of the resources (for example Aurora cluster, Open Search collection). To prevent data loss, it is recommended to use it on a new environment or at least enable retain on cleanup (needs to be deployed before enabling the use of CMK). For more information on Aurora migration, please refer to the documentation.",
+    },
+    {
+      type: "confirm",
+      name: "retainOnDelete",
+      message:
+        "Do you want to retain data stores on cleanup of the project (Logs, S3, Tables, Indexes, Cognito User pools)?",
+      initial: true,
+      hint: "It reduces the risk of deleting data. It will however not delete all the resources on cleanup (would require manual removal if relevant)",
+    },
     {
       type: "confirm",
       name: "bedrockEnable",
@@ -718,7 +736,7 @@ async function processCreateOptions(options: any): Promise<void> {
       {
         type: "input",
         name: "name",
-        message: "KnowledgeBase source name",
+        message: "Bedrock KnowledgeBase source name",
         validate(v: string) {
           return RegExp(/^\w[\w-_]*\w$/).test(v);
         },
@@ -1103,10 +1121,11 @@ async function processCreateOptions(options: any): Promise<void> {
   }
 
   const randomSuffix = randomBytes(8).toString("hex");
-
   // Create the config object
   const config = {
     prefix: answers.prefix,
+    createCMKs: answers.createCMKs,
+    retainOnDelete: answers.retainOnDelete,
     vpc: answers.existingVpc
       ? {
           vpcId: answers.vpcId.toLowerCase(),

integtests/clients/cognito_client.py

@@ -74,9 +74,9 @@ def get_credentials(self, email: str) -> Credentials:
 
     def get_password(self):
         return "".join(
-            random.choices(
+            random.choices(  # NOSONAR Only used for testing. Temporary password
                 string.ascii_uppercase, k=10
-            )  # NOSONAR Only used for testing. Temporary password
+            )
             + random.choices(string.ascii_lowercase, k=10)  # NOSONAR
             + random.choices(string.digits, k=5)  # NOSONAR
             + random.choices(string.punctuation, k=3)  # NOSONAR

integtests/user_interface/react_app/test_login.py

@@ -18,11 +18,11 @@ def test_invalid_credentials(selenium_driver):
             **{
                 "id_token": "",
                 "email": "invalid",
-                "password": "invalid",
+                "password": "invalid",  # NOSONAR
                 "aws_access_key": "",
                 "aws_secret_key": "",
                 "aws_token": "",
             }
-        )  # NOSONAR
+        )
     )
     assert page.get_error() != None

lib/authentication/index.ts

@@ -20,7 +20,10 @@ export class Authentication extends Construct {
     super(scope, id);
 
     const userPool = new cognito.UserPool(this, "UserPool", {
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      removalPolicy:
+        config.retainOnDelete === true
+          ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+          : cdk.RemovalPolicy.DESTROY,
       selfSignUpEnabled: false,
       mfa: cognito.Mfa.OPTIONAL,
       advancedSecurityMode: cognito.AdvancedSecurityMode.ENFORCED,

lib/chatbot-api/appsync-ws.ts

@@ -16,10 +16,12 @@ import { ITopic } from "aws-cdk-lib/aws-sns";
 import { UserPool } from "aws-cdk-lib/aws-cognito";
 import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
 import * as path from "path";
+import { IKey } from "aws-cdk-lib/aws-kms";
 
 interface RealtimeResolversProps {
   readonly queue: IQueue;
   readonly topic: ITopic;
+  readonly topicKey: IKey;
   readonly userPool: UserPool;
   readonly shared: Shared;
   readonly api: appsync.GraphqlApi;
@@ -89,6 +91,13 @@ export class RealtimeResolvers extends Construct {
     outgoingMessageHandler.addEventSource(new SqsEventSource(props.queue));
 
     props.topic.grantPublish(resolverFunction);
+    if (props.topicKey && resolverFunction.role) {
+      props.topicKey.grant(
+        resolverFunction.role,
+        "kms:GenerateDataKey",
+        "kms:Decrypt"
+      );
+    }
 
     const functionDataSource = props.api.addLambdaDataSource(
       "realtimeResolverFunction",

lib/chatbot-api/chatbot-dynamodb-tables/index.ts

@@ -1,12 +1,18 @@
 import * as cdk from "aws-cdk-lib";
 import { Construct } from "constructs";
 import * as dynamodb from "aws-cdk-lib/aws-dynamodb";
+import * as kms from "aws-cdk-lib/aws-kms";
+
+export interface ChatBotDynamoDBTablesProps {
+  readonly retainOnDelete?: boolean;
+  readonly kmsKey?: kms.Key;
+}
 
 export class ChatBotDynamoDBTables extends Construct {
   public readonly sessionsTable: dynamodb.Table;
   public readonly byUserIdIndex: string = "byUserId";
 
-  constructor(scope: Construct, id: string) {
+  constructor(scope: Construct, id: string, props: ChatBotDynamoDBTablesProps) {
     super(scope, id);
 
     const sessionsTable = new dynamodb.Table(this, "SessionsTable", {
@@ -19,8 +25,14 @@ export class ChatBotDynamoDBTables extends Construct {
         type: dynamodb.AttributeType.STRING,
       },
       billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
-      encryption: dynamodb.TableEncryption.AWS_MANAGED,
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      encryption: props.kmsKey
+        ? dynamodb.TableEncryption.CUSTOMER_MANAGED
+        : dynamodb.TableEncryption.AWS_MANAGED,
+      encryptionKey: props.kmsKey,
+      removalPolicy:
+        props.retainOnDelete === true
+          ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+          : cdk.RemovalPolicy.DESTROY,
       pointInTimeRecovery: true,
     });
 

lib/chatbot-api/chatbot-s3-buckets/index.ts

@@ -1,29 +1,48 @@
 import * as cdk from "aws-cdk-lib";
 import { Construct } from "constructs";
 import * as s3 from "aws-cdk-lib/aws-s3";
+import * as kms from "aws-cdk-lib/aws-kms";
 import { NagSuppressions } from "cdk-nag";
 
+export interface ChatBotS3BucketsProps {
+  readonly retainOnDelete?: boolean;
+  readonly kmsKey?: kms.Key;
+}
+
 export class ChatBotS3Buckets extends Construct {
   public readonly filesBucket: s3.Bucket;
   public readonly userFeedbackBucket: s3.Bucket;
 
-  constructor(scope: Construct, id: string) {
+  constructor(scope: Construct, id: string, props: ChatBotS3BucketsProps) {
     super(scope, id);
 
     const logsBucket = new s3.Bucket(this, "LogsBucket", {
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
-      autoDeleteObjects: true,
+      removalPolicy:
+        props.retainOnDelete === true
+          ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+          : cdk.RemovalPolicy.DESTROY,
+      autoDeleteObjects: props.retainOnDelete !== true,
       enforceSSL: true,
+      encryption: s3.BucketEncryption.S3_MANAGED,
+      versioned: true,
     });
 
     const filesBucket = new s3.Bucket(this, "FilesBucket", {
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
-      autoDeleteObjects: true,
+      removalPolicy:
+        props.retainOnDelete === true
+          ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+          : cdk.RemovalPolicy.DESTROY,
+      autoDeleteObjects: props.retainOnDelete !== true,
       transferAcceleration: true,
       enforceSSL: true,
       serverAccessLogsBucket: logsBucket,
+      encryption: props.kmsKey
+        ? s3.BucketEncryption.KMS
+        : s3.BucketEncryption.S3_MANAGED,
+      encryptionKey: props.kmsKey,
+      versioned: true,
       cors: [
         {
           allowedHeaders: ["*"],
@@ -42,10 +61,18 @@ export class ChatBotS3Buckets extends Construct {
 
     const userFeedbackBucket = new s3.Bucket(this, "UserFeedbackBucket", {
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
-      autoDeleteObjects: true,
+      removalPolicy:
+        props.retainOnDelete === true
+          ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+          : cdk.RemovalPolicy.DESTROY,
+      autoDeleteObjects: props.retainOnDelete !== true,
       enforceSSL: true,
       serverAccessLogsBucket: logsBucket,
+      encryption: props.kmsKey
+        ? s3.BucketEncryption.KMS
+        : s3.BucketEncryption.S3_MANAGED,
+      encryptionKey: props.kmsKey,
+      versioned: true,
     });
 
     this.filesBucket = filesBucket;

lib/chatbot-api/index.ts

@@ -42,8 +42,14 @@ export class ChatBotApi extends Construct {
   constructor(scope: Construct, id: string, props: ChatBotApiProps) {
     super(scope, id);
 
-    const chatTables = new ChatBotDynamoDBTables(this, "ChatDynamoDBTables");
-    const chatBuckets = new ChatBotS3Buckets(this, "ChatBuckets");
+    const chatTables = new ChatBotDynamoDBTables(this, "ChatDynamoDBTables", {
+      kmsKey: props.shared.kmsKey,
+      retainOnDelete: props.config.retainOnDelete,
+    });
+    const chatBuckets = new ChatBotS3Buckets(this, "ChatBuckets", {
+      kmsKey: props.shared.kmsKey,
+      retainOnDelete: props.config.retainOnDelete,
+    });
 
     const loggingRole = new iam.Role(this, "apiLoggingRole", {
       assumedBy: new iam.ServicePrincipal("appsync.amazonaws.com"),

lib/chatbot-api/websocket-api.ts

@@ -35,6 +35,8 @@ export class RealtimeGraphqlApiBackend extends Construct {
     super(scope, id);
     // Create the main Message Topic acting as a message bus
     const messagesTopic = new sns.Topic(this, "MessagesTopic", {
+      enforceSSL: true,
+      masterKey: props.shared.kmsKey,
       tracingConfig: props.advancedMonitoring
         ? sns.TracingConfig.ACTIVE
         : sns.TracingConfig.PASS_THROUGH,
@@ -73,10 +75,18 @@ export class RealtimeGraphqlApiBackend extends Construct {
     }
 
     const deadLetterQueue = new sqs.Queue(this, "OutgoingMessagesDLQ", {
+      encryption: props.shared.queueKmsKey
+        ? sqs.QueueEncryption.KMS
+        : undefined,
+      encryptionMasterKey: props.shared.queueKmsKey,
       enforceSSL: true,
     });
 
     const queue = new sqs.Queue(this, "OutgoingMessagesQueue", {
+      encryption: props.shared.queueKmsKey
+        ? sqs.QueueEncryption.KMS
+        : undefined,
+      encryptionMasterKey: props.shared.queueKmsKey,
       removalPolicy: cdk.RemovalPolicy.DESTROY,
       enforceSSL: true,
       deadLetterQueue: {
@@ -100,6 +110,7 @@ export class RealtimeGraphqlApiBackend extends Construct {
     const resolvers = new RealtimeResolvers(this, "Resolvers", {
       queue: queue,
       topic: messagesTopic,
+      topicKey: props.shared.kmsKey,
       userPool: props.userPool,
       shared: props.shared,
       api: props.api,

lib/model-interfaces/idefics/index.ts

@@ -14,7 +14,6 @@ import { Construct } from "constructs";
 import * as path from "path";
 import { Shared } from "../../shared";
 import { SystemConfig } from "../../shared/types";
-import { RemovalPolicy } from "aws-cdk-lib";
 import { NagSuppressions } from "cdk-nag";
 
 interface IdeficsInterfaceProps {
@@ -80,6 +79,9 @@ export class IdeficsInterface extends Construct {
     props.chatbotFilesBucket.grantRead(requestHandler);
     props.sessionsTable.grantReadWriteData(requestHandler);
     props.messagesTopic.grantPublish(requestHandler);
+    if (props.shared.kmsKey && requestHandler.role) {
+      props.shared.kmsKey.grantEncrypt(requestHandler.role);
+    }
     props.shared.configParameter.grantRead(requestHandler);
     requestHandler.addToRolePolicy(
       new iam.PolicyStatement({
@@ -91,8 +93,17 @@ export class IdeficsInterface extends Construct {
 
     const deadLetterQueue = new sqs.Queue(this, "DLQ", {
       enforceSSL: true,
+      encryption: props.shared.queueKmsKey
+        ? sqs.QueueEncryption.KMS
+        : undefined,
+      encryptionMasterKey: props.shared.queueKmsKey,
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
     });
     const queue = new sqs.Queue(this, "IdeficsIngestionQueue", {
+      encryption: props.shared.queueKmsKey
+        ? sqs.QueueEncryption.KMS
+        : undefined,
+      encryptionMasterKey: props.shared.queueKmsKey,
       removalPolicy: cdk.RemovalPolicy.DESTROY,
       // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-queueconfig
       visibilityTimeout: cdk.Duration.minutes(lambdaDurationInMinutes * 6),
@@ -177,7 +188,11 @@ export class IdeficsInterface extends Construct {
       this,
       "ChatbotFilesPrivateApiAccessLogs",
       {
-        removalPolicy: RemovalPolicy.DESTROY,
+        removalPolicy:
+          this.props.config.retainOnDelete === true
+            ? cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+            : cdk.RemovalPolicy.DESTROY,
+        retention: this.props.config.logRetention,
       }
     );
 
@@ -202,7 +217,8 @@ export class IdeficsInterface extends Construct {
             actions: ["execute-api:Invoke"],
             effect: iam.Effect.ALLOW,
             resources: ["execute-api:/*/*/*"],
-            principals: [new iam.AnyPrincipal()],
+            principals: [new iam.AnyPrincipal()], // NOSONAR
+            // Private integration with deny based on the VPCe
           }),
           new iam.PolicyStatement({
             actions: ["execute-api:Invoke"],
@@ -265,10 +281,11 @@ export class IdeficsInterface extends Construct {
       },
     });
 
+    // prettier-ignore
     api.root
       .addResource("{folder}")
       .addResource("{key}")
-      .addMethod("GET", s3Integration, {
+      .addMethod("GET", s3Integration, { // NOSONAR Private integration
         methodResponses: [
           {
             statusCode: "200",

lib/model-interfaces/langchain/index.ts

@@ -213,6 +213,9 @@ export class LangChainInterface extends Construct {
 
     props.sessionsTable.grantReadWriteData(requestHandler);
     props.messagesTopic.grantPublish(requestHandler);
+    if (props.shared.kmsKey && requestHandler.role) {
+      props.shared.kmsKey.grantEncrypt(requestHandler.role);
+    }
     props.shared.apiKeysSecret.grantRead(requestHandler);
     props.shared.configParameter.grantRead(requestHandler);
 
@@ -235,10 +238,16 @@ export class LangChainInterface extends Construct {
     );
 
     const deadLetterQueue = new sqs.Queue(this, "DLQ", {
+      encryption: props.shared.kmsKey ? sqs.QueueEncryption.KMS : undefined,
+      encryptionMasterKey: props.shared.kmsKey,
       enforceSSL: true,
     });
 
     const queue = new sqs.Queue(this, "LangChainIngestionQueue", {
+      encryption: props.shared.queueKmsKey
+        ? sqs.QueueEncryption.KMS
+        : undefined,
+      encryptionMasterKey: props.shared.queueKmsKey,
       removalPolicy: cdk.RemovalPolicy.DESTROY,
       // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-queueconfig
       visibilityTimeout: cdk.Duration.minutes(15 * 6),