From 7db3e8ebc907c3c60843841d265c9399a56f5890 Mon Sep 17 00:00:00 2001 From: santhosh Date: Thu, 7 Mar 2024 22:46:14 +0530 Subject: [PATCH 1/9] enhance cli to capture existing vpc details --- cli/magic-config.ts | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cli/magic-config.ts b/cli/magic-config.ts index cadcb091c..860bf9105 100644 --- a/cli/magic-config.ts +++ b/cli/magic-config.ts @@ -58,6 +58,9 @@ const embeddingModels = [ fs.readFileSync("./bin/config.json").toString("utf8") ); options.prefix = config.prefix; + options.vpc = config.vpc ? true : false; + options.vpcId = config.vpc?.vpcId; + options.createVpcEndpoints = config.vpc?.createVpcEndpoints; options.privateWebsite = config.privateWebsite; options.certificate = config.certificate; options.domain = config.domain; @@ -117,6 +120,34 @@ async function processCreateOptions(options: any): Promise { initial: options.prefix, askAnswered: false, }, + { + type: "confirm", + name: "existingVpc", + message: "Do you want to use existing vpc? (selecting false will create a new vpc)", + initial: options.vpc || false, + }, + { + type: "input", + name: "vpcId", + message: "Specify existing VpcId (vpc-xxxxxxxxxxxxxxxxx)", + initial: options.vpcId, + validate: (vpcId: string) =>{ + return RegExp(/^vpc-[0-9a-f]{8,17}$/i).test(vpcId) ? true + : "Enter a valid VpcId in vpc-0123456abdef format" + }, + skip(): boolean { + return !(this as any).state.answers.existingVpc; + }, + }, + { + type: "confirm", + name: "createVpcEndpoints", + message: "Do you want create VPC Endpoints?", + initial: options.createVpcEndpoints || false, + skip(): boolean { + return !(this as any).state.answers.existingVpc; + }, + }, { type: "confirm", name: "privateWebsite", @@ -337,6 +368,12 @@ async function processCreateOptions(options: any): Promise { // Create the config object const config = { prefix: answers.prefix, + vpc: answers.existingVpc + ? { + vpcId: answers.vpcId.toLowerCase(), + createVpcEndpoints: answers.createVpcEndpoints, + } + : undefined, privateWebsite: answers.privateWebsite, certificate: answers.certificate, domain: answers.domain, From 588205634fce1a2f93cc3c26f166006a73261553 Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 02:49:37 +0000 Subject: [PATCH 2/9] consider question skipping status while validating vpc id --- cli/magic-config.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cli/magic-config.ts b/cli/magic-config.ts index 860bf9105..bb816e677 100644 --- a/cli/magic-config.ts +++ b/cli/magic-config.ts @@ -58,7 +58,6 @@ const embeddingModels = [ fs.readFileSync("./bin/config.json").toString("utf8") ); options.prefix = config.prefix; - options.vpc = config.vpc ? true : false; options.vpcId = config.vpc?.vpcId; options.createVpcEndpoints = config.vpc?.createVpcEndpoints; options.privateWebsite = config.privateWebsite; @@ -124,16 +123,16 @@ async function processCreateOptions(options: any): Promise { type: "confirm", name: "existingVpc", message: "Do you want to use existing vpc? (selecting false will create a new vpc)", - initial: options.vpc || false, + initial: options.vpcId ? true : false, }, { type: "input", name: "vpcId", message: "Specify existing VpcId (vpc-xxxxxxxxxxxxxxxxx)", initial: options.vpcId, - validate: (vpcId: string) =>{ - return RegExp(/^vpc-[0-9a-f]{8,17}$/i).test(vpcId) ? true - : "Enter a valid VpcId in vpc-0123456abdef format" + validate(vpcId: string) { + return ((this as any).skipped || RegExp(/^vpc-[0-9a-f]{8,17}$/i).test(vpcId)) ? + true : 'Enter a valid VpcId in vpc-xxxxxxxxxxx format' }, skip(): boolean { return !(this as any).state.answers.existingVpc; From 3820c39f27bc496833ada01af003c1d3f1c11a0c Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 11:38:47 +0000 Subject: [PATCH 3/9] ability to specify rds default database name --- cli/magic-config.ts | 17 +++++++++++++++++ .../functions/pgvector-setup/index.py | 17 ++++++++++------- lib/rag-engines/aurora-pgvector/index.ts | 8 ++++++++ .../python/genai_core/aurora/connection.py | 19 ++++++++++++------- lib/shared/types.ts | 1 + 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/cli/magic-config.ts b/cli/magic-config.ts index bb816e677..dee32e27f 100644 --- a/cli/magic-config.ts +++ b/cli/magic-config.ts @@ -80,6 +80,7 @@ const embeddingModels = [ ) { options.ragsToEnable.pop("kendra"); } + options.auroraDefaultDBName = config.rag.engines?.aurora?.defaultDatabaseName options.embeddings = config.rag.embeddingsModels.map((m: any) => m.name); options.defaultEmbedding = (config.rag.embeddingsModels ?? []).filter( (m: any) => m.default @@ -255,6 +256,20 @@ async function processCreateOptions(options: any): Promise { }, initial: options.ragsToEnable || [], }, + { + type:"input", + name:"auroraDefaultDBName", + message: "Enter a default database name for Aurora", + initial: options.auroraDefaultDBName, + validate(v: string) { + return (v.length === 0 || (this as any).skipped || + RegExp(/^[a-zA-Z][a-zA-Z0-9_]{0,62}$/).test(v)) ? + true : "You need to enter a valid database name"; + }, + skip(): boolean { + return !(this as any).state.answers.ragsToEnable.includes("aurora"); + }, + }, { type: "confirm", name: "kendraEnterprise", @@ -392,6 +407,8 @@ async function processCreateOptions(options: any): Promise { engines: { aurora: { enabled: answers.ragsToEnable.includes("aurora"), + defaultDatabaseName: answers.ragsToEnable.includes("aurora") && answers.auroraDefaultDBName.length > 0 ? + answers.auroraDefaultDBName : undefined }, opensearch: { enabled: answers.ragsToEnable.includes("opensearch"), diff --git a/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py b/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py index 146435216..41abdb60a 100644 --- a/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py +++ b/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py @@ -20,15 +20,18 @@ def lambda_handler(event, context: LambdaContext): SecretId=AURORA_DB_SECRET_ID ) database_secrets = json.loads(secret_response["SecretString"]) - dbhost = database_secrets["host"] - dbport = database_secrets["port"] - dbuser = database_secrets["username"] - dbpass = database_secrets["password"] if request_type == "Create" or request_type == "Update": - dbconn = psycopg2.connect( - host=dbhost, user=dbuser, password=dbpass, port=dbport, connect_timeout=10 - ) + db_conn_args = { + "host": database_secrets["host"], + "user": database_secrets["username"], + "password": database_secrets["password"], + "port": database_secrets["port"], + "connect_timeout":10, + } + if "dbname" in database_secrets: + db_conn_args["dbname"] = database_secrets["dbname"] + dbconn = psycopg2.connect(**db_conn_args) dbconn.set_session(autocommit=True) diff --git a/lib/rag-engines/aurora-pgvector/index.ts b/lib/rag-engines/aurora-pgvector/index.ts index 043c4d963..7a628c7a8 100644 --- a/lib/rag-engines/aurora-pgvector/index.ts +++ b/lib/rag-engines/aurora-pgvector/index.ts @@ -9,6 +9,7 @@ import * as ec2 from "aws-cdk-lib/aws-ec2"; import * as lambda from "aws-cdk-lib/aws-lambda"; import * as logs from "aws-cdk-lib/aws-logs"; import * as rds from "aws-cdk-lib/aws-rds"; +import * as kms from "aws-cdk-lib/aws-kms"; import * as cr from "aws-cdk-lib/custom-resources"; import * as sfn from "aws-cdk-lib/aws-stepfunctions"; import { NagSuppressions } from "cdk-nag"; @@ -26,6 +27,11 @@ export class AuroraPgVector extends Construct { constructor(scope: Construct, id: string, props: AuroraPgVectorProps) { super(scope, id); + const rdsKey = new kms.Key(this, "RDSKey", { + enableKeyRotation: true, + description: "Key for RDS", + }); + const dbCluster = new rds.DatabaseCluster(this, "AuroraDatabase", { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: rds.AuroraPostgresEngineVersion.VER_15_3, @@ -35,6 +41,8 @@ export class AuroraPgVector extends Construct { vpc: props.shared.vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED }, iamAuthentication: true, + defaultDatabaseName: props.config.rag.engines.aurora.defaultDatabaseName, + storageEncryptionKey: rdsKey, }); const databaseSetupFunction = new lambda.Function( diff --git a/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py b/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py index b303fad7c..243b299ab 100644 --- a/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py +++ b/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py @@ -21,15 +21,20 @@ def __init__(self, autocommit=True): self.dbport = database_secrets["port"] self.dbuser = database_secrets["username"] self.dbpass = database_secrets["password"] + if "dbname" in database_secrets: + self.dbname = database_secrets["dbname"] def __enter__(self): - connection = psycopg2.connect( - host=self.dbhost, - user=self.dbuser, - password=self.dbpass, - port=self.dbport, - connect_timeout=10, - ) + db_conn_args = { + "host":self.dbhost, + "user":self.dbuser, + "password":self.dbpass, + "port":self.dbport, + "connect_timeout":10, + } + if hasattr(self, "dbname"): + db_conn_args["dbname"] = self.dbname + connection = psycopg2.connect(**db_conn_args) connection.set_session(autocommit=self.autocommit) psycopg2.extras.register_uuid() diff --git a/lib/shared/types.ts b/lib/shared/types.ts index 63046009c..ccf9f42d9 100644 --- a/lib/shared/types.ts +++ b/lib/shared/types.ts @@ -90,6 +90,7 @@ export interface SystemConfig { engines: { aurora: { enabled: boolean; + defaultDatabaseName?: string; }; opensearch: { enabled: boolean; From b3d1ccbad54516233b89c93eafa5bac375d8237d Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 11:48:17 +0000 Subject: [PATCH 4/9] fix indentation issue --- .../layers/python-sdk/python/genai_core/aurora/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py b/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py index 243b299ab..f81b167b7 100644 --- a/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py +++ b/lib/shared/layers/python-sdk/python/genai_core/aurora/connection.py @@ -26,7 +26,7 @@ def __init__(self, autocommit=True): def __enter__(self): db_conn_args = { - "host":self.dbhost, + "host":self.dbhost, "user":self.dbuser, "password":self.dbpass, "port":self.dbport, From f0b1d6cb32f222c72f30a950ad848a05e9418935 Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 11:50:15 +0000 Subject: [PATCH 5/9] delete rds kms key --- lib/rag-engines/aurora-pgvector/index.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/rag-engines/aurora-pgvector/index.ts b/lib/rag-engines/aurora-pgvector/index.ts index 7a628c7a8..eac2ea0a1 100644 --- a/lib/rag-engines/aurora-pgvector/index.ts +++ b/lib/rag-engines/aurora-pgvector/index.ts @@ -27,11 +27,6 @@ export class AuroraPgVector extends Construct { constructor(scope: Construct, id: string, props: AuroraPgVectorProps) { super(scope, id); - const rdsKey = new kms.Key(this, "RDSKey", { - enableKeyRotation: true, - description: "Key for RDS", - }); - const dbCluster = new rds.DatabaseCluster(this, "AuroraDatabase", { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: rds.AuroraPostgresEngineVersion.VER_15_3, @@ -42,7 +37,6 @@ export class AuroraPgVector extends Construct { vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED }, iamAuthentication: true, defaultDatabaseName: props.config.rag.engines.aurora.defaultDatabaseName, - storageEncryptionKey: rdsKey, }); const databaseSetupFunction = new lambda.Function( From 174be8de66116fe624a1f050d0af10f26687ccec Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 12:09:49 +0000 Subject: [PATCH 6/9] remove unused imports --- lib/rag-engines/aurora-pgvector/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rag-engines/aurora-pgvector/index.ts b/lib/rag-engines/aurora-pgvector/index.ts index eac2ea0a1..3c9f51094 100644 --- a/lib/rag-engines/aurora-pgvector/index.ts +++ b/lib/rag-engines/aurora-pgvector/index.ts @@ -9,7 +9,6 @@ import * as ec2 from "aws-cdk-lib/aws-ec2"; import * as lambda from "aws-cdk-lib/aws-lambda"; import * as logs from "aws-cdk-lib/aws-logs"; import * as rds from "aws-cdk-lib/aws-rds"; -import * as kms from "aws-cdk-lib/aws-kms"; import * as cr from "aws-cdk-lib/custom-resources"; import * as sfn from "aws-cdk-lib/aws-stepfunctions"; import { NagSuppressions } from "cdk-nag"; From f17a9cd0e88ae34dea0b4f510311e2469186eb45 Mon Sep 17 00:00:00 2001 From: santhosh Date: Fri, 8 Mar 2024 12:15:18 +0000 Subject: [PATCH 7/9] fix to populate db name from existing config json --- cli/magic-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/magic-config.ts b/cli/magic-config.ts index dee32e27f..a97dc1e92 100644 --- a/cli/magic-config.ts +++ b/cli/magic-config.ts @@ -80,7 +80,7 @@ const embeddingModels = [ ) { options.ragsToEnable.pop("kendra"); } - options.auroraDefaultDBName = config.rag.engines?.aurora?.defaultDatabaseName + options.auroraDefaultDBName = config.rag.engines.aurora.defaultDatabaseName options.embeddings = config.rag.embeddingsModels.map((m: any) => m.name); options.defaultEmbedding = (config.rag.embeddingsModels ?? []).filter( (m: any) => m.default From 17e7d62cf86b7c3966cb03b9dd4f6456f90c104d Mon Sep 17 00:00:00 2001 From: santhosh Date: Sat, 9 Mar 2024 17:24:02 +0000 Subject: [PATCH 8/9] update initial value for database name --- cli/magic-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/magic-config.ts b/cli/magic-config.ts index a97dc1e92..c5fe5cb3e 100644 --- a/cli/magic-config.ts +++ b/cli/magic-config.ts @@ -260,7 +260,7 @@ async function processCreateOptions(options: any): Promise { type:"input", name:"auroraDefaultDBName", message: "Enter a default database name for Aurora", - initial: options.auroraDefaultDBName, + initial: options.auroraDefaultDBName || undefined, validate(v: string) { return (v.length === 0 || (this as any).skipped || RegExp(/^[a-zA-Z][a-zA-Z0-9_]{0,62}$/).test(v)) ? From d8bd35acc1143af35673d36ae17e4cc79d3bbca1 Mon Sep 17 00:00:00 2001 From: santhosh Date: Mon, 6 May 2024 09:52:25 +0000 Subject: [PATCH 9/9] ability to pass the default db name --- .../functions/pgvector-setup/index.py | 20 +++++++++++++++++-- lib/rag-engines/aurora-pgvector/index.ts | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py b/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py index 41abdb60a..aeea4e25e 100644 --- a/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py +++ b/lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py @@ -15,6 +15,7 @@ def lambda_handler(event, context: LambdaContext): request_type = event["RequestType"] resource_properties = event["ResourceProperties"] AURORA_DB_SECRET_ID = resource_properties["AURORA_DB_SECRET_ID"] + AURORA_DB_NAME= resource_properties["AURORA_DB_NAME"] secret_response = secretsmanager_client.get_secret_value( SecretId=AURORA_DB_SECRET_ID @@ -32,11 +33,26 @@ def lambda_handler(event, context: LambdaContext): if "dbname" in database_secrets: db_conn_args["dbname"] = database_secrets["dbname"] dbconn = psycopg2.connect(**db_conn_args) - dbconn.set_session(autocommit=True) - cur = dbconn.cursor() + # Check if database exists, if not create it. + cur.execute(f"SELECT * FROM pg_database WHERE LOWER(datname) = LOWER('{AURORA_DB_NAME}')") + rows = cur.fetchall() + if len(rows) == 0: + cur.execute(f"CREATE DATABASE {AURORA_DB_NAME}") + logger.info(f"Created database {AURORA_DB_NAME}") + # close existing connection and re-open connect to the new database + cur.close() + db_conn_args["dbname"] = AURORA_DB_NAME + dbconn = psycopg2.connect(**db_conn_args) + dbconn.set_session(autocommit=True) + cur = dbconn.cursor() + else: + logger.info(f"Database {AURORA_DB_NAME} already exists") + cur.close() + + # Create vector extension if not exists. This will create the vector type. cur.execute("CREATE EXTENSION IF NOT EXISTS vector;") register_vector(dbconn) diff --git a/lib/rag-engines/aurora-pgvector/index.ts b/lib/rag-engines/aurora-pgvector/index.ts index 3c9f51094..7e89c0699 100644 --- a/lib/rag-engines/aurora-pgvector/index.ts +++ b/lib/rag-engines/aurora-pgvector/index.ts @@ -35,7 +35,6 @@ export class AuroraPgVector extends Construct { vpc: props.shared.vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED }, iamAuthentication: true, - defaultDatabaseName: props.config.rag.engines.aurora.defaultDatabaseName, }); const databaseSetupFunction = new lambda.Function( @@ -78,6 +77,7 @@ export class AuroraPgVector extends Construct { serviceToken: databaseSetupProvider.serviceToken, properties: { AURORA_DB_SECRET_ID: dbCluster.secret?.secretArn as string, + AURORA_DB_NAME: props.config.rag.engines.aurora.defaultDatabaseName, }, } );