Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ability to specify Aurora DB Name for storing embeddings #405

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7db3e8e
enhance cli to capture existing vpc details
surukonda Mar 7, 2024
5882056
consider question skipping status while validating vpc id
Mar 8, 2024
873becd
Merge branch 'main' into cli-vpc-option
surukonda Mar 8, 2024
3820c39
ability to specify rds default database name
Mar 8, 2024
b3d1ccb
fix indentation issue
Mar 8, 2024
f0b1d6c
delete rds kms key
Mar 8, 2024
e992e6e
Merge branch 'aws-samples:main' into aurora-db-name
surukonda Mar 8, 2024
174be8d
remove unused imports
Mar 8, 2024
f17a9cd
fix to populate db name from existing config json
Mar 8, 2024
0212b79
Merge branch 'main' into aurora-db-name
surukonda Mar 8, 2024
0cd7b53
Merge branch 'main' into aurora-db-name
bigadsoleiman Mar 8, 2024
17e7d62
update initial value for database name
Mar 9, 2024
95a81d6
Merge branch 'aurora-db-name' of https://github.com/surukonda/aws-gen…
Mar 9, 2024
f8c6d85
Merge branch 'main' into aurora-db-name
surukonda Mar 11, 2024
10195a8
Merge branch 'main' into aurora-db-name
surukonda Mar 15, 2024
3346e2e
Merge branch 'main' into aurora-db-name
surukonda Mar 16, 2024
11c2c72
Merge branch 'main' into aurora-db-name
bigadsoleiman Mar 18, 2024
e46e014
Merge branch 'main' into aurora-db-name
surukonda Mar 18, 2024
0c35793
Merge branch 'main' into aurora-db-name
surukonda Mar 19, 2024
946b370
Merge branch 'main' into aurora-db-name
surukonda Apr 11, 2024
d8bd35a
ability to pass the default db name
May 6, 2024
d01b518
Merge branch 'aurora-db-name' of https://github.com/surukonda/aws-gen…
May 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions cli/magic-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ const embeddingModels = [
) {
options.ragsToEnable.pop("kendra");
}
options.auroraDefaultDBName = config.rag.engines.aurora.defaultDatabaseName
options.embeddings = config.rag.embeddingsModels.map((m: any) => m.name);
options.defaultEmbedding = (config.rag.embeddingsModels ?? []).filter(
(m: any) => m.default
Expand Down Expand Up @@ -255,6 +256,20 @@ async function processCreateOptions(options: any): Promise<void> {
},
initial: options.ragsToEnable || [],
},
{
type:"input",
name:"auroraDefaultDBName",
message: "Enter a default database name for Aurora",
initial: options.auroraDefaultDBName,
surukonda marked this conversation as resolved.
Show resolved Hide resolved
validate(v: string) {
return (v.length === 0 || (this as any).skipped ||
RegExp(/^[a-zA-Z][a-zA-Z0-9_]{0,62}$/).test(v)) ?
true : "You need to enter a valid database name";
},
skip(): boolean {
return !(this as any).state.answers.ragsToEnable.includes("aurora");
},
},
{
type: "confirm",
name: "kendraEnterprise",
Expand Down Expand Up @@ -392,6 +407,8 @@ async function processCreateOptions(options: any): Promise<void> {
engines: {
aurora: {
enabled: answers.ragsToEnable.includes("aurora"),
defaultDatabaseName: answers.ragsToEnable.includes("aurora") && answers.auroraDefaultDBName.length > 0 ?
answers.auroraDefaultDBName : undefined
},
opensearch: {
enabled: answers.ragsToEnable.includes("opensearch"),
Expand Down
17 changes: 10 additions & 7 deletions lib/rag-engines/aurora-pgvector/functions/pgvector-setup/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,18 @@ def lambda_handler(event, context: LambdaContext):
SecretId=AURORA_DB_SECRET_ID
)
database_secrets = json.loads(secret_response["SecretString"])
dbhost = database_secrets["host"]
dbport = database_secrets["port"]
dbuser = database_secrets["username"]
dbpass = database_secrets["password"]

if request_type == "Create" or request_type == "Update":
dbconn = psycopg2.connect(
host=dbhost, user=dbuser, password=dbpass, port=dbport, connect_timeout=10
)
db_conn_args = {
"host": database_secrets["host"],
"user": database_secrets["username"],
"password": database_secrets["password"],
"port": database_secrets["port"],
"connect_timeout":10,
}
if "dbname" in database_secrets:
db_conn_args["dbname"] = database_secrets["dbname"]
dbconn = psycopg2.connect(**db_conn_args)

dbconn.set_session(autocommit=True)

Expand Down
1 change: 1 addition & 0 deletions lib/rag-engines/aurora-pgvector/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export class AuroraPgVector extends Construct {
vpc: props.shared.vpc,
vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED },
iamAuthentication: true,
defaultDatabaseName: props.config.rag.engines.aurora.defaultDatabaseName,
});

const databaseSetupFunction = new lambda.Function(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,20 @@ def __init__(self, autocommit=True):
self.dbport = database_secrets["port"]
self.dbuser = database_secrets["username"]
self.dbpass = database_secrets["password"]
if "dbname" in database_secrets:
self.dbname = database_secrets["dbname"]

def __enter__(self):
connection = psycopg2.connect(
host=self.dbhost,
user=self.dbuser,
password=self.dbpass,
port=self.dbport,
connect_timeout=10,
)
db_conn_args = {
"host":self.dbhost,
"user":self.dbuser,
"password":self.dbpass,
"port":self.dbport,
"connect_timeout":10,
}
if hasattr(self, "dbname"):
db_conn_args["dbname"] = self.dbname
connection = psycopg2.connect(**db_conn_args)

connection.set_session(autocommit=self.autocommit)
psycopg2.extras.register_uuid()
Expand Down
1 change: 1 addition & 0 deletions lib/shared/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ export interface SystemConfig {
engines: {
aurora: {
enabled: boolean;
defaultDatabaseName?: string;
};
opensearch: {
enabled: boolean;
Expand Down
Loading