From 00b4f2b1e1364c91e1f2baadc8874fc3da93b10a Mon Sep 17 00:00:00 2001 From: Hasan <45375125+hasanaburayyan@users.noreply.github.com> Date: Thu, 9 May 2024 07:46:34 -0400 Subject: [PATCH] feat(postgres): support AWS RDS (#219) ## Summary - Supports creating RDS postgres databases - Support referencing existing databases using `DatabaseRef` class ## TODO: - [x] cleanup the required parameters ## Blocked by - https://github.com/winglang/wing/pull/6377 - https://github.com/winglang/wing/pull/6345 --- postgres/README.md | 36 ++++++- postgres/lib.w | 233 +++++++++++++++++++++++++++++++++++++++++- postgres/package.json | 2 +- postgres/pg.js | 21 ++++ 4 files changed, 285 insertions(+), 7 deletions(-) diff --git a/postgres/README.md b/postgres/README.md index b81719ef..65c67cf1 100644 --- a/postgres/README.md +++ b/postgres/README.md @@ -67,9 +67,43 @@ new sim.Container( ) ``` +### Reference Existing Postgres Database +If you want to import a reference to an existing postgres database, you can use the `DatabaseRef` class: + +```js +bring postgres; + +let db = new postgres.DatabaseRef() as "somedatabase"; + + +new cloud.Function(inflight() => { + let users = db.query("select * from users"); +}); +``` +This will automatically create a secret resource that is required for the database connection. To seed this secret, use the `secrets` subcommand: + +```sh +❯ wing secrets main.w +1 secret(s) found + +? Enter the secret value for connectionString_somedatabase: [input is hidden] +``` + +> When referencing an existing database for the `tf-aws` target you will also need to specify VPC information in your `wing.toml` file (unless your database is publicly accessible). Or you will see an warning like this: +```sh +WARNING: Unless your database is accessible from the public internet, you must provide vpc info under `tf-aws` in your wing.toml file +For more info see: https://www.winglang.io/docs/platforms/tf-aws#parameters +``` + ## `tf-aws` -On the `tf-aws` target, the postgres database is managed using [Neon](https://neon.tech/), a serverless postgres offering. +On the `tf-aws` target, the postgres database can be created and hosted by either AWS RDS or Neon. To configure which one to use, simply specify the parameter `postgresEngine` in your `wing.toml` file: + +```toml +postgresEngine = "rds" # or "neon" +``` + +### Neon Setup Neon has a [free tier](https://neon.tech/docs/introduction/free-tier) that can be used for personal projects and prototyping. diff --git a/postgres/lib.w b/postgres/lib.w index 1f13088b..669f0e5c 100644 --- a/postgres/lib.w +++ b/postgres/lib.w @@ -9,6 +9,10 @@ bring "cdktf" as cdktf; bring "@rybickic/cdktf-provider-neon" as rawNeon; bring "@cdktf/provider-aws" as tfaws; +pub struct AwsParameters { + postgresEngine: str?; +} + pub struct ConnectionOptions { host: str; port: str; @@ -32,22 +36,80 @@ pub struct DatabaseProps { pub interface IDatabase { inflight query(query: str): Array>; - inflight connectionOptions(): ConnectionOptions; + inflight connectionOptions(): ConnectionOptions?; +} + +struct RequiredTFAwsProps { + /** + * Supported: [neon, rds] + */ + postgresEngine: str; +} + +struct RequiredRDSParameters { + instanceCount: num; + /** + * See: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Concepts.DBInstanceClass.html + */ + instanceClass: str; + /** Deault postgres */ + masterUsername: str?; +} + +struct AWSDatabaseRefParameters { + vpcId: str; +} + +pub class DatabaseRef { + connectionSecret: cloud.Secret; + + new() { + // For database refs, we just need to have a secret connection string to use for queries + this.connectionSecret = new cloud.Secret(name: "connectionString_{this.node.id}"); + + let target = util.env("WING_TARGET"); + if target == "tf-aws" { + let params = nodeof(this).app.parameters; + + let allParams = params.read(); + if !allParams.has("tf-aws") { + // TODO: find a good way to enforce this with parameters. (currently cant duplicate required params schemas) + log("WARNING: Unless your database is accessible from the public internet, you must provide vpc info under `tf-aws` in your wing.toml file\nFor more info see: https://www.winglang.io/docs/platforms/tf-aws#parameters"); + } + } + } + + pub inflight query(query: str): Array> { + return PgUtil._queryWithConnectionString(query, this.connectionSecret.value()); + } } pub class Database { pub connection: ConnectionOptions; inner: IDatabase; new(props: DatabaseProps) { + let app = nodeof(this).app; + let params = app.parameters.read(schema: AwsParameters.schema()); + let target = util.env("WING_TARGET"); if target == "sim" { let sim = new DatabaseSim(props); this.connection = sim.connection; this.inner = sim; } elif target == "tf-aws" { - let neon = new DatabaseNeon(props); - this.connection = neon.connection; - this.inner = neon; + let tfawsParams = RequiredTFAwsProps.fromJson(app.parameters.read(schema: RequiredTFAwsProps.schema())); + if tfawsParams.postgresEngine == "rds" { + let aurora = new DatabaseRDS(props); + this.connection = aurora.connection; + this.inner = aurora; + + } elif tfawsParams.postgresEngine == "neon" { + let neon = new DatabaseNeon(props); + this.connection = neon.connection; + this.inner = neon; + } else { + throw "Unsupported postgres engine for tf-aws: " + tfawsParams.postgresEngine; + } } else { throw "Unsupported target: " + target; } @@ -57,7 +119,167 @@ pub class Database { return this.inner.query(query); } pub inflight connectionOptions(): ConnectionOptions { - return this.inner.connectionOptions(); + return this.inner.connectionOptions()!; + } +} + +struct TfawsApp { + vpc: tfaws.vpc.Vpc; + subnets: TfawsAppSubnets; +} + +struct TfawsAppSubnets { + private: Array; + public: Array; +} + +class DatabaseRDS impl IDatabase { + var secretRef: aws.SecretRef; + var endpoint: str; + params: RequiredRDSParameters; + databaseName: str; + pub connection: ConnectionOptions; + + new (props: DatabaseProps) { + this.databaseName = props.name; + + let app = nodeof(this).app; + this.params = RequiredRDSParameters.fromJson(app.parameters.read(schema: RequiredRDSParameters.schema())); + let tfawsApp: TfawsApp = unsafeCast(nodeof(this).app); + let vpc = tfawsApp.vpc; + let subnetIds: MutArray = MutArray[]; + + let sg = new tfaws.securityGroup.SecurityGroup( + vpcId: vpc.id, + egress: [ + { + fromPort: 0, + toPort: 0, + protocol: "-1", + cidrBlocks: ["0.0.0.0/0"] + } + ], + ingress: [ + { + fromPort: "5432", + toPort: "5432", + protocol: "tcp", + cidrBlocks: ["0.0.0.0/0"] + } + ] + ); + + for net in tfawsApp.subnets.private { + subnetIds.push(net.id); + } + + let subnetGroup = new tfaws.dbSubnetGroup.DbSubnetGroup( + subnetIds: subnetIds.copy() + ); + + let cluster = new tfaws.rdsCluster.RdsCluster( + engine: "aurora-postgresql", + databaseName: this.databaseName, + masterUsername: this.params.masterUsername ?? "postgres", + manageMasterUserPassword: true, + dbSubnetGroupName: subnetGroup.name, + vpcSecurityGroupIds: [sg.id] + ); + + for i in 0..this.params.instanceCount { + new tfaws.rdsClusterInstance.RdsClusterInstance( + clusterIdentifier: cluster.id, + instanceClass: this.params.instanceClass, + engine: cluster.engine, + dbSubnetGroupName: subnetGroup.name + ) as "rdsInstance{i}"; + } + + let role = new tfaws.iamRole.IamRole( + assumeRolePolicy: Json.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { + Service: "rds.amazonaws.com" + }, + Action: "sts:AssumeRole" + } + ] + }) + ); + + let policy = new tfaws.iamPolicy.IamPolicy( + policy: Json.stringify({ + Version: "2012-10-17", + Statement: [ + { + Action: [ + "secretsmanager:GetSecretValue" + ], + Effect: "Allow", + Resource: cluster.masterUserSecret.get(0).secretArn + } + ] + }) + ); + + new tfaws.iamRolePolicyAttachment.IamRolePolicyAttachment( + policyArn: policy.arn, + role: role.name + ); + + let proxy = new tfaws.dbProxy.DbProxy( + name: "proxy{this.node.addr.substring(0, -8)}", + engineFamily: "POSTGRESQL", + auth: { + secret_arn: cluster.masterUserSecret.get(0).secretArn + }, + vpcSubnetIds: subnetIds.copy(), + roleArn: role.arn, + vpcSecurityGroupIds: [ sg.id ] + ); + + let targetGroup = new tfaws.dbProxyDefaultTargetGroup.DbProxyDefaultTargetGroup( + dbProxyName: proxy.name, + ); + + new tfaws.dbProxyTarget.DbProxyTarget( + dbProxyName: proxy.name, + dbClusterIdentifier: cluster.id, + targetGroupName: targetGroup.name + ); + + + this.endpoint = proxy.endpoint; + this.secretRef = new aws.SecretRef(cluster.masterUserSecret.get(0).secretArn); + new cdktf.TerraformOutput(value: cluster.masterUserSecret); + + this.connection = { + host: proxy.endpoint, + user: this.params.masterUsername ?? "postgres", + password: cluster.masterPassword, + database: this.databaseName, + ssl: true, + port: "5432" + }; + } + + pub inflight query(query: str): Array> { + return PgUtil._query(query, this.connectionOptions()); + } + + pub inflight connectionOptions(): ConnectionOptions { + let databaseSecrets = Json.parse(this.secretRef.value()); + return { + host: this.endpoint, + user: databaseSecrets.get("username").asStr(), + password: databaseSecrets.get("password").asStr(), + database: this.databaseName, + ssl: true, + port: "5432" + }; } } @@ -173,5 +395,6 @@ class DatabaseNeon impl IDatabase { class PgUtil { pub extern "./pg.js" static inflight _query(query: str, creds: ConnectionOptions): Array>; + pub extern "./pg.js" static inflight _queryWithConnectionString(query: str, connectionString: str): Array>; pub extern "./util.ts" static inflight isPortOpen(port: str): bool; } diff --git a/postgres/package.json b/postgres/package.json index a0833994..9b965722 100644 --- a/postgres/package.json +++ b/postgres/package.json @@ -1,6 +1,6 @@ { "name": "@winglibs/postgres", - "version": "0.1.3", + "version": "0.1.4", "description": "Postgres support for Wing", "repository": { "type": "git", diff --git a/postgres/pg.js b/postgres/pg.js index 5725feaa..f442ec73 100644 --- a/postgres/pg.js +++ b/postgres/pg.js @@ -6,6 +6,27 @@ exports._query = async function(query, opts) { await client.end(); return res.rows; } +exports._queryWithConnectionString = async function(query, connectionString) { + const client = await connectWithRetry({ connectionString: encodePasswordInConnectionString(connectionString) }); + const res = await client.query(query); + await client.end(); + return res.rows; +} + +function encodePasswordInConnectionString(connectionString) { + const regex = /^(postgresql:\/\/)([^:]+):([^@]+)@([^:\/]+):(\d+)\/(.+)$/; + const match = connectionString.match(regex); + + if (!match) { + throw new Error('Invalid connection string format'); + } + + const [, protocol, username, password, host, port, database] = match; + + const encodedPassword = encodeURIComponent(password); + + return `${protocol}${username}:${encodedPassword}@${host}:${port}/${database}`; +} // workaround for readiness probe async function connectWithRetry(opts, maxRetries = 5, interval = 1000) {