Support setting ssl client certificate information via environment va…

…riables (#574)

### What

We'd like to support setting ssl certificate information via environment
variables.

### How

1. We create a new function, `get_connect_options`, which will read both
the uri and the ssl information, and use it and `connect_with`
everywhere instead of `connect` with just the `uri`. We make sure all
operations using configuration version 5 including the connector and the
cli use `get_connect_options` (except tests).
2. We read the `client_cert`, `client_key` and `root_cert` from the
environment and put them directly into the sqlx connection options.

#### How we tested this

We used [this
article](https://dev.to/danvixent/how-to-setup-postgresql-with-ssl-inside-a-docker-container-5f3)
as a guide on how to set up postgres+certs with docker.

After running all of the commands, we had to do the following as well:

```sh
$ certstrap request-cert --common-name postgresdb  --domain localhost
$ cp certs/out/myCA.crt out/
$ cp certs/out/myCA.key out/
$ certstrap sign postgresdb --CA myCA
```

Then, we added the following environment variables:

```sh
$ export CLIENT_CERT="$(cat /tmp/ssl/out/postgresdb.crt)"
$ export CLIENT_KEY="$(cat /tmp/ssl/out/postgresdb.key)"
$ export ROOT_CERT="$(cat /tmp/ssl/certs/out/myCA.crt)"
```

Initialized and updated the connector:

```sh
$ mdkir /tmp/ssltest
$ CONNECTION_URI="postgresql://postgres:postgres@localhost:64009/postgres?sslmode=verify-ca" target/debug/ndc-postgres-cli --context /tmp/ssltest initialize
$ CONNECTION_URI="postgresql://postgres:postgres@localhost:64009/postgres?sslmode=verify-ca" target/debug/ndc-postgres-cli --context /tmp/ssltest update
```

Added a native query:

```sh
$ echo "select 'gil' as "name", 35 as 'age'" > /tmp/ssltest/a.sql

$ CONNECTION_URI="postgresql://postgres:postgres@localhost:64009/postgres?sslmode=verify-ca" target/debug/ndc-postgres-cli --context /tmp/ssltest native-operation create --kind query --operation-path a.sql
```

Started the connector:

```sh
CONNECTION_URI="postgresql://postgres:postgres@localhost:64009/postgres?sslmode=verify-ca" target/debug/ndc-postgres serve --configuration /tmp/ssltest
```

And ran a query:

```sh
curl -X POST \
    -H 'Host: example.hasura.app' \
    -H 'Content-Type: application/json' \
    -H 'x-hasura-role: admin' \
    http://localhost:8080/query \
    -d '{ "collection": "a", "query": { "fields": { "name": { "type": "column", "column": "name" } } }, "arguments": {}, "collection_relationships": {} }' | jq
```

changelog.md

@@ -4,12 +4,15 @@
 
 ### Added
 
+- Support setting ssl client certificate information via environment variables.
+  [#574](https://github.com/hasura/ndc-postgres/pull/574)
+
 ### Changed
 
 ### Fixed
 
 - Make array element types nullable in the schema.
-  [#565](https://github.com/hasura/ndc-postgres/pull/571)
+  [#571](https://github.com/hasura/ndc-postgres/pull/571)
 
 ## [v1.0.2]
 

crates/cli/src/lib.rs

@@ -98,11 +98,28 @@ async fn initialize(with_metadata: bool, context: Context<impl Environment>) ->
                     ),
                 },
             ),
-            supported_environment_variables: vec![metadata::EnvironmentVariableDefinition {
-                name: "CONNECTION_URI".to_string(),
-                description: "The PostgreSQL connection URI".to_string(),
-                default_value: Some("postgresql://read_only_user:[email protected]:5432/v3-docs-sample-app".to_string()),
-            }],
+            supported_environment_variables: vec![
+				metadata::EnvironmentVariableDefinition {
+					name: "CONNECTION_URI".to_string(),
+					description: "The PostgreSQL connection URI".to_string(),
+					default_value: Some("postgresql://read_only_user:[email protected]:5432/v3-docs-sample-app".to_string()),
+				},
+				metadata::EnvironmentVariableDefinition {
+					name: "CLIENT_CERT".to_string(),
+					description: "The SSL client certificate".to_string(),
+					default_value: None,
+				},
+				metadata::EnvironmentVariableDefinition {
+					name: "CLIENT_KEY".to_string(),
+					description: "The SSL client key".to_string(),
+					default_value: None,
+				},
+				metadata::EnvironmentVariableDefinition {
+					name: "ROOT_CERT".to_string(),
+					description: "The SSL root certificate".to_string(),
+					default_value: None,
+				},
+			],
             commands: metadata::Commands {
                 update: Some("hasura-ndc-postgres update".to_string()),
                 watch: None,

crates/cli/src/native_operations.rs

@@ -190,6 +190,7 @@ async fn create(
 
             let new_native_operation = configuration::version5::native_operations::create(
                 configuration,
+                &context.environment,
                 &connection_string,
                 &operation_path,
                 &file_contents,

crates/cli/tests/snapshots/initialize_tests__initialize_directory_with_metadata.snap

@@ -9,6 +9,12 @@ supportedEnvironmentVariables:
 - name: CONNECTION_URI
   description: The PostgreSQL connection URI
   defaultValue: postgresql://read_only_user:[email protected]:5432/v3-docs-sample-app
+- name: CLIENT_CERT
+  description: The SSL client certificate
+- name: CLIENT_KEY
+  description: The SSL client key
+- name: ROOT_CERT
+  description: The SSL root certificate
 commands:
   update: hasura-ndc-postgres update
 cliPlugin:

crates/cli/tests/snapshots/initialize_tests__initialize_directory_with_metadata_and_release_version.snap

@@ -9,6 +9,12 @@ supportedEnvironmentVariables:
 - name: CONNECTION_URI
   description: The PostgreSQL connection URI
   defaultValue: postgresql://read_only_user:[email protected]:5432/v3-docs-sample-app
+- name: CLIENT_CERT
+  description: The SSL client certificate
+- name: CLIENT_KEY
+  description: The SSL client key
+- name: ROOT_CERT
+  description: The SSL root certificate
 commands:
   update: hasura-ndc-postgres update
 cliPlugin:

crates/configuration/src/connect.rs

@@ -0,0 +1,66 @@
+//! Connection settings.
+
+use std::borrow::Cow;
+
+use sqlx::postgres::PgConnectOptions;
+use sqlx::ConnectOptions;
+
+use crate::environment::{Environment, Variable};
+use crate::values::{ConnectionUri, Secret};
+
+/// Get the connect options from the connection string and environment.
+pub fn get_connect_options(
+    connection_uri: &ConnectionUri,
+    environment: impl Environment,
+) -> anyhow::Result<PgConnectOptions> {
+    let uri = match &connection_uri {
+        ConnectionUri(Secret::Plain(value)) => Cow::Borrowed(value),
+        ConnectionUri(Secret::FromEnvironment { variable }) => {
+            Cow::Owned(environment.read(variable)?)
+        }
+    };
+
+    let connect_options = PgConnectOptions::from_url(&uri.parse()?)?;
+
+    let ssl = read_ssl_info(environment);
+
+    // Add ssl info if present.
+    Ok(match ssl {
+        None => connect_options,
+        Some(secret) => connect_options
+            .ssl_client_cert_from_pem(secret.certificate)
+            .ssl_client_key_from_pem(secret.key)
+            .ssl_root_cert_from_pem(secret.root_certificate),
+    })
+}
+
+/// SSL client certificate information.
+struct SslClientInfo {
+    certificate: String,
+    key: String,
+    root_certificate: Vec<u8>,
+}
+
+/// Read ssl certificate and key from the environment.
+fn read_ssl_info(environment: impl Environment) -> Option<SslClientInfo> {
+    // read ssl info
+    let certificate = environment.read(&Variable::from("CLIENT_CERT")).ok();
+    let key = environment.read(&Variable::from("CLIENT_KEY")).ok();
+    let root_certificate = environment
+        .read(&Variable::from("ROOT_CERT"))
+        .ok()
+        .map(|text| text.as_bytes().to_vec());
+
+    match (certificate, key, root_certificate) {
+        (Some(certificate), Some(key), Some(root_certificate))
+            if !certificate.is_empty() && !key.is_empty() && !root_certificate.is_empty() =>
+        {
+            Some(SslClientInfo {
+                certificate,
+                key,
+                root_certificate,
+            })
+        }
+        _ => None,
+    }
+}

crates/configuration/src/lib.rs

@@ -1,4 +1,5 @@
 mod configuration;
+mod connect;
 mod values;
 
 pub mod environment;
@@ -18,6 +19,8 @@ pub use values::{ConnectionUri, IsolationLevel, PoolSettings, Secret};
 
 pub use metrics::Metrics;
 
+pub use connect::get_connect_options;
+
 #[derive(Debug, Copy, Clone)]
 pub enum VersionTag {
     Version3,

crates/configuration/src/version5/mod.rs

@@ -8,7 +8,6 @@ mod options;
 mod to_runtime_configuration;
 mod upgrade_from_v4;
 
-use std::borrow::Cow;
 use std::collections::HashSet;
 use std::path::Path;
 pub use to_runtime_configuration::make_runtime_configuration;
@@ -25,7 +24,6 @@ use metadata::database;
 
 use crate::environment::Environment;
 use crate::error::{ParseConfigurationError, WriteParsedConfigurationError};
-use crate::values::{ConnectionUri, Secret};
 
 const CONFIGURATION_FILENAME: &str = "configuration.json";
 const CONFIGURATION_JSONSCHEMA_FILENAME: &str = "schema.json";
@@ -134,14 +132,10 @@ pub async fn introspect(
     args: ParsedConfiguration,
     environment: impl Environment,
 ) -> anyhow::Result<ParsedConfiguration> {
-    let uri = match &args.connection_settings.connection_uri {
-        ConnectionUri(Secret::Plain(value)) => Cow::Borrowed(value),
-        ConnectionUri(Secret::FromEnvironment { variable }) => {
-            Cow::Owned(environment.read(variable)?)
-        }
-    };
+    let connect_options =
+        crate::get_connect_options(&args.connection_settings.connection_uri, environment)?;
 
-    let mut connection = PgConnection::connect(&uri)
+    let mut connection = PgConnection::connect_with(&connect_options)
         .instrument(info_span!("Connect to database"))
         .await?;
 

crates/configuration/src/version5/native_operations.rs

@@ -6,9 +6,11 @@ use std::path::Path;
 
 use query_engine_sql::sql;
 
+use sqlx::Column;
 use sqlx::Connection;
 use sqlx::Executor;
-use sqlx::{Column, PgConnection};
+
+use crate::environment::Environment;
 
 use super::metadata;
 use tracing::{info_span, Instrument};
@@ -24,12 +26,15 @@ pub enum Kind {
 /// and add it to the configuration if it is.
 pub async fn create(
     configuration: &super::ParsedConfiguration,
+    environment: &impl Environment,
     connection_string: &str,
     operation_path: &Path,
     operation_file_contents: &str,
 ) -> anyhow::Result<metadata::NativeQueryInfo> {
+    let connect_options =
+        crate::get_connect_options(&crate::ConnectionUri::from(connection_string), environment)?;
     // Connect to the db.
-    let mut connection = sqlx::PgConnection::connect(connection_string).await?;
+    let mut connection = sqlx::PgConnection::connect_with(&connect_options).await?;
 
     // Create an entry for a Native Operation and insert it into the configuration.
 
@@ -89,7 +94,8 @@ pub async fn create(
     let mut oids: BTreeSet<i64> = arguments_to_oids.values().copied().collect();
     oids.extend::<BTreeSet<i64>>(columns_to_oids.values().copied().map(|x| x.0).collect());
     let oids_vec: Vec<_> = oids.into_iter().collect();
-    let oids_map = oids_to_typenames(configuration, connection_string, &oids_vec).await?;
+    let oids_map =
+        oids_to_typenames(configuration, connection_string, environment, &oids_vec).await?;
 
     let mut arguments = BTreeMap::new();
     for (name, oid) in arguments_to_oids {
@@ -149,9 +155,13 @@ pub async fn create(
 pub async fn oids_to_typenames(
     configuration: &super::ParsedConfiguration,
     connection_string: &str,
+    environment: &impl Environment,
     oids: &Vec<i64>,
-) -> Result<BTreeMap<i64, models::ScalarTypeName>, sqlx::Error> {
-    let mut connection = PgConnection::connect(connection_string)
+) -> anyhow::Result<BTreeMap<i64, models::ScalarTypeName>> {
+    let connect_options =
+        crate::get_connect_options(&crate::ConnectionUri::from(connection_string), environment)?;
+    // Connect to the db.
+    let mut connection = sqlx::PgConnection::connect_with(&connect_options)
         .instrument(info_span!("Connect to database"))
         .await?;
 

crates/connectors/ndc-postgres/Cargo.toml

@@ -26,6 +26,7 @@ query-engine-translation = { path = "../../query-engine/translation" }
 
 ndc-sdk = { workspace = true }
 
+anyhow = { workspace = true }
 async-trait = { workspace = true }
 mimalloc = { workspace = true }
 percent-encoding = { workspace = true }

crates/connectors/ndc-postgres/src/connector.rs

@@ -277,8 +277,10 @@ impl<Env: Environment + Send + Sync> ConnectorSetup for PostgresSetup<Env> {
         configuration: &<Self::Connector as Connector>::Configuration,
         metrics: &mut prometheus::Registry,
     ) -> Result<<Self::Connector as Connector>::State> {
+        // create the state
         state::create_state(
             &configuration.connection_uri,
+            &self.environment,
             &configuration.pool_settings,
             metrics,
             configuration.configuration_version_tag,

crates/connectors/ndc-postgres/src/state.rs

@@ -2,13 +2,16 @@
 //!
 //! This is initialized on startup.
 
+use ndc_postgres_configuration::get_connect_options;
 use percent_encoding::percent_decode_str;
-use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions, PgRow};
-use sqlx::{ConnectOptions, Connection, Row};
+use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
+use sqlx::{Connection, Row};
 use thiserror::Error;
 use tracing::{info_span, Instrument};
 use url::Url;
 
+use ndc_postgres_configuration::environment::Environment;
+use ndc_postgres_configuration::ConnectionUri;
 use ndc_postgres_configuration::PoolSettings;
 use query_engine_execution::database_info::{self, DatabaseInfo, DatabaseVersion};
 use query_engine_execution::metrics;
@@ -25,14 +28,15 @@ pub struct State {
 /// Create a connection pool and wrap it inside a connector State.
 pub async fn create_state(
     connection_uri: &str,
+    environment: &impl Environment,
     pool_settings: &PoolSettings,
     metrics_registry: &mut prometheus::Registry,
     version_tag: ndc_postgres_configuration::VersionTag,
 ) -> Result<State, InitializationError> {
     let connection_url: Url = connection_uri
         .parse()
         .map_err(InitializationError::InvalidConnectionUri)?;
-    let pool = create_pool(&connection_url, pool_settings)
+    let pool = create_pool(connection_uri, environment, pool_settings)
         .instrument(info_span!(
             "Create connection pool",
             internal.visibility = "user",
@@ -84,11 +88,12 @@ pub async fn create_state(
 /// Create a connection pool with default settings.
 /// - <https://docs.rs/sqlx/latest/sqlx/pool/struct.PoolOptions.html>
 async fn create_pool(
-    connection_url: &Url,
+    connection_url: &str,
+    environment: impl Environment,
     pool_settings: &PoolSettings,
 ) -> Result<PgPool, InitializationError> {
-    let connect_options = PgConnectOptions::from_url(connection_url)
-        .map_err(InitializationError::UnableToCreatePool)?;
+    let connect_options = get_connect_options(&ConnectionUri::from(connection_url), environment)
+        .map_err(InitializationError::InvalidConnectOptions)?;
 
     let pool_options = match pool_settings.check_connection_after_idle {
         // Unless specified otherwise, sqlx will always ping on acquire.
@@ -166,6 +171,8 @@ fn decode_uri_component(component: &str) -> String {
 pub enum InitializationError {
     #[error("invalid connection URI: {0}")]
     InvalidConnectionUri(url::ParseError),
+    #[error("Invalid connect options: {0}")]
+    InvalidConnectOptions(anyhow::Error),
     #[error("unable to initialize connection pool: {0}")]
     UnableToCreatePool(sqlx::Error),
     #[error("unable to connect to the database: {0}")]

crates/tests/tests-common/src/common_tests/configuration_tests.rs

@@ -1,5 +1,6 @@
 //! Tests the configuration generation has not changed.
 
+use ndc_postgres_configuration::environment::EmptyEnvironment;
 use ndc_postgres_configuration::version5;
 use ndc_postgres_configuration::ParsedConfiguration;
 use std::collections::HashMap;
@@ -24,6 +25,7 @@ pub async fn test_native_operation_create_v5(
         ParsedConfiguration::Version5(parsed_configuration) => {
             let result = version5::native_operations::create(
                 &parsed_configuration,
+                &EmptyEnvironment,
                 connection_string,
                 &PathBuf::from("test.sql"),
                 &sql,