Support secure connections to ClickHouse DB (antrea-io#5171)

The Flow Aggregator can now use a secure (TLS) connection to connect to
the ClickHouse DB. This is especially important when connecting to a remote
ClickHouse instance, e.g., one running in a different K8s cluster.

1. Support TLS when connecting the Flow Aggregator to ClickHouse.
2. Support HTTP/HTTPS. Users can directly modify the clickHouse.databaseURL in order to switch protocol.
3. Support custom CA certificate. Users need to create a clickhouse-ca Secret in the flow-aggregator Namespace. The custom CA certificate will be used when clickhouse.tls.caCert is true.
4. Update network-flow-visibility.md.
5. Add e2e test for HTTP/HTTPS/TLS protocols when connecting the Flow Aggregator to ClickHouse.

Fixes antrea-io#4902 

Signed-off-by: Yun-Tang Hsu <[email protected]>

build/charts/flow-aggregator/README.md

@@ -26,9 +26,11 @@ Kubernetes: `>= 1.16.0-0`
 | clickHouse.commitInterval | string | `"8s"` | CommitInterval is the periodical interval between batch commit of flow records to DB. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". |
 | clickHouse.compress | bool | `true` | Compress enables lz4 compression when committing flow records. |
 | clickHouse.connectionSecret | object | `{"password":"clickhouse_operator_password","username":"clickhouse_operator"}` | Credentials to connect to ClickHouse. They will be stored in a Secret. |
-| clickHouse.databaseURL | string | `"tcp://clickhouse-clickhouse.flow-visibility.svc:9000"` | DatabaseURL is the url to the database. TCP protocol is required. |
+| clickHouse.databaseURL | string | `"tcp://clickhouse-clickhouse.flow-visibility.svc:9000"` | DatabaseURL is the url to the database. Provide the database URL as a string with format <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be one of the following: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls will be enabled. |
 | clickHouse.debug | bool | `false` | Debug enables debug logs from ClickHouse sql driver. |
 | clickHouse.enable | bool | `false` | Determine whether to enable exporting flow records to ClickHouse. |
+| clickHouse.tls.caCert | bool | `false` | Indicates whether to use custom CA certificate. Default root CAs will be used if this field is false. If true, a Secret named "clickhouse-ca" must be provided with the following keys: ca.crt: <CA certificate> |
+| clickHouse.tls.insecureSkipVerify | bool | `false` | Determine whether to skip the verification of the server's certificate chain and host name. Default is false. |
 | flowAggregatorAddress | string | `""` | Provide an extra DNS name or IP address of flow aggregator for generating TLS certificate. |
 | flowCollector.address | string | `""` | Provide the flow collector address as string with format <IP>:<port>[:<proto>],  where proto is tcp or udp. If no L4 transport proto is given, we consider tcp as default. |
 | flowCollector.enable | bool | `false` | Determine whether to enable exporting flow records to external flow collector. |

build/charts/flow-aggregator/conf/flow-aggregator.conf

@@ -72,9 +72,23 @@ clickHouse:
   # Database is the name of database where Antrea "flows" table is created.
   database: "default"
 
-  # DatabaseURL is the url to the database. TCP protocol is required.
+  # DatabaseURL is the url to the database. Provide the database URL as a string with format
+  # <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be
+  # one of the following: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls
+  # will be enabled.
   databaseURL: {{ .Values.clickHouse.databaseURL | quote }}
 
+  # TLS configuration options, when using TLS to connect to the ClickHouse service.
+  tls:
+    # InsecureSkipVerify determines whether to skip the verification of the server's certificate chain and host name.
+    # Default is false.
+    insecureSkipVerify: {{ .Values.clickHouse.tls.insecureSkipVerify }}
+
+    # CACert indicates whether to use custom CA certificate. Default root CAs will be used if this field is false.
+    # If true, a Secret named "clickhouse-ca" must be provided with the following keys:
+    # ca.crt: <CA certificate>
+    caCert: {{ .Values.clickHouse.tls.caCert }}
+
   # Debug enables debug logs from ClickHouse sql driver.
   debug: {{ .Values.clickHouse.debug }}
 

build/charts/flow-aggregator/templates/deployment.yaml

@@ -79,6 +79,8 @@ spec:
           readOnly: true
         - mountPath: /var/log/antrea/flow-aggregator
           name: host-var-log-antrea-flow-aggregator
+        - name: clickhouse-ca
+          mountPath: /etc/flow-aggregator/certs
       nodeSelector:
         kubernetes.io/os: linux
         kubernetes.io/arch: amd64
@@ -91,3 +93,9 @@ spec:
         hostPath:
           path: /var/log/antrea/flow-aggregator
           type: DirectoryOrCreate
+      # Make it optional as we only read it when caCert=true.
+      - name: clickhouse-ca
+        secret:
+          secretName: clickhouse-ca
+          defaultMode: 0400
+          optional: true

build/charts/flow-aggregator/values.yaml

@@ -45,8 +45,18 @@ flowCollector:
 clickHouse:
   # -- Determine whether to enable exporting flow records to ClickHouse.
   enable: false
-  # -- DatabaseURL is the url to the database. TCP protocol is required.
+  # -- DatabaseURL is the url to the database. Provide the database URL as a string with format
+  # <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be one of
+  # the following: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls will be enabled.
   databaseURL: "tcp://clickhouse-clickhouse.flow-visibility.svc:9000"
+  # TLS configuration options, when using TLS to connect to the ClickHouse service.
+  tls:
+    # -- Determine whether to skip the verification of the server's certificate chain and host name. Default is false.
+    insecureSkipVerify: false
+    # -- Indicates whether to use custom CA certificate. Default root CAs will be used if this field is false.
+    # If true, a Secret named "clickhouse-ca" must be provided with the following keys:
+    # ca.crt: <CA certificate>
+    caCert: false
   # -- Debug enables debug logs from ClickHouse sql driver.
   debug: false
   # -- Compress enables lz4 compression when committing flow records.

build/yamls/flow-aggregator.yml

@@ -224,9 +224,23 @@ data:
       # Database is the name of database where Antrea "flows" table is created.
       database: "default"
 
-      # DatabaseURL is the url to the database. TCP protocol is required.
+      # DatabaseURL is the url to the database. Provide the database URL as a string with format
+      # <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be
+      # one of the following: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls
+      # will be enabled.
       databaseURL: "tcp://clickhouse-clickhouse.flow-visibility.svc:9000"
 
+      # TLS configuration options, when using TLS to connect to the ClickHouse service.
+      tls:
+        # InsecureSkipVerify determines whether to skip the verification of the server's certificate chain and host name.
+        # Default is false.
+        insecureSkipVerify: false
+
+        # CACert indicates whether to use custom CA certificate. Default root CAs will be used if this field is false.
+        # If true, a Secret named "clickhouse-ca" must be provided with the following keys:
+        # ca.crt: <CA certificate>
+        caCert: false
+
       # Debug enables debug logs from ClickHouse sql driver.
       debug: false
 
@@ -434,6 +448,8 @@ spec:
           readOnly: true
         - mountPath: /var/log/antrea/flow-aggregator
           name: host-var-log-antrea-flow-aggregator
+        - mountPath: /etc/flow-aggregator/certs
+          name: clickhouse-ca
       nodeSelector:
         kubernetes.io/arch: amd64
         kubernetes.io/os: linux
@@ -446,3 +462,8 @@ spec:
           path: /var/log/antrea/flow-aggregator
           type: DirectoryOrCreate
         name: host-var-log-antrea-flow-aggregator
+      - name: clickhouse-ca
+        secret:
+          defaultMode: 256
+          optional: true
+          secretName: clickhouse-ca

ci/kind/test-e2e-kind.sh

@@ -45,12 +45,13 @@ function print_usage {
 }
 
 
-TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
-YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh"
-FLOWAGGREGATOR_YML_CMD=$(dirname $0)"/../../hack/generate-manifest-flow-aggregator.sh"
-FLOW_VISIBILITY_HELM_VALUES=$(dirname $0)"/values-flow-exporter.yml"
-CH_OPERATOR_YML=$(dirname $0)"/../../build/yamls/clickhouse-operator-install-bundle.yml"
-FLOW_VISIBILITY_YML=$(dirname $0)"/../../build/yamls/flow-visibility-e2e.yml"
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+TESTBED_CMD="$THIS_DIR/kind-setup.sh"
+YML_CMD="$THIS_DIR/../../hack/generate-manifest.sh"
+FLOWAGGREGATOR_YML_CMD="$THIS_DIR/../../hack/generate-manifest-flow-aggregator.sh"
+FLOW_VISIBILITY_HELM_VALUES="$THIS_DIR/values-flow-exporter.yml"
+CH_OPERATOR_YML="$THIS_DIR/../../build/yamls/clickhouse-operator-install-bundle.yml"
+FLOW_VISIBILITY_CHART="$THIS_DIR/../../test/e2e/charts/flow-visibility"
 
 function quit {
   result=$?
@@ -147,6 +148,16 @@ case $key in
 esac
 done
 
+source $THIS_DIR/../../hack/verify-helm.sh
+
+if [ -z "$HELM" ]; then
+    HELM="$(verify_helm)"
+elif ! $HELM version > /dev/null 2>&1; then
+    echoerr "$HELM does not appear to be a valid helm binary"
+    print_help
+    exit 1
+fi
+
 if [[ $cleanup_only == "true" ]];then
   $TESTBED_CMD destroy kind
   exit 0
@@ -250,14 +261,16 @@ function run_test {
   fi
 
   if $flow_visibility; then
-      timeout="10m"
+      timeout="15m"
       flow_visibility_args="-run=TestFlowAggregator --flow-visibility"
       if $coverage; then
           $FLOWAGGREGATOR_YML_CMD --coverage | docker exec -i kind-control-plane dd of=/root/flow-aggregator-coverage.yml
       else
           $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
       fi
-      cat $FLOW_VISIBILITY_YML | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml
+      $HELM template "$FLOW_VISIBILITY_CHART"  | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml
+      $HELM template "$FLOW_VISIBILITY_CHART" --set "secureConnection.enable=true" | docker exec -i kind-control-plane dd of=/root/flow-visibility-tls.yml
+
       curl -o $CH_OPERATOR_YML https://raw.githubusercontent.com/Altinity/clickhouse-operator/release-0.21.0/deploy/operator/clickhouse-operator-install-bundle.yaml
       sed -i -e "s|\"image\": \"clickhouse/clickhouse-server:22.3\"|\"image\": \"projects.registry.vmware.com/antrea/clickhouse-server:23.4\"|g" $CH_OPERATOR_YML
       sed -i -e "s|image: altinity/clickhouse-operator:0.21.0|image: projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0|g" $CH_OPERATOR_YML

docs/network-flow-visibility.md

@@ -17,6 +17,8 @@
 - [Flow Aggregator](#flow-aggregator)
   - [Deployment](#deployment)
   - [Configuration](#configuration-1)
+    - [Configuring secure connections to the ClickHouse database](#configuring-secure-connections-to-the-clickhouse-database)
+    - [Example of flow-aggregator.conf](#example-of-flow-aggregatorconf)
   - [IPFIX Information Elements (IEs) in an Aggregated Flow Record](#ipfix-information-elements-ies-in-an-aggregated-flow-record)
     - [IEs from Antrea IE Registry](#ies-from-antrea-ie-registry-1)
   - [Supported Capabilities](#supported-capabilities-1)
@@ -283,7 +285,43 @@ it is deployed following the [deployment steps](#deployment-steps-1), the
 ClickHouse server is already exposed via a K8s Service, and no further
 configuration is required. If a different FQDN or IP is desired, please use
 the URL for `clickHouse.databaseURL` in the following format:
-`tcp://<ClickHouse server FQDN or IP>:<ClickHouse TCP port>`.
+`<protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>`.
+
+#### Configuring secure connections to the ClickHouse database
+
+Starting with Antrea v1.13, you can enable TLS when connecting to the ClickHouse
+Server by setting `clickHouse.databaseURL` with protocol `tls` or `https`.
+You can also change the value of `clickHouse.tls.insecureSkipVerify` to
+determine whether to skip the verification of the server's certificate.
+If you want to provide a custom CA certificate, you can set
+`clickHouse.tls.caCert` to `true` and the flow Aggregator will read the
+certificate key pair from the`clickhouse-ca` Secret.
+
+Make sure to follow the following form when creating the `clickhouse-ca` Secret
+with the custom CA certificate:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clickhouse-ca
+  namespace: flow-aggregator
+data:
+  ca.crt: <BASE64 ENCODED CA CERTIFICATE>
+```
+
+You can use `kubectl apply -f <PATH TO SECRET YAML>` to create the above secret
+, or use `kubectl create secret`:
+
+```bash
+kubectl create secret generic clickhouse-ca -n flow-aggregator --from-file=ca.crt=<PATH TO CA CERTIFICATE>
+```
+
+Prior to Antrea v1.13, secure connections to ClickHouse are not supported,
+and TCP is the only supported protocol when connecting to the ClickHouse
+server from the Flow Aggregator.
+
+#### Example of flow-aggregator.conf
 
 ```yaml
 flow-aggregator.conf: |  
@@ -357,9 +395,23 @@ flow-aggregator.conf: |
     # Database is the name of database where Antrea "flows" table is created.
     database: "default"
 
-    # DatabaseURL is the url to the database. TCP protocol is required.
+    # DatabaseURL is the url to the database. Provide the database URL as a string with format
+    # <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be
+    # one of the following: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls
+    # will be enabled.
     databaseURL: "tcp://clickhouse-clickhouse.flow-visibility.svc:9000"
 
+    # TLS configuration options, when using TLS to connect to the ClickHouse service.
+    tls:
+      # InsecureSkipVerify determines whether to skip the verification of the server's certificate chain and host name.
+      # Default is false.
+      insecureSkipVerify: false
+
+      # CACert indicates whether to use custom CA certificate. Default root CAs will be used if this field is false.
+      # If true, a Secret named "clickhouse-ca" must be provided with the following keys:
+      # ca.crt: <CA certificate>
+      caCert: false
+
     # Debug enables debug logs from ClickHouse sql driver.
     debug: false
 

pkg/config/flowaggregator/config.go

@@ -95,7 +95,9 @@ type ClickHouseConfig struct {
 	Enable bool `yaml:"enable,omitempty"`
 	// Database is the name of database where Antrea "flows" table is created.
 	Database string `yaml:"database,omitempty"`
-	// DatabaseURL is the url to the database. TCP protocol is required.
+	// DatabaseURL is the url to the database. Provide the database URL as a string with format
+	// <Protocol>://<ClickHouse server FQDN or IP>:<ClickHouse port>. The protocol has to be one
+	// from below: "tcp", "tls", "http", "https". When "tls" or "https" is used, tls will be enabled.
 	// Defaults to "tcp://clickhouse-clickhouse.flow-visibility.svc:9000"
 	DatabaseURL string `yaml:"databaseURL,omitempty"`
 	// Debug enables debug logs from ClickHouse sql driver. Defaults to false.
@@ -106,6 +108,18 @@ type ClickHouseConfig struct {
 	// Defaults to "8s". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
 	// Min value allowed is "1s".
 	CommitInterval string `yaml:"commitInterval,omitempty"`
+	// TLS configuration options, when using TLS to connect to the ClickHouse service.
+	TLS TLSConfig `yaml:"tls,omitempty"`
+}
+
+type TLSConfig struct {
+	// InsecureSkipVerify determines whether to skip the verification of the server's certificate chain and host name.
+	// Default is false.
+	InsecureSkipVerify bool `yaml:"insecureSkipVerify,omitempty"`
+	// CACert determines whether to use custom CA certificate. Default root CAs will be used if false.
+	// If true, a Secret named "flow-aggregator-ca" must be provided with the following keys:
+	// ca.crt: <CA certificate>
+	CACert bool `yaml:"caCert,omitempty"`
 }
 
 type S3UploaderConfig struct {