control egress traffic with iptables redirection (#76)

Signed-off-by: Maksim Paskal <[email protected]>

.goreleaser.yml

@@ -1,3 +1,8 @@
+release:
+  footer: |
+    ## Docker Images
+    - `paskalmaksim/envoy-control-plane:latest`
+    - `paskalmaksim/envoy-control-plane:{{ .Tag }}`
 dockers:
 - goos: linux
   goarch: amd64

Makefile

@@ -103,23 +103,23 @@ sslInit:
 	go run ./cmd/gencerts -cert.path=certs
 sslTest:
 	openssl rsa -in ./certs/CA.key -check -noout
-	openssl rsa -in ./certs/test.key -check -noout
-	openssl verify -CAfile ./certs/CA.crt ./certs/test.crt
+	openssl rsa -in ./certs/server.key -check -noout
+	openssl verify -CAfile ./certs/CA.crt ./certs/server.crt
 	openssl verify -CAfile ./certs/CA.crt ./certs/envoy.crt
 
-	openssl x509 -in ./certs/test.crt -text
+	openssl x509 -in ./certs/server.crt -text
 	openssl x509 -in ./certs/envoy.crt -text
 
 	openssl x509 -pubkey -in ./certs/CA.crt -noout | openssl md5
 	openssl pkey -pubout -in ./certs/CA.key | openssl md5
 
-	openssl x509 -pubkey -in ./certs/test.crt -noout | openssl md5
-	openssl pkey -pubout -in ./certs/test.key | openssl md5
+	openssl x509 -pubkey -in ./certs/server.crt -noout | openssl md5
+	openssl pkey -pubout -in ./certs/server.key | openssl md5
 sslTestClient:
-	curl -v --cacert ./certs/CA.crt --resolve "test2-id:8001:127.0.0.1" --key ./certs/test.key --cert ./certs/test.crt https://test2-id:8001
-	curl -v --cacert ./certs/CA.crt --resolve "test3-id:8002:127.0.0.1" --key ./certs/test.key --cert ./certs/test.crt https://test3-id:8002
+	curl -v --cacert ./certs/CA.crt --resolve "test2-id:8001:127.0.0.1" --key ./certs/server.key --cert ./certs/server.crt https://test2-id:8001
+	curl -v --cacert ./certs/CA.crt --resolve "test3-id:8002:127.0.0.1" --key ./certs/server.key --cert ./certs/server.crt https://test3-id:8002
 sslTestControlPlane:
-	curl -vk --http2 --cacert ./certs/CA.crt --resolve "envoy-control-plane:18080:127.0.0.1" --key ./certs/test.key --cert ./certs/test.crt https://envoy-control-plane:18080
+	curl -vk --http2 --cacert ./certs/CA.crt --resolve "envoy-control-plane:18080:127.0.0.1" --key ./certs/server.key --cert ./certs/server.crt https://envoy-control-plane:18080
 test-e2e:
 	make clean k8sConfig
 	kubectl scale deploy test-001 test-002 --replicas=${initialPodCount}

README.md

@@ -63,8 +63,11 @@ containers:
   lifecycle:
     preStop:
       exec:
-        # gracefully drain all connection
-        command: ["/bin/sh", "-c", "cli -drainEnvoy; sleep 5s; pkill -SIGTERM envoy"]
+        # gracefully drain all connection and shutdown
+        command:
+        - cli
+        - -drainEnvoy
+        - -timeout=10s
   image: paskalmaksim/envoy-docker-image:latest
   imagePullPolicy: Always
   args:
@@ -98,13 +101,13 @@ containers:
   readinessProbe:
     httpGet:
       path: /ready
-      port: 18000
+      port: 18001
     initialDelaySeconds: 3
     periodSeconds: 5
   livenessProbe:
     httpGet:
       path: /server_info
-      port: 18000
+      port: 18001
     initialDelaySeconds: 60
     periodSeconds: 10
 

charts/envoy-control-plane/templates/envoy.yaml

@@ -44,8 +44,11 @@ spec:
         lifecycle:
           preStop:
             exec:
-              # gracefully drain all connection
-              command: ["/bin/sh", "-c", "cli -drainEnvoy; sleep 5s; pkill -SIGTERM envoy"]
+              # gracefully drain all connection and shutdown
+              command:
+              - cli
+              - -drainEnvoy
+              - -timeout=10s
         image: {{ tpl .Values.envoy.registry.image . | quote }}
         imagePullPolicy: {{ .Values.envoy.registry.imagePullPolicy | quote }}
         args:
@@ -80,13 +83,13 @@ spec:
         readinessProbe:
           httpGet:
             path: /ready
-            port: 18000
+            port: 18001
           initialDelaySeconds: 3
           periodSeconds: 5
         livenessProbe:
           httpGet:
             path: /server_info
-            port: 18000
+            port: 18001
           initialDelaySeconds: 60
           periodSeconds: 10
         ports:

cmd/cli/main.go

@@ -30,6 +30,7 @@ import (
 const (
 	serverPort      = 18081
 	serverAdminPort = 18000
+	defaultTimeout  = 10 * time.Second
 )
 
 var (
@@ -42,7 +43,9 @@ var (
 	port           = flag.Int("port", serverPort, "controlplane port")
 	wait           = flag.Bool("wait", true, "wait controlplane")
 	debug          = flag.Bool("debug", false, "debug mode")
+	envoyLogLevel  = flag.String("envoyLogLevel", "", "set envoy log level")
 	drainEnvoy     = flag.Bool("drainEnvoy", false, "drain envoy")
+	timeout        = flag.Duration("timeout", defaultTimeout, "timeout to shutdown envoy")
 	envoyAdminPort = flag.Int("envoyAdminPort", serverAdminPort, "envoy admin port")
 	logFlags       = flag.Int("logFlags", 0, "log flags")
 	tlsInsecure    = flag.Bool("tls.insecure", false, "use insecure TLS")
@@ -86,7 +89,8 @@ func waitForAPI() {
 	}
 }
 
-func requestEnvoyAdmin(method string, path string) {
+func requestEnvoyAdmin(path string) {
+	method := http.MethodPost
 	url := fmt.Sprintf("http://127.0.0.1:%d%s", *envoyAdminPort, path)
 
 	req, err := http.NewRequestWithContext(ctx, method, url, nil)
@@ -146,9 +150,23 @@ func main() {
 		},
 	}
 
+	if len(*envoyLogLevel) > 0 {
+		requestEnvoyAdmin(fmt.Sprintf("/logging?level=%s", *envoyLogLevel))
+
+		return
+	}
+
 	if *drainEnvoy {
-		requestEnvoyAdmin(http.MethodPost, "/drain_listeners?graceful")
-		requestEnvoyAdmin(http.MethodPost, "/healthcheck/fail")
+		// draining connections
+		requestEnvoyAdmin("/drain_listeners?graceful")
+		requestEnvoyAdmin("/healthcheck/fail")
+
+		// wait some time
+		log.Printf("Waiting %s to Envoy quit", *timeout)
+		time.Sleep(*timeout)
+
+		// shutdown envoy
+		requestEnvoyAdmin("/quitquitquit")
 
 		return
 	}

cmd/gencerts/main.go

@@ -17,37 +17,49 @@ import (
 	"io/fs"
 	"io/ioutil"
 	"path"
+	"strings"
 
 	"github.com/maksim-paskal/envoy-control-plane/pkg/certs"
+	"github.com/maksim-paskal/envoy-control-plane/pkg/config"
 	log "github.com/sirupsen/logrus"
 )
 
 func main() {
 	certPath := flag.String("cert.path", "certs", "path to generate certificates")
+	dnsNames := flag.String("dns.names", "test", "dns names for server certificate")
 
 	flag.Parse()
 
 	files := make(map[string][]byte)
 
-	log.Info("generate certificates")
+	if err := certs.Init(); err != nil {
+		log.WithError(err).Fatal()
+	}
+
+	rootCrt := certs.GetLoadedRootCert()
+	rootCrtBytes := certs.GetLoadedRootCertBytes()
+
+	rootKey := certs.GetLoadedRootKey()
 
-	rootCrt, rootCrtBytes, rootKey, rootKeyBytes, err := certs.GenCARoot()
+	rootKeyBytes, err := certs.GetLoadedRootKeyBytes()
 	if err != nil {
-		log.Fatal(err)
+		log.WithError(err).Fatal()
 	}
 
-	files["CA.crt"] = rootCrtBytes
-	files["CA.key"] = rootKeyBytes
+	if len(*config.Get().SSLCrt) == 0 && len(*config.Get().SSLKey) == 0 {
+		files["CA.crt"] = rootCrtBytes
+		files["CA.key"] = rootKeyBytes
+	}
 
-	_, serverCrtBytes, _, serverKeyBytes, err := certs.GenServerCert("test", rootCrt, rootKey, certs.CertValidity)
+	_, serverCrtBytes, _, serverKeyBytes, err := certs.GenServerCert(strings.Split(*dnsNames, ","), rootCrt, rootKey, certs.CertValidityMax) //nolint:lll
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	files["test.crt"] = serverCrtBytes
-	files["test.key"] = serverKeyBytes
+	files["server.crt"] = serverCrtBytes
+	files["server.key"] = serverKeyBytes
 
-	_, envoyCrtBytes, _, envoyKeyBytes, err := certs.GenServerCert("envoy", rootCrt, rootKey, certs.CertValidityMax)
+	_, envoyCrtBytes, _, envoyKeyBytes, err := certs.GenServerCert([]string{"envoy"}, rootCrt, rootKey, certs.CertValidityMax) //nolint:lll
 	if err != nil {
 		log.Fatal(err)
 	}

envoy/Dockerfile

@@ -4,6 +4,7 @@ ADD --chown=envoy:envoy https://github.com/maksim-paskal/go-template/releases/do
 ADD --chown=envoy:envoy https://github.com/maksim-paskal/jaeger-client-cpp/releases/download/v0.5.0/libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
 
 COPY ./cli /usr/local/bin/cli
+COPY ./envoy/scripts /scripts
 COPY ./envoy/entrypoint.sh /entrypoint.sh
 COPY ./envoy/envoy.defaults /envoy.defaults/
 COPY ./envoy/certs /certs/
@@ -14,8 +15,11 @@ RUN touch /tmp/checksum \
 && cat /tmp/checksum \
 && sha256sum -c /tmp/checksum \
 && rm /tmp/checksum \
-&& chmod +x /usr/local/bin/go-template /entrypoint.sh /usr/local/bin/cli \
-&& chown -R 101:101 /etc/envoy
+&& chmod +x /usr/local/bin/go-template /entrypoint.sh /usr/local/bin/cli /scripts/* \
+&& chown -R 101:101 /etc/envoy \
+&& apt update; apt install -y iptables \
+&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+&& rm -rf /var/lib/apt/lists/*
 
 USER 101
 

envoy/envoy.defaults/envoy.yaml

@@ -34,6 +34,32 @@ dynamic_resources:
           cluster_name: xds_cluster
 
 static_resources:
+  listeners:
+  - name: envoy_healthcheck
+    address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 18001
+    traffic_direction: INBOUND
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: envoy_healthcheck
+          route_config:
+            name: envoy_healthcheck
+            virtual_hosts:
+            - name: envoy_healthcheck
+              domains:
+              - "*"
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: envoy_admin_cluster
+          http_filters:
+          - name: envoy.filters.http.router
   clusters:
   - name: {{ env "ENVOY_SERVICE_NAME" }}
     connect_timeout: 0.25s
@@ -95,6 +121,19 @@ static_resources:
               socket_address:
                 address: {{ env "XDS_CLUSTER_ADDRESS" }}
                 port_value: 18080
+  - name: envoy_admin_cluster
+    connect_timeout: 1s
+    type: STATIC
+    dns_lookup_family: V4_ONLY
+    load_assignment:
+      cluster_name: envoy_admin_cluster
+      endpoints:
+      - lb_endpoints:
+          - endpoint:
+              address:
+                socket_address:
+                  address: 127.0.0.1
+                  port_value: 18000
 
 admin:
   address:

envoy/scripts/prepare_proxy.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+# Copyright [email protected]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+: ${KUBERNETES_SERVICE_HOST:='127.0.0.1'}
+: ${ENVOY_PORT:='15001'}
+: ${ENVOY_UID:='101'}
+
+iptables -t nat -N PROXY_INBOUND
+iptables -t nat -N PROXY_OUTBOUND
+iptables -t nat -N PROXY_REDIRECT
+iptables -t nat -N PROXY_IN_REDIRECT
+iptables -t nat -N PROXY_OUT_REDIRECT
+
+# Redirects outbound TCP traffic hitting PROXY_OUT_REDIRECT chain to Envoy's outbound listener port
+iptables -t nat -A PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port $ENVOY_PORT
+
+# Traffic to the Proxy Admin port flows to the Proxy -- not redirected
+iptables -t nat -A PROXY_OUT_REDIRECT -p tcp --dport 18000 -j ACCEPT
+
+# For outbound TCP traffic jump from OUTPUT chain to PROXY_OUTBOUND chain
+iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTBOUND
+
+# Outbound traffic from Envoy to the local app over the loopback interface should jump to the inbound proxy redirect chain.
+# So when an app directs traffic to itself via the k8s service, traffic flows as follows:
+# app -> local envoy's outbound listener -> iptables -> local envoy's inbound listener -> app
+iptables -t nat -A PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner $ENVOY_UID -j PROXY_IN_REDIRECT
+
+# Outbound traffic from the app to itself over the loopback interface is not be redirected via the proxy.
+# E.g. when app sends traffic to itself via the pod IP.
+iptables -t nat -A PROXY_OUTBOUND -o lo -m owner ! --uid-owner $ENVOY_UID -j RETURN
+
+# Don't redirect Envoy traffic back to itself, return it to the next chain for processing
+iptables -t nat -A PROXY_OUTBOUND -m owner --uid-owner $ENVOY_UID -j RETURN
+
+# Skip localhost traffic, doesn't need to be routed via the proxy
+iptables -t nat -A PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
+
+# Skip traffic to kubernetes API
+iptables -t nat -A PROXY_OUTBOUND -d $KUBERNETES_SERVICE_HOST/32 -j RETURN
+
+# Redirect remaining outbound traffic to Envoy
+iptables -t nat -A PROXY_OUTBOUND -j PROXY_OUT_REDIRECT
+
+# Redirects inbound TCP traffic hitting the PROXY_IN_REDIRECT chain to Envoy's inbound listener port
+iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port $ENVOY_PORT
+
+# For inbound traffic jump from PREROUTING chain to PROXY_INBOUND chain
+iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND
+
+# Skip traffic to Envoy sidecar metrics being directed to Envoy 
+iptables -t nat -A PROXY_INBOUND -p tcp --dport 18001 -j RETURN
+
+# Skip traffic being directed to Envoy - this needed for POD livenessProbe, readinessProbe, startupProbe
+# or application metrics for Prometheus scraping
+iptables -t nat -A PROXY_INBOUND -p tcp --dport 18002 -j RETURN
+iptables -t nat -A PROXY_INBOUND -p tcp --dport 18003 -j RETURN
+iptables -t nat -A PROXY_INBOUND -p tcp --dport 18004 -j RETURN
+iptables -t nat -A PROXY_INBOUND -p tcp --dport 18005 -j RETURN
+
+# Redirect remaining inbound traffic to Envoy
+iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT

examples/.gitignore

@@ -0,0 +1 @@
+certs

examples/egress/Makefile

@@ -0,0 +1,23 @@
+KUBECONFIG=$(HOME)/.kube/dev
+
+deploy:
+	make clean
+	kubectl create cm envoy-config \
+	--from-file=envoy.yaml \
+	--from-file=certs/server.key \
+	--from-file=certs/server.crt
+	kubectl apply -f test.yaml
+clean:
+	kubectl delete cm envoy-config || true
+	kubectl delete -f test.yaml || true
+run:
+	docker-compose down --remove-orphans; docker-compose up
+genCerts:
+	rm -rf certs
+	mkdir certs
+	go run ../../cmd/gencerts -dns.names=\
+	get.paskal-dev.com,\
+	google.com,\
+	www.recaptcha.net
+
+	openssl x509 -in ./certs/server.crt -text -noout