Configure Cilium agents to connect to apiserver explicitly

* Cilium v1.14 seems to have problems reliably accessing the
apiserver via default in-cluster service discovery (relies on
kube-proxy instead of DNS) after some time
* Configure Cilium agents to use the DNS name resolving to the
cluster's load balanced apiserver and port. Regrettably, this
relies on external DNS rather than being self-contained, but its
what Cilium pushes users towards

manifests.tf

@@ -32,6 +32,8 @@ locals {
         cluster_domain_suffix  = var.cluster_domain_suffix
         cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
         server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        apiserver_host         = var.api_servers[0]
+        apiserver_port         = var.external_apiserver_port
         daemonset_tolerations  = var.daemonset_tolerations
         token_id               = random_password.bootstrap-token-id.result
         token_secret           = random_password.bootstrap-token-secret.result

resources/cilium/daemonset.yaml

@@ -109,6 +109,16 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
         ports:
           # Not yet used, prefer exec's
           - name: health

resources/cilium/deployment.yaml

@@ -47,6 +47,16 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
         - name: CILIUM_DEBUG
           valueFrom:
             configMapKeyRef:

resources/manifests/in-cluster.yaml

@@ -0,0 +1,10 @@
+# in-cluster ConfigMap is for control plane components that must reach
+# kube-apiserver before service IPs are available (e.g. 10.3.0.1)
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: in-cluster
+  namespace: kube-system
+data:
+  apiserver-host: ${apiserver_host}
+  apiserver-port: "${apiserver_port}"