diff --git a/docs/releases/1.31-NOTES.md b/docs/releases/1.31-NOTES.md index 905f14d14bb0b..18b990bdd16e1 100644 --- a/docs/releases/1.31-NOTES.md +++ b/docs/releases/1.31-NOTES.md @@ -24,6 +24,8 @@ Lorem ipsum.... # Other changes of note +* Cilium has been upgraded to v1.16. + * Spotinst cluster controller V1 is replaced with Ocean kubernetes controller V2, all old k8s resource are removed except spotinst-kubernetes-cluster-controller Secret. diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index ba3b9ef90598f..a3b303b7edf85 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -1293,8 +1293,8 @@ func validateNetworkingCilium(cluster *kops.Cluster, v *kops.CiliumNetworkingSpe allErrs = append(allErrs, field.Invalid(versionFld, v.Version, "Could not parse as semantic version")) } - if version.Minor != 15 { - allErrs = append(allErrs, field.Invalid(versionFld, v.Version, "Only version 1.15 is supported")) + if version.Minor != 16 { + allErrs = append(allErrs, field.Invalid(versionFld, v.Version, "Only version 1.16 is supported")) } if v.Hubble != nil && fi.ValueOf(v.Hubble.Enabled) { diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index d6fe4806a77d6..da2136c5c9db4 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -1137,7 +1137,7 @@ func Test_Validate_Cilium(t *testing.T) { }, { Cilium: kops.CiliumNetworkingSpec{ - Version: "v1.15.0", + Version: "v1.16.0", Ingress: &kops.CiliumIngressSpec{ Enabled: fi.PtrTo(true), DefaultLoadBalancerMode: "bad-value", @@ -1147,7 +1147,7 @@ func Test_Validate_Cilium(t *testing.T) { }, { Cilium: kops.CiliumNetworkingSpec{ - Version: "v1.15.0", + Version: "v1.16.0", Ingress: &kops.CiliumIngressSpec{ Enabled: fi.PtrTo(true), DefaultLoadBalancerMode: "dedicated", @@ -1156,7 +1156,7 @@ func Test_Validate_Cilium(t *testing.T) { }, { Cilium: kops.CiliumNetworkingSpec{ - Version: "v1.15.0", + Version: "v1.16.0", Hubble: &kops.HubbleSpec{ Enabled: fi.PtrTo(true), }, diff --git a/pkg/model/components/cilium.go b/pkg/model/components/cilium.go index eb3c38e87df0e..f360c9e832152 100644 --- a/pkg/model/components/cilium.go +++ b/pkg/model/components/cilium.go @@ -40,7 +40,7 @@ func (b *CiliumOptionsBuilder) BuildOptions(o *kops.Cluster) error { } if c.Version == "" { - c.Version = "v1.15.6" + c.Version = "v1.16.1" } if c.EnableEndpointHealthChecking == nil { diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content index 1a03a06ac6976..773dac5024b54 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content @@ -226,7 +226,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: disabled - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content index 7924c4d186dbd..da3e25d979a31 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content @@ -106,7 +106,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: b9879c934ae3fc644e07f15629981bb9bf0162335a4ef5be413182fcfc66897a + manifestHash: da0ef2e57342372e25f1280da556dbe12a2a0e2b81f9d2463b20c804820abd7e name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content index 1a2219da9accb..68698ed1db076 100644 --- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -62,6 +62,7 @@ data: kube-proxy-replacement: "false" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: '[::1]:9234' preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: native @@ -135,6 +136,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -184,11 +188,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -260,6 +263,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -318,6 +325,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -340,6 +350,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -364,6 +379,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -499,6 +517,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -550,7 +573,7 @@ spec: value: api.internal.minimal-ipv6.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -590,6 +613,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -601,12 +640,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -630,7 +674,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -647,7 +691,7 @@ spec: value: api.internal.minimal-ipv6.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -666,11 +710,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -687,10 +737,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -698,6 +755,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -713,14 +786,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.minimal-ipv6.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -734,7 +821,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -811,6 +898,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -889,7 +984,7 @@ spec: value: api.internal.minimal-ipv6.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -901,6 +996,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: ::1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data b/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data index 8b4d4b0b4bb6c..f5b57c88cd9ad 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data @@ -153,7 +153,7 @@ ConfigServer: - https://kops-controller.internal.minimal-warmpool.example.com:3988/ InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: Qk29AY0f5+WYSZtngVmowAvt0IFItqN2mBDATTa1yqU= +NodeupConfigHash: 9eR3ArCmiOtRlM5MiKgIeyh9zBfs2MNlwaMYUH85wUs= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content index bacf9521bd689..799913da06baa 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content @@ -218,7 +218,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: vxlan - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content index b62670d5838c2..11dc2cbedd44c 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content @@ -99,7 +99,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: a1a193f3b5a7e4978166141793abd91ca31da43c5d22ccac28cbe8a9e971620e + manifestHash: 4f58454b1058faea22637f20d8a07415aa92609904d8d9047ccf132ba7d8aad6 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content index ca3076af4d942..4a8707e53dc28 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -62,6 +62,7 @@ data: kube-proxy-replacement: "false" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: tunnel @@ -136,6 +137,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -185,11 +189,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -261,6 +264,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -319,6 +326,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -341,6 +351,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -365,6 +380,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -500,6 +518,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -551,7 +574,7 @@ spec: value: api.internal.minimal-warmpool.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -591,6 +614,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -602,12 +641,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -631,7 +675,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -648,7 +692,7 @@ spec: value: api.internal.minimal-warmpool.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -667,11 +711,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -688,10 +738,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -699,6 +756,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -714,14 +787,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.minimal-warmpool.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -735,7 +822,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -812,6 +899,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -890,7 +985,7 @@ spec: value: api.internal.minimal-warmpool.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -902,6 +997,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_nodeupconfig-nodes_content index 2e414a4fdf504..96a1b013484a9 100644 --- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_nodeupconfig-nodes_content @@ -64,7 +64,8 @@ containerdConfig: usesLegacyGossip: false usesNoneDNS: false warmPoolImages: -- quay.io/cilium/cilium:v1.15.6 -- quay.io/cilium/operator:v1.15.6 +- quay.io/cilium/cilium:v1.16.1 +- quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 +- quay.io/cilium/operator:v1.16.1 - registry.k8s.io/kube-proxy:v1.26.0 - registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.11 diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_cluster-completed.spec_content index 7764fbe3827f2..a8d798d1a4eff 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_cluster-completed.spec_content @@ -199,7 +199,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: vxlan - version: v1.15.6 + version: v1.16.1 nonMasqueradeCIDR: 100.64.0.0/10 podCIDR: 100.96.0.0/11 secretStore: memfs://tests/scw-minimal.k8s.local/secrets diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content index bbaff6a4da892..100660860edfb 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content @@ -55,7 +55,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 7b74c26eba86a08e584e9621b100ef63a3aedca452958210ae67304f84d40542 + manifestHash: 867fc89c551b1efeb56de4cce715099a543f713551a05428cb1d0a3299fc46b4 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-networking.cilium.io-k8s-1.16_content index 6ea1c05f6aea4..6d352164a4097 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-networking.cilium.io-k8s-1.16_content @@ -62,6 +62,7 @@ data: kube-proxy-replacement: "true" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: tunnel @@ -136,6 +137,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -185,11 +189,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -261,6 +264,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -319,6 +326,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -341,6 +351,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -365,6 +380,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -500,6 +518,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -551,7 +574,7 @@ spec: value: api.internal.scw-minimal.k8s.local - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -591,6 +614,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -602,12 +641,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -631,7 +675,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -648,7 +692,7 @@ spec: value: api.internal.scw-minimal.k8s.local - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -667,11 +711,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -688,10 +738,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -699,6 +756,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -714,14 +787,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.scw-minimal.k8s.local - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -735,7 +822,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -812,6 +899,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -890,7 +985,7 @@ spec: value: api.internal.scw-minimal.k8s.local - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -902,6 +997,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content index a975d0b93afdd..6a29439e47942 100644 --- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content @@ -220,7 +220,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: disabled - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content index 141bb6014d32f..8ce2ef2ba0f7c 100644 --- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content @@ -99,7 +99,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 73bb75823f5a80f87197e6fcb8dc72a63ee1c24883175dac77300e6902681161 + manifestHash: 7d691d06fc71e313cb156d6a75dcdb2f3f1a03fe41661fbe2260b5d1823ccb0d name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content index bc984798b19b5..31c17b411bad4 100644 --- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -65,6 +65,7 @@ data: kube-proxy-replacement: "false" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: native @@ -138,6 +139,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -187,11 +191,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -263,6 +266,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -321,6 +328,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -343,6 +353,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -367,6 +382,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -502,6 +520,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -553,7 +576,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: postStart: @@ -575,10 +598,10 @@ spec: # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). - if [[ "$(iptables-save | grep -c AWS-SNAT-CHAIN)" != "0" ]]; + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -v AWS-SNAT-CHAIN | iptables-restore + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' preStop: @@ -618,6 +641,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -629,12 +668,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -658,7 +702,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -675,7 +719,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -694,11 +738,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -715,10 +765,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -726,6 +783,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -741,14 +814,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -762,7 +849,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -839,6 +926,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -917,7 +1012,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -929,6 +1024,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content index b2a3952a2c756..8a50e103c9900 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content @@ -228,7 +228,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: vxlan - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content index 0790825e5a728..bbbbdf4cd7a69 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content @@ -99,7 +99,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 12325ce4b4f85d7aa094ccd86197641ff7aff6a90c32da34b64678aa9454a18e + manifestHash: 492810dae91d3d96f60f547fcb0b34c14b4a2d3d953171101cf3af8d4addff70 name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content index e34bb36cf7bb0..39a0faae6a569 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -62,6 +62,7 @@ data: kube-proxy-replacement: "false" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: tunnel @@ -136,6 +137,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -185,11 +189,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -261,6 +264,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -319,6 +326,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -341,6 +351,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -365,6 +380,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -501,6 +519,10 @@ spec: template: metadata: annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined test1: "true" test2: "123" test3: awesome @@ -555,7 +577,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -595,6 +617,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -606,12 +644,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -635,7 +678,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -652,7 +695,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -671,11 +714,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -692,10 +741,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -703,6 +759,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -718,14 +790,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -739,7 +825,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -816,6 +902,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -898,7 +992,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -910,6 +1004,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content index d6f9a3e3138fe..7009594ed5414 100644 --- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content @@ -225,7 +225,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: vxlan - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content index 4d3fc775dc122..6c9b00a8e2bf8 100644 --- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content @@ -155,7 +155,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 3fdb869ea26ce50ae6db32e1b997749f18cbb30ebf31468f2c5da2c692681a54 + manifestHash: 0a96b2e9786d0cc7e87eff42a6b38e011a45cb6c485825aaa491034e2c7d631b name: networking.cilium.io needsPKI: true needsRollingUpdate: all diff --git a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content index aae25ca1e121f..2e856a0b4d5da 100644 --- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -94,6 +94,7 @@ data: kube-proxy-replacement: "false" monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: tunnel @@ -217,6 +218,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -266,11 +270,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -342,6 +345,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -400,6 +407,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -422,6 +432,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -446,6 +461,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -754,6 +772,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -805,7 +828,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -853,6 +876,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -864,12 +903,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -896,7 +940,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -913,7 +957,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -932,11 +976,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -953,10 +1003,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -964,6 +1021,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -979,14 +1052,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -1000,7 +1087,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -1077,11 +1164,26 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel - name: hubble-tls projected: defaultMode: 256 sources: - secret: + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt name: hubble-server-certs optional: true updateStrategy: @@ -1162,7 +1264,7 @@ spec: value: api.internal.privatecilium.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -1174,6 +1276,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m @@ -1256,18 +1368,23 @@ spec: - serve command: - hubble-relay - image: quay.io/cilium/hubble-relay:v1.15.6 + image: quay.io/cilium/hubble-relay:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: - tcpSocket: - port: grpc + failureThreshold: 12 + grpc: + port: 4222 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 name: hubble-relay ports: - containerPort: 4245 name: grpc readinessProbe: - tcpSocket: - port: grpc + grpc: + port: 4222 + timeoutSeconds: 3 securityContext: capabilities: drop: @@ -1275,6 +1392,12 @@ spec: runAsGroup: 65532 runAsNonRoot: true runAsUser: 65532 + startupProbe: + failureThreshold: 20 + grpc: + port: 4222 + initialDelaySeconds: 10 + periodSeconds: 3 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/hubble-relay @@ -1288,7 +1411,7 @@ spec: fsGroup: 65532 serviceAccount: hubble-relay serviceAccountName: hubble-relay - terminationGracePeriodSeconds: 0 + terminationGracePeriodSeconds: 1 topologySpreadConstraints: - labelSelector: matchLabels: diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content index 857a3b9f74978..c37d24b281384 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content @@ -232,7 +232,7 @@ spec: sidecarIstioProxyImage: cilium/istio_proxy toFqdnsDnsRejectResponseCode: refused tunnel: disabled - version: v1.15.6 + version: v1.16.1 nodeTerminationHandler: cpuRequest: 50m deleteSQSMsgIfNodeNotFound: false diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content index 7d20c6f2a0033..f9de56478d775 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content @@ -99,7 +99,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: be09a607c2a87737bee2f1fbf38420f09ae2ff560e021fab080a98f3225f0c51 + manifestHash: 0fed3b36276ff3f87b1c01bbc1b81576a14fd45da3958df8947230afd410dbff name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content index bd390faf8555f..2f21d73c3d94d 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content @@ -75,6 +75,7 @@ data: kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}' monitor-aggregation: medium nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 preallocate-bpf-maps: "false" remove-cilium-node-taints: "true" routing-mode: native @@ -148,6 +149,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -197,11 +201,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch @@ -273,6 +276,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -331,6 +338,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -353,6 +363,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -377,6 +392,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -512,6 +530,11 @@ spec: kubernetes.io/cluster-service: "true" template: metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined creationTimestamp: null labels: app.kubernetes.io/name: cilium-agent @@ -563,7 +586,7 @@ spec: value: api.internal.privateciliumadvanced.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent lifecycle: postStart: @@ -585,10 +608,10 @@ spec: # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). - if [[ "$(iptables-save | grep -c AWS-SNAT-CHAIN)" != "0" ]]; + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -v AWS-SNAT-CHAIN | iptables-restore + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' preStop: @@ -628,6 +651,22 @@ spec: cpu: 25m memory: 128Mi securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL privileged: true startupProbe: failureThreshold: 105 @@ -639,12 +678,17 @@ spec: path: /healthz port: 9879 scheme: HTTP + initialDelaySeconds: 5 periodSeconds: 2 successThreshold: 1 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional + mountPropagation: HostToContainer name: bpf-maps - mountPath: /run/cilium/cgroupv2 name: cilium-cgroup @@ -674,7 +718,7 @@ spec: hostNetwork: true initContainers: - command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -691,7 +735,7 @@ spec: value: api.internal.privateciliumadvanced.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: config terminationMessagePolicy: FallbackToLogsOnError @@ -710,11 +754,17 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: mount-cgroup securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /hostproc @@ -731,10 +781,17 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -742,6 +799,22 @@ spec: name: hostproc - mountPath: /hostbin name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps - command: - /init-container.sh env: @@ -757,14 +830,28 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true - name: KUBERNETES_SERVICE_HOST value: api.internal.privateciliumadvanced.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -778,7 +865,7 @@ spec: name: cilium-run - command: - /install-plugin.sh - image: quay.io/cilium/cilium:v1.15.6 + image: quay.io/cilium/cilium:v1.16.1 imagePullPolicy: IfNotPresent name: install-cni-binaries resources: @@ -866,6 +953,14 @@ spec: - configMap: name: cilium-config name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel updateStrategy: type: OnDelete @@ -944,7 +1039,7 @@ spec: value: api.internal.privateciliumadvanced.example.com - name: KUBERNETES_SERVICE_PORT value: "443" - image: quay.io/cilium/operator:v1.15.6 + image: quay.io/cilium/operator:v1.16.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -956,6 +1051,16 @@ spec: periodSeconds: 10 timeoutSeconds: 3 name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 resources: requests: cpu: 25m diff --git a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/values.yaml b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/values.yaml new file mode 100644 index 0000000000000..57f56c9c6a8e7 --- /dev/null +++ b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/values.yaml @@ -0,0 +1,18 @@ +topologySpreadConstraints: + - labelSelector: + matchLabels: + app.kubernetes.io/name: '{{ template "coredns.name" . }}' + app.kubernetes.io/instance: '{{ .Release.Name }}' + topologyKey: topology.kubernetes.io/zone + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app.kubernetes.io/name: '{{ template "coredns.name" . }}' + app.kubernetes.io/instance: '{{ .Release.Name }}' + topologyKey: kubernetes.io/hostname + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + +autoscaler: + enabled: true diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/helm-values.yaml b/upup/models/cloudup/resources/addons/networking.cilium.io/helm-values.yaml new file mode 100644 index 0000000000000..b44d824945437 --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/helm-values.yaml @@ -0,0 +1,33 @@ +# This file is only used to help generate the .yaml.template file +hubble: + metrics: + enabled: [drop] + relay: + enabled: true +ingressController: + enabled: true + secretsNamespace: + create: false +serviceAccounts: + envoy: + create: false +envoy: + enabled: false +envoyConfig: + secretsNamespace: + create: false +gatewayAPI: + secretsNamespace: + create: false +bgpControlPlane: + secretsNamespace: + create: false +updateStrategy: + type: OnDelete + rollingUpdate: null +monitor: + enabled: true +ipv4: + enabled: false +ipv6: + enabled: true \ No newline at end of file diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.15.yaml.template index 08dd28f961eac..8eafffbd4195e 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.15.yaml.template @@ -1,6 +1,9 @@ +# helm template --release-name cilium cilium/cilium \ +# --version 1.16.1 \ +# --namespace kube-system \ +# --values helm-values.yaml {{ with .Networking.Cilium }} {{ $semver := (trimPrefix "v" .Version) }} -{{ $healthPort := (ternary 9879 9876 (semverCompare ">=1.11.6" $semver)) }} {{ $operatorHealthPort := 9234 }} {{- if CiliumSecret }} apiVersion: v1 @@ -39,7 +42,7 @@ metadata: name: cilium-config namespace: kube-system data: - agent-health-port: "{{ $healthPort }}" + agent-health-port: "9879" {{- if .EtcdManaged }} kvstore: etcd @@ -224,10 +227,6 @@ data: # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action tofqdns-enable-poller: "{{- if .ToFQDNsEnablePoller -}}true{{- else -}}false{{- end -}}" - {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }} - # wait-bpf-mount makes init container wait until bpf filesystem is mounted - wait-bpf-mount: "false" - {{- end }} # Enable fetching of container-runtime specific metadata # # By default, the Kubernetes pod and namespace labels are retrieved and @@ -257,6 +256,7 @@ data: enable-node-port: "{{ .EnableNodePort }}" kube-proxy-replacement: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}" + operator-api-serve-addr: "{{- if IsIPv6Only -}}[::1]{{- else -}}127.0.0.1{{- end -}}:9234" {{ with .IPAM }} ipam: {{ . }} {{ if eq . "eni" }} @@ -429,6 +429,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -479,11 +482,10 @@ rules: - apiGroups: - cilium.io resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch --- @@ -556,6 +558,10 @@ rules: - get - list - watch + - create + - update + - delete + - patch - apiGroups: - cilium.io resources: @@ -620,6 +626,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -646,6 +655,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -666,6 +680,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -959,6 +976,10 @@ spec: prometheus.io/scrape: "true" prometheus.io/port: "{{ .AgentPrometheusPort }}" {{ end }} + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" {{- range $key, $value := .AgentPodAnnotations }} {{ $key }}: "{{ $value }}" {{- end }} @@ -989,7 +1010,7 @@ spec: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz - port: {{ $healthPort }} + port: 9879 scheme: HTTP httpHeaders: - name: "brief" @@ -997,11 +1018,12 @@ spec: failureThreshold: 105 periodSeconds: 2 successThreshold: 1 + initialDelaySeconds: 5 livenessProbe: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz - port: {{ $healthPort }} + port: 9879 scheme: HTTP httpHeaders: - name: "brief" @@ -1018,7 +1040,7 @@ spec: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz - port: {{ $healthPort }} + port: 9879 scheme: HTTP httpHeaders: - name: "brief" @@ -1081,10 +1103,10 @@ spec: # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). - if [[ "$(iptables-save | grep -c AWS-SNAT-CHAIN)" != "0" ]]; + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -v AWS-SNAT-CHAIN | iptables-restore + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' {{- end }} @@ -1113,20 +1135,42 @@ spec: terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel - name: bpf-maps mountPath: /sys/fs/bpf - {{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }} - mountPropagation: Bidirectional - {{- end }} + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer - name: cilium-cgroup mountPath: /run/cilium/cgroupv2 - name: cilium-run mountPath: /var/run/cilium - {{- if not (semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver) }} - - name: cni-path - mountPath: /host/opt/cni/bin - {{- end }} - name: etc-cni-netd mountPath: /host/etc/cni/net.d {{ if .EtcdManaged }} @@ -1173,7 +1217,7 @@ spec: for i in {1..5}; do \ [ -S /var/run/cilium/monitor1_2.sock ] && break || sleep 10;\ done; \ - cilium monitor --type=agent + cilium-dbg monitor terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cilium-run @@ -1184,7 +1228,7 @@ spec: image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent command: - - cilium + - cilium-dbg - build-config env: - name: K8S_NODE_NAME @@ -1234,7 +1278,13 @@ spec: mountPath: /hostbin terminationMessagePolicy: FallbackToLogsOnError securityContext: - privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL - name: apply-sysctl-overwrites image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent @@ -1261,6 +1311,32 @@ spec: terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.16.1@sha256:0b4a3ab41a4760d86b7fc945b8783747ba27f29dac30dd434d94f2c9e3679f39" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional - name: clean-cilium-state image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" imagePullPolicy: IfNotPresent @@ -1279,34 +1355,37 @@ spec: name: cilium-config key: clean-cilium-bpf-state optional: true - - name: KUBERNETES_SERVICE_HOST - value: "{{ APIInternalName }}" - - name: KUBERNETES_SERVICE_PORT - value: "443" - {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }} - - name: CILIUM_WAIT_BPF_MOUNT + - name: WRITE_CNI_CONF_WHEN_READY valueFrom: configMapKeyRef: - key: wait-bpf-mount name: cilium-config + key: write-cni-conf-when-ready optional: true - {{- end }} + - name: KUBERNETES_SERVICE_HOST + value: "{{ APIInternalName }}" + - name: KUBERNETES_SERVICE_PORT + value: "443" terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf - {{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }} mountPropagation: HostToContainer - {{- end }} # Required to mount cgroup filesystem from the host to cilium agent pod - name: cilium-cgroup mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer - name: cilium-run mountPath: /var/run/cilium - {{- if semverCompare "~1.11.15 || ~1.12.8 || >=1.13.1" $semver }} # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries image: "{{ or .Registry "quay.io" }}/cilium/cilium:{{ .Version }}" @@ -1320,13 +1399,12 @@ spec: securityContext: capabilities: drop: - - ALL + - ALL terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cni-path mountPath: /host/opt/cni/bin - {{- end }} restartPolicy: Always priorityClassName: system-node-critical {{ if ContainerdSELinuxEnabled }} @@ -1430,6 +1508,14 @@ spec: secret: secretName: cilium-ipsec-keys {{ end }} + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory {{ if WithDefaultBool .Hubble.Enabled false }} - name: hubble-tls projected: @@ -1439,6 +1525,13 @@ spec: - secret: name: hubble-server-certs optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt {{ end }} --- apiVersion: apps/v1 @@ -1518,9 +1611,9 @@ spec: value: "443" {{ if .EnablePrometheusMetrics }} ports: - - containerPort: 6942 - hostPort: 6942 - name: prometheus + - name: prometheus + containerPort: 9963 + hostPort: 9963 protocol: TCP {{ end }} resources: @@ -1531,11 +1624,21 @@ spec: httpGet: host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' path: /healthz - port: {{ $operatorHealthPort }} + port: 9234 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 + readinessProbe: + httpGet: + host: '{{- if IsIPv6Only -}}::1{{- else -}}127.0.0.1{{- end -}}' + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /tmp/cilium/config-map @@ -1649,11 +1752,22 @@ spec: - name: grpc containerPort: 4245 readinessProbe: - tcpSocket: - port: grpc + grpc: + port: 4222 + timeoutSeconds: 3 livenessProbe: - tcpSocket: - port: grpc + grpc: + port: 4222 + timeoutSeconds: 10 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 12 + startupProbe: + grpc: + port: 4222 + initialDelaySeconds: 10 + failureThreshold: 20 + periodSeconds: 3 terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: config @@ -1672,7 +1786,7 @@ spec: restartPolicy: Always serviceAccount: hubble-relay serviceAccountName: hubble-relay - terminationGracePeriodSeconds: 0 + terminationGracePeriodSeconds: 1 topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 06763e1cb65a8..01591cc75902d 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -99,7 +99,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 3cd28effb6499670f52244fa0fe1814c2a6921a3e7eaac43b0064dab804127d7 + manifestHash: be919b9d3124ee841a8f46a8309b8ec689715bd651bc44f8cebc3717eafd019f name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml index 1cae7bea1ec1a..55f655730a0e2 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml @@ -113,7 +113,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 3cd28effb6499670f52244fa0fe1814c2a6921a3e7eaac43b0064dab804127d7 + manifestHash: be919b9d3124ee841a8f46a8309b8ec689715bd651bc44f8cebc3717eafd019f name: networking.cilium.io needsRollingUpdate: all selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml index ae8331474793e..5831a60946949 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml @@ -170,7 +170,7 @@ spec: version: 9.99.0 - id: k8s-1.16 manifest: networking.cilium.io/k8s-1.16-v1.15.yaml - manifestHash: 3cd28effb6499670f52244fa0fe1814c2a6921a3e7eaac43b0064dab804127d7 + manifestHash: be919b9d3124ee841a8f46a8309b8ec689715bd651bc44f8cebc3717eafd019f name: networking.cilium.io needsRollingUpdate: all selector: