diff --git a/kubernetes/main/talos/k8s-0.sops.yaml.j2 b/kubernetes/main/talos/k8s-0.sops.yaml.j2 index 2f847c9a165b..b5908dc6a8ef 100644 --- a/kubernetes/main/talos/k8s-0.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-0.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -41,12 +38,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -54,27 +55,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -92,7 +96,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt @@ -152,6 +156,19 @@ cluster: image: registry.k8s.io/kube-scheduler:{{ ENV.KUBERNETES_VERSION }} extraArgs: bind-address: 0.0.0.0 + config: + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + profiles: + - schedulerName: default-scheduler + pluginConfig: + - name: PodTopologySpread + args: + defaultingType: List + defaultConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway discovery: enabled: true registries: diff --git a/kubernetes/main/talos/k8s-1.sops.yaml.j2 b/kubernetes/main/talos/k8s-1.sops.yaml.j2 index 8573c1b01f1a..5fa1c3561511 100644 --- a/kubernetes/main/talos/k8s-1.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-1.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -41,12 +38,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -54,27 +55,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -92,7 +96,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt @@ -152,6 +156,19 @@ cluster: image: registry.k8s.io/kube-scheduler:{{ ENV.KUBERNETES_VERSION }} extraArgs: bind-address: 0.0.0.0 + config: + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + profiles: + - schedulerName: default-scheduler + pluginConfig: + - name: PodTopologySpread + args: + defaultingType: List + defaultConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway discovery: enabled: true registries: diff --git a/kubernetes/main/talos/k8s-2.sops.yaml.j2 b/kubernetes/main/talos/k8s-2.sops.yaml.j2 index 7accc8c5282f..f6a386eda4aa 100644 --- a/kubernetes/main/talos/k8s-2.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-2.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -41,12 +38,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -54,27 +55,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -92,7 +96,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt @@ -152,6 +156,19 @@ cluster: image: registry.k8s.io/kube-scheduler:{{ ENV.KUBERNETES_VERSION }} extraArgs: bind-address: 0.0.0.0 + config: + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + profiles: + - schedulerName: default-scheduler + pluginConfig: + - name: PodTopologySpread + args: + defaultingType: List + defaultConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway discovery: enabled: true registries: diff --git a/kubernetes/main/talos/k8s-3.sops.yaml.j2 b/kubernetes/main/talos/k8s-3.sops.yaml.j2 index 7afb33002fde..7c91215cf636 100644 --- a/kubernetes/main/talos/k8s-3.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-3.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -39,12 +36,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -52,27 +53,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -84,7 +88,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt diff --git a/kubernetes/main/talos/k8s-4.sops.yaml.j2 b/kubernetes/main/talos/k8s-4.sops.yaml.j2 index 28245025c039..d73ce85de63e 100644 --- a/kubernetes/main/talos/k8s-4.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-4.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -39,12 +36,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -52,27 +53,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -84,7 +88,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt diff --git a/kubernetes/main/talos/k8s-5.sops.yaml.j2 b/kubernetes/main/talos/k8s-5.sops.yaml.j2 index 510271d83be0..28f466cfd02b 100644 --- a/kubernetes/main/talos/k8s-5.sops.yaml.j2 +++ b/kubernetes/main/talos/k8s-5.sops.yaml.j2 @@ -18,10 +18,7 @@ machine: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw + options: ["bind", "rshared", "rw"] defaultRuntimeSeccompProfileEnabled: true nodeIP: validSubnets: @@ -39,12 +36,16 @@ machine: diskSelector: model: Samsung SSD 870 extraKernelArgs: + - cpufreq.default_governor=performance + - intel_pstate=disable - mitigations=off - module_blacklist=e1000e image: factory.talos.dev/installer/d715f723f882b1e1e8063f1b89f237dcc0e3bd000f9f970243af59c8baae0100:{{ ENV.TALOS_VERSION }} wipe: false files: - - content: | + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -52,27 +53,30 @@ machine: discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + - op: overwrite + path: /etc/nfsmount.conf + permissions: 0o644 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True noatime=True - nodiratime=True rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" + fs.inotify.max_user_watches: 1048576 + fs.inotify.max_user_instances: 8192 + net.core.netdev_max_backlog: 30000 + net.core.rmem_max: 67108864 + net.core.wmem_max: 67108864 + net.ipv4.tcp_rmem: 4096 87380 33554432 + net.ipv4.tcp_wmem: 4096 65536 33554432 + net.ipv4.tcp_tw_reuse: 1 + net.ipv4.tcp_window_scaling: 1 + sysfs: + block.nvme0n1.queue.scheduler: none + devices.system.cpu.cpufreq.boost: 0 features: rbac: true stableHostname: true @@ -84,7 +88,7 @@ machine: hostDNS: enabled: true resolveMemberNames: true - forwardKubeDNSToHost: false + forwardKubeDNSToHost: false # Incompatible with Cilium bpf masquerade udev: rules: # Thunderbolt