From b96725cb8d59045a2be3c2e75691ee3a237f9bd5 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Wed, 11 Mar 2020 17:26:56 -0400 Subject: [PATCH 01/49] #cncf/cnf-conformance/issues/56 sample_setup, sample_setup_args, and sample_cleanup now generically setup a sample cnf --- .../sample-coredns-cnf/cnf-conformance.yml | 2 + spec/cnf_conformance_spec.cr | 14 ++- spec/sample_coredns_setup_spec.cr | 53 ++++++++- spec/security_spec.cr | 8 +- src/tasks/cleanup.cr | 10 +- src/tasks/setup.cr | 2 +- src/tasks/utils.cr | 104 ++++++++++++++++++ 7 files changed, 180 insertions(+), 13 deletions(-) diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index 61f5dfc81..ea4dc23e6 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -1,6 +1,8 @@ --- helm_directory: cnfs/coredns/helm_chart/coredns +git_clone_url: https://github.com/coredns/coredns.git install_script: cnfs/coredns/Makefile +release_name: coredns deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_spec.cr index 94afdd22c..f8dc39a0a 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_spec.cr @@ -5,18 +5,24 @@ describe CnfConformance do before_all do # puts `pwd` # puts `echo $KUBECONFIG` - `crystal src/cnf-conformance.cr cleanup` + `crystal src/cnf-conformance.cr samples_cleanup` $?.success?.should be_true `crystal src/cnf-conformance.cr setup` $?.success?.should be_true - # Helm chart deploys take a while to spin up - # TODO put sleep in setup installs - # sleep 15 + end + after_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true end + it "'all' should run the whole test suite" do # puts `pwd` # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true response_s = `crystal src/cnf-conformance.cr all verbose` puts response_s $?.success?.should be_true diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 830570373..60c78b703 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -2,6 +2,7 @@ require "./spec_helper" require "colorize" require "../src/tasks/utils.cr" require "file_utils" +require "sam" describe "Utils" do before_all do @@ -9,10 +10,29 @@ describe "Utils" do # puts `echo $KUBECONFIG` `crystal src/cnf-conformance.cr helm_local_install` $?.success?.should be_true - `crystal src/cnf-conformance.cr sample_coredns_cleanup` + end + + after_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + end + + before_each do + `crystal src/cnf-conformance.cr cleanup` + $?.success?.should be_true + end + + after_each do + `crystal src/cnf-conformance.cr cleanup` $?.success?.should be_true end + it "'wait_for_install' should wait for a cnf to be installed" do + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + current_dir = FileUtils.pwd puts current_dir helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" @@ -23,4 +43,35 @@ describe "Utils" do current_replicas = `kubectl get deployments coredns-coredns -o=jsonpath='{.status.readyReplicas}'` (current_replicas.to_i > 0).should be_true end + + it "'sample_setup' should set up a sample cnf" do + args = Sam::Args.new + sample_setup(sample_dir: "sample-cnfs/sample-coredns-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "cnfs/coredns/helm_chart/coredns", git_clone_url: "https://github.com/coredns/coredns.git" ) + # check if directory exists + (Dir.exists? "sample-cnfs/sample-coredns-cnf").should be_true + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_true + end + + it "'sample_setup_args' should set up a sample cnf from a argument", tags: "WIP" do + args = Sam::Args.new + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) + # check if directory exists + (Dir.exists? "sample-cnfs/sample-coredns-cnf").should be_true + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_true + end + + it "'sample_cleanup' should clean up a sample cnf from a argument", tags: "WIP" do + args = Sam::Args.new + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) + sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) + # check if directory exists + (Dir.exists? "cnfs/coredns").should be_false + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_false + (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_false + (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_false + end end diff --git a/spec/security_spec.cr b/spec/security_spec.cr index 1c0d383f6..85f8971e8 100644 --- a/spec/security_spec.cr +++ b/spec/security_spec.cr @@ -5,14 +5,10 @@ describe CnfConformance do before_all do # puts `pwd` # puts `echo $KUBECONFIG` - # `crystal src/cnf-conformance.cr cleanup` - # $?.success?.should be_true + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true # `crystal src/cnf-conformance.cr setup` # $?.success?.should be_true - # Helm chart deploys take a while to spin up - # TODO put sleep in setup installs - # sleep 15 - end it "'privileged' should pass with a non-privileged cnf", tags: "privileged" do begin diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index 098fa10b5..f2d922e21 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -4,6 +4,14 @@ require "colorize" require "totem" desc "Cleans up the CNF Conformance test suite, the K8s cluster, and upstream projects" -task "cleanup", ["sample_coredns_cleanup", "bad_helm_cnf_cleanup", "helm_local_cleanup"] do |_, args| +task "cleanup", ["samples_cleanup"] do |_, args| end +task "samples_cleanup", ["sample_coredns_cleanup", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup"] do |_, args| +end + +task "tools_cleanup", ["helm_local_cleanup"] do |_, args| +end + +task "cleanup_all", ["cleanup_samples", "tools_cleanup"] do |_, args| +end diff --git a/src/tasks/setup.cr b/src/tasks/setup.cr index b5d23f1af..b4440dd41 100644 --- a/src/tasks/setup.cr +++ b/src/tasks/setup.cr @@ -4,6 +4,6 @@ require "colorize" require "totem" desc "Sets up the CNF Conformance test suite, the K8s cluster, and upstream projects" -task "setup", ["install_opa" , "install_api_snoop", "install_sonobuoy", "install_chart_testing", "sample_coredns_setup", "helm_local_install"] do |_, args| +task "setup", ["install_opa" , "install_api_snoop", "install_sonobuoy", "install_chart_testing", "helm_local_install"] do |_, args| end diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index ebb6f31a4..bbf54f666 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -1,3 +1,4 @@ +require "totem" # TODO make constants local or always retrieve from environment variables # TODO Move constants out CNF_DIR = "cnfs" @@ -24,6 +25,14 @@ def cnf_conformance_yml Totem.from_file "./#{cnf_conformance}" end +def sample_conformance_yml(sample_dir) + cnf_conformance = `find #{sample_dir}/* -name "cnf-conformance.yml"`.split("\n")[0] + if cnf_conformance.empty? + raise "No cnf_conformance.yml found in #{sample_dir}!" + end + Totem.from_file "./#{cnf_conformance}" +end + def wait_for_install(deployment_name, wait_count=180) second_count = 0 current_replicas = `kubectl get deployments #{deployment_name} -o=jsonpath='{.status.readyReplicas}'` @@ -38,3 +47,98 @@ def wait_for_install(deployment_name, wait_count=180) second_count = second_count + 1 end end +def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) + # # Parse the cnf-conformance.yml + config = sample_conformance_yml(sample_dir) + + if args.named.keys.includes? "release_name" + release_name = args.named["release_name"] + else + release_name = config.get("release_name").as_s + end + puts "release_name: #{release_name}" if verbose + + if args.named.keys.includes? "deployment_name" + deployment_name = args.named["deployment_name"] + else + deployment_name = config.get("deployment_name").as_s + end + puts "deployment_name: #{deployment_name}" if verbose + + if args.named.keys.includes? "helm_chart" + helm_chart = args.named["helm_chart"] + else + helm_chart = config.get("helm_chart").as_s + end + puts "helm_chart: #{helm_chart}" if verbose + + if args.named.keys.includes? "helm_directory" + helm_directory = args.named["helm_directory"] + else + helm_directory = config.get("helm_directory").as_s + end + puts "helm_directory: #{helm_directory}" if verbose + + if args.named.keys.includes? "git_clone_url" + git_clone_url = args.named["git_clone_url"] + else + git_clone_url = config.get("git_clone_url").as_s + end + puts "git_clone_url: #{git_clone_url}" if verbose + + sample_setup(sample_dir: sample_dir, release_name: release_name, deployment_name: deployment_name, helm_chart: helm_chart, helm_directory: helm_directory, git_clone_url: git_clone_url, deploy_with_chart: deploy_with_chart, verbose: verbose ) + +end + +def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url, deploy_with_chart=true, verbose=false) + + current_dir = FileUtils.pwd + puts current_dir if verbose + + # TODO enable recloning/fetching etc + # TODO pass in block + git_clone = `git clone #{git_clone_url} #{current_dir}/#{CNF_DIR}/#{release_name}` + puts git_clone if verbose + + # Copy the cnf-conformance.yml + yml_cp = `cp #{sample_dir}/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/#{release_name}` + puts yml_cp if verbose + + begin + + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if verbose + helm_install = `#{helm} install #{release_name} #{helm_chart}` + puts helm_install if verbose + + + # Retrieve the helm chart source + FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart") + helm_pull = `#{helm} pull #{helm_chart}` + puts helm_pull if verbose + core_mv = `mv #{release_name}-*.tgz #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart` + puts core_mv if verbose + tar = `cd #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart/#{release_name}-*.tgz` + puts tar if verbose + wait_for_install(deployment_name) + if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 + puts "Successfully setup #{release_name}".colorize(:green) + end + ensure + cd = `cd #{current_dir}` + puts cd if verbose + end +end + +def sample_cleanup(sample_dir, verbose=true) + config = sample_conformance_yml(sample_dir) + release_name = config.get("release_name").as_s + + current_dir = FileUtils.pwd + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if verbose + helm_uninstall = `#{helm} uninstall #{release_name}` + puts helm_uninstall if verbose + rm = `rm -rf #{current_dir}/#{CNF_DIR}/#{release_name}` + puts rm if verbose +end From ef0eb0927ab81e0cadec06fc9e0c6043d3759a61 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 11 Mar 2020 19:03:31 -0400 Subject: [PATCH 02/49] Add spec tests for livenessProbe cncf/cnf-conformance#56 --- sample-cnfs/sample_coredns/README.md | 39 ++++ sample-cnfs/sample_coredns/chart/.helmignore | 22 ++ sample-cnfs/sample_coredns/chart/Chart.yaml | 23 ++ sample-cnfs/sample_coredns/chart/README.md | 138 ++++++++++++ .../sample_coredns/chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 122 +++++++++++ .../chart/templates/poddisruptionbudget.yaml | 28 +++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ sample-cnfs/sample_coredns/chart/values.yaml | 198 ++++++++++++++++++ .../sample_coredns/cnf-conformance.yml | 8 + .../sample_coredns_bad_liveness/README.md | 39 ++++ .../chart/.helmignore | 22 ++ .../chart/Chart.yaml | 23 ++ .../chart/README.md | 138 ++++++++++++ .../chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 113 ++++++++++ .../chart/templates/poddisruptionbudget.yaml | 28 +++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ .../chart/values.yaml | 198 ++++++++++++++++++ .../cnf-conformance.yml | 8 + spec/configuration_lifecycle_spec.cr | 39 ++++ src/tasks/sample_coredns_setup.cr | 127 ++++++++++- 48 files changed, 2589 insertions(+), 10 deletions(-) create mode 100644 sample-cnfs/sample_coredns/README.md create mode 100755 sample-cnfs/sample_coredns/chart/.helmignore create mode 100755 sample-cnfs/sample_coredns/chart/Chart.yaml create mode 100755 sample-cnfs/sample_coredns/chart/README.md create mode 100755 sample-cnfs/sample_coredns/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample_coredns/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample_coredns/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/poddisruptionbudget.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/service.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample_coredns/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample_coredns/chart/values.yaml create mode 100644 sample-cnfs/sample_coredns/cnf-conformance.yml create mode 100644 sample-cnfs/sample_coredns_bad_liveness/README.md create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/.helmignore create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/Chart.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/README.md create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/poddisruptionbudget.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/service.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample_coredns_bad_liveness/chart/values.yaml create mode 100644 sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml create mode 100644 spec/configuration_lifecycle_spec.cr diff --git a/sample-cnfs/sample_coredns/README.md b/sample-cnfs/sample_coredns/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample_coredns/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample_coredns/chart/.helmignore b/sample-cnfs/sample_coredns/chart/.helmignore new file mode 100755 index 000000000..7c04072e1 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +OWNERS diff --git a/sample-cnfs/sample_coredns/chart/Chart.yaml b/sample-cnfs/sample_coredns/chart/Chart.yaml new file mode 100755 index 000000000..862d36cde --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.10.0 diff --git a/sample-cnfs/sample_coredns/chart/README.md b/sample-cnfs/sample_coredns/chart/README.md new file mode 100755 index 000000000..b4fbbc91b --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/README.md @@ -0,0 +1,138 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `podDisruptionBudget` | Optional PodDisruptionBudget | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample_coredns/chart/templates/NOTES.txt b/sample-cnfs/sample_coredns/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample_coredns/chart/templates/_helpers.tpl b/sample-cnfs/sample_coredns/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample_coredns/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_coredns/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/clusterrole.yaml b/sample-cnfs/sample_coredns/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_coredns/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/configmap.yaml b/sample-cnfs/sample_coredns/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_coredns/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/deployment.yaml b/sample-cnfs/sample_coredns/chart/templates/deployment.yaml new file mode 100755 index 000000000..11db35a0f --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/deployment.yaml @@ -0,0 +1,122 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/poddisruptionbudget.yaml b/sample-cnfs/sample_coredns/chart/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..8ade224f8 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/poddisruptionbudget.yaml @@ -0,0 +1,28 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_coredns/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/service-metrics.yaml b/sample-cnfs/sample_coredns/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/service.yaml b/sample-cnfs/sample_coredns/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample_coredns/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_coredns/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_coredns/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_coredns/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample_coredns/chart/values.yaml b/sample-cnfs/sample_coredns/chart/values.yaml new file mode 100755 index 000000000..d7e074d93 --- /dev/null +++ b/sample-cnfs/sample_coredns/chart/values.yaml @@ -0,0 +1,198 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget +podDisruptionBudget: {} + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml new file mode 100644 index 000000000..b3d011943 --- /dev/null +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -0,0 +1,8 @@ +--- +helm_directory: cnfs/sample_coredns/chart +install_script: cnfs/sample_coredns/chart +deployment_name: coredns-coredns +application_deployment_names: [coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: coredns-coredns +white_list_helm_chart_container_names: [] diff --git a/sample-cnfs/sample_coredns_bad_liveness/README.md b/sample-cnfs/sample_coredns_bad_liveness/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/.helmignore b/sample-cnfs/sample_coredns_bad_liveness/chart/.helmignore new file mode 100755 index 000000000..7c04072e1 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +OWNERS diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/Chart.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/Chart.yaml new file mode 100755 index 000000000..862d36cde --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.10.0 diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/README.md b/sample-cnfs/sample_coredns_bad_liveness/chart/README.md new file mode 100755 index 000000000..b4fbbc91b --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/README.md @@ -0,0 +1,138 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `podDisruptionBudget` | Optional PodDisruptionBudget | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/NOTES.txt b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/_helpers.tpl b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml new file mode 100755 index 000000000..fd1d89aa8 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/poddisruptionbudget.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..8ade224f8 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/poddisruptionbudget.yaml @@ -0,0 +1,28 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service-metrics.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/values.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/values.yaml new file mode 100755 index 000000000..d7e074d93 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/values.yaml @@ -0,0 +1,198 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget +podDisruptionBudget: {} + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml new file mode 100644 index 000000000..88fb77cc6 --- /dev/null +++ b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml @@ -0,0 +1,8 @@ +--- +helm_directory: cnfs/sample_coredns_bad_liveness/chart +install_script: cnfs/sample_coredns_bad_liveness/chart +deployment_name: bad-liveness-coredns +application_deployment_names: [bad-liveness-coredns] +helm_chart: stable/coredns +helm_chart_container_name: bad-liveness-coredns +white_list_helm_chart_container_names: [] diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr new file mode 100644 index 000000000..90d81256c --- /dev/null +++ b/spec/configuration_lifecycle_spec.cr @@ -0,0 +1,39 @@ +require "./spec_helper" +require "colorize" + +describe CnfConformance do + before_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + + # `crystal src/cnf-conformance.cr samples_cleanup` + # $?.success?.should be_true + + # `crystal src/cnf-conformance.cr setup` + # $?.success?.should be_true + end + it "'liveness' should pass when livenessProbe is set", tags: "liveness" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr liveness verbose` + puts response_s + $?.success?.should be_true + (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns` + end + end + it "'liveness' should fail when livenessProbe is not set", tags: "liveness" do + begin + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr liveness verbose` + puts response_s + $?.success?.should be_true + (/FAILURE: No livenessProbe found/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` + end + end +end diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 2a04a9624..272b401d9 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -67,6 +67,56 @@ task "sample_coredns_setup", ["helm_local_install"] do |_, args| end end +task "sample_coredns", ["helm_local_install"] do |_, args| + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + + # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml + chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns #{current_dir}/#{CNF_DIR}/` + puts chart_cp if check_verbose(args) + + # Parse the cnf-conformance.yml + config = cnf_conformance_yml + + if args.named.keys.includes? "deployment_name" + deployment_name = args.named["deployment_name"] + else + deployment_name = config.get("deployment_name").as_s + end + puts "deployment_name: #{deployment_name}" if check_verbose(args) + + if args.named.keys.includes? "helm_chart" + helm_chart = args.named["helm_chart"] + else + helm_chart = config.get("helm_chart").as_s + end + puts "helm_chart: #{helm_chart}" if check_verbose(args) + + if args.named.keys.includes? "helm_directory" + helm_directory = args.named["helm_directory"] + else + helm_directory = config.get("helm_directory").as_s + end + puts "helm_directory: #{helm_directory}" if check_verbose(args) + + begin + + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if check_verbose(args) + helm_install = `#{helm} install coredns #{helm_directory}` + puts helm_install if check_verbose(args) + + wait_for_install(deployment_name) + if helm_install.to_s.size > 0 + puts "Successfully setup sample_coredns".colorize(:green) + end + ensure + cd = `cd #{current_dir}` + puts cd if check_verbose(args) + end +end + + task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| current_dir = FileUtils.pwd puts current_dir if check_verbose(args) @@ -187,18 +237,13 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ end end -task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| +task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| current_dir = FileUtils.pwd puts current_dir if check_verbose(args) - # Retrieve the cnf source - # TODO enable recloning/fetching etc - # git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/coredns` - # puts git_clone if check_verbose(args) - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` + yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` puts chart_cp if check_verbose(args) puts yml_mv if check_verbose(args) @@ -243,6 +288,56 @@ task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, ar end end +task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + + # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml + chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns_bad_liveness #{current_dir}/#{CNF_DIR}/` + puts chart_cp if check_verbose(args) + + # Parse the cnf-conformance.yml + config = cnf_conformance_yml + + if args.named.keys.includes? "deployment_name" + deployment_name = args.named["deployment_name"] + else + deployment_name = config.get("deployment_name").as_s + end + puts "deployment_name: #{deployment_name}" if check_verbose(args) + + if args.named.keys.includes? "helm_chart" + helm_chart = args.named["helm_chart"] + else + helm_chart = config.get("helm_chart").as_s + end + puts "helm_chart: #{helm_chart}" if check_verbose(args) + + if args.named.keys.includes? "helm_directory" + helm_directory = args.named["helm_directory"] + else + helm_directory = config.get("helm_directory").as_s + end + puts "helm_directory: #{helm_directory}" if check_verbose(args) + + begin + + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if check_verbose(args) + helm_install = `#{helm} install bad-liveness #{helm_directory}` + puts helm_install if check_verbose(args) + + wait_for_install(deployment_name) + if helm_install.to_s.size > 0 + puts "Successfully setup sample_coredns_bad_liveness".colorize(:green) + end + ensure + cd = `cd #{current_dir}` + puts cd if check_verbose(args) + end +end + + task "sample_coredns_cleanup" do |_, args| current_dir = FileUtils.pwd helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" @@ -253,6 +348,18 @@ task "sample_coredns_cleanup" do |_, args| puts rm if check_verbose(args) end + +task "cleanup_sample_coredns" do |_, args| + current_dir = FileUtils.pwd + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if check_verbose(args) + helm_uninstall = `#{helm} uninstall coredns` + puts helm_uninstall if check_verbose(args) + rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns` + puts rm if check_verbose(args) +end + + task "bad_helm_cnf_cleanup" do |_, args| current_dir = FileUtils.pwd helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" @@ -274,13 +381,13 @@ task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| puts rm if check_verbose(args) end -task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| +task "sample_coredns_bad_liveness_cleanup" do |_, args| current_dir = FileUtils.pwd helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall privileged-coredns` + helm_uninstall = `#{helm} uninstall bad-liveness` puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` + rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns_bad_liveness` puts rm if check_verbose(args) end From 82a61920a9576fc71b1a81a292b723e8cb803055 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 11 Mar 2020 19:59:28 -0400 Subject: [PATCH 03/49] Fix bug for liveness test cncf/cnf-conformance#56 --- spec/configuration_lifecycle_spec.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index 90d81256c..3cc122f80 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -6,8 +6,8 @@ describe CnfConformance do # puts `pwd` # puts `echo $KUBECONFIG` - # `crystal src/cnf-conformance.cr samples_cleanup` - # $?.success?.should be_true + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true # `crystal src/cnf-conformance.cr setup` # $?.success?.should be_true From 2259eb68df56750b4d79c5e3296b5839da6fb986 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 11 Mar 2020 20:01:21 -0400 Subject: [PATCH 04/49] Fix incorrect puts message for deploy & missing privliged sample cnf cncf/cnf-conformance#55 --- src/tasks/sample_coredns_setup.cr | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 272b401d9..358eb3dc7 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -181,18 +181,13 @@ task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| end end -task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| +task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| current_dir = FileUtils.pwd puts current_dir if check_verbose(args) - # Retrieve the cnf source - # TODO enable recloning/fetching etc - # git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/coredns` - # puts git_clone if check_verbose(args) - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` + yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` puts chart_cp if check_verbose(args) puts yml_mv if check_verbose(args) @@ -229,7 +224,7 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ wait_for_install(deployment_name) if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_non_whitelisted".colorize(:green) + puts "Successfully setup sample_privileged_cnf_whitelisted_setup".colorize(:green) end ensure cd = `cd #{current_dir}` @@ -237,6 +232,7 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ end end + task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| current_dir = FileUtils.pwd puts current_dir if check_verbose(args) @@ -280,7 +276,7 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ wait_for_install(deployment_name) if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_non_whitelisted".colorize(:green) + puts "Successfully setup sample_privileged_cnf_non_whitelisted_setup".colorize(:green) end ensure cd = `cd #{current_dir}` @@ -370,6 +366,15 @@ task "bad_helm_cnf_cleanup" do |_, args| puts rm if check_verbose(args) end +task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| + current_dir = FileUtils.pwd + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + puts helm if check_verbose(args) + helm_uninstall = `#{helm} uninstall privileged-coredns` + puts helm_uninstall if check_verbose(args) + rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` + puts rm if check_verbose(args) +end task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| current_dir = FileUtils.pwd From 2043f6f95ab9241948808bf7afe1ae2593434d8b Mon Sep 17 00:00:00 2001 From: wolfpack Date: Thu, 12 Mar 2020 15:30:49 -0400 Subject: [PATCH 05/49] #cncf/cnf-conformance/issues/56 README now reflects running suite with sample cnf --- INSTALL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 298d56e61..11650ef49 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -53,6 +53,10 @@ white_list_helm_chart_container_names: ``` crystal src/cnf-conformance.cr cleanup ``` + * To set up a *sample cnf* for use with cnf-conformance + ``` + crystal src/cnf-conformance.cr sample_coredns_setup + ``` ### Get ready to rock and roll! ## Example Usage (or see the [complete usage documentation](https://github.com/cncf/cnf-conformance/blob/master/USAGE.md)) From c3bffb61bb5066087507432733e8a7b94b9f67ad Mon Sep 17 00:00:00 2001 From: Lucina <11701267+lixuna@users.noreply.github.com> Date: Thu, 12 Mar 2020 16:34:18 -0500 Subject: [PATCH 06/49] update USAGE using To Do, PoC and :heavy check: # CNF Conformance PR Template ## Description - update tests in USAGE.md using To Do, PoC and :heavy check: ## Issues: - https://github.com/cncf/cnf-conformance/issues/95 ## How has this been tested: - [ ] Covered by existing integration testing - [ ] Added integration testing to cover - [ ] Verified all A/C passes * [x] develop * [ ] master * [ ] tag/other branch - [ ] Test environment * [ ] Shared Packet K8s cluster * [ ] New Packet K8s cluster * [ ] Kind cluster - [x] Have not tested ## Types of changes: - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [x] Documentation update ## Checklist: **Documentation** - [ ] My change requires a change to the documentation. - [x] I have updated the documentation accordingly. - [ ] No updates required. **Code Review** - [ ] Does the test handle fatal exceptions, ie. rescue block **Issue** - [x] Tasks in issue are checked off --- USAGE.md | 59 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/USAGE.md b/USAGE.md index 6f277d158..93e754463 100644 --- a/USAGE.md +++ b/USAGE.md @@ -24,7 +24,7 @@ crystal src/cnf-conformance.cr all ``` crystal src/cnf-conformance.cr compatibility ``` -#### (WIP) To check of the CNF's CNI plugin accepts valid calls from the [CNI specification](https://github.com/containernetworking/cni/blob/master/SPEC.md) +#### (To Do) To check of the CNF's CNI plugin accepts valid calls from the [CNI specification](https://github.com/containernetworking/cni/blob/master/SPEC.md) ``` crystal src/cnf-conformance.cr cni_spec ``` @@ -32,11 +32,11 @@ crystal src/cnf-conformance.cr cni_spec ``` pseudo code ``` -#### (WIP) To check for the use of beta K8s API endpoints +#### (To Do) To check for the use of beta K8s API endpoints ``` crystal src/cnf-conformance.cr api_snoop_beta ``` -#### (WIP) To check for the use of generally available (GA) K8s API endpoints +#### (To Do) To check for the use of generally available (GA) K8s API endpoints ``` crystal src/cnf-conformance.cr api_snoop_general_apis ``` @@ -47,11 +47,11 @@ crystal src/cnf-conformance.cr api_snoop_general_apis ``` crystal src/cnf-conformance.cr stateless ``` -#### (WIP) To test if the CNF responds properly [when being restarted](//https://github.com/litmuschaos/litmus) +#### (To Do) To test if the CNF responds properly [when being restarted](//https://github.com/litmuschaos/litmus) ``` crystal src/cnf-conformance.cr reset_cnf ``` -#### (WIP) To test if, when parent processes are restarted, the [child processes](https://github.com/falcosecurity/falco) are [reaped](https://github.com/draios/sysdig-inspect) +#### (To Do) To test if, when parent processes are restarted, the [child processes](https://github.com/falcosecurity/falco) are [reaped](https://github.com/draios/sysdig-inspect) ``` crystal src/cnf-conformance.cr check_reaped ``` @@ -66,11 +66,11 @@ crystal src/cnf-conformance.cr security ``` crystal src/cnf-conformance.cr privileged ``` -#### (WIP) To check if there are any [shells running in the container](https://github.com/open-policy-agent/gatekeeper) +#### (To Do) To check if there are any [shells running in the container](https://github.com/open-policy-agent/gatekeeper) ``` crystal src/cnf-conformance.cr shells ``` -#### [WIP] To check if there are any [protected directories](https://github.com/open-policy-agent/gatekeeper) or files that are accessed from within the container +#### [To Do] To check if there are any [protected directories](https://github.com/open-policy-agent/gatekeeper) or files that are accessed from within the container ``` crystal src/cnf-conformance.cr protected_access ``` @@ -98,20 +98,20 @@ crystal src/cnf-conformance.cr increase_decrease_capacity deployment_name=coredn # e.g. deployment_name: coredns/coredns ``` -#### (WIP) To test small scale autoscaling +#### (To Do) To test small scale autoscaling ``` crystal src/cnf-conformance.cr small_autoscaling ``` -#### (WIP) To test [large scale autoscaling](https://github.com/cncf/cnf-testbed) +#### (To Do) To test [large scale autoscaling](https://github.com/cncf/cnf-testbed) ``` crystal src/cnf-conformance.cr large_autoscaling ``` -#### (WIP) To test if the CNF responds to [network](https://github.com/alexei-led/pumba) [chaos](https://github.com/worstcase/blockade) +#### (To Do) To test if the CNF responds to [network](https://github.com/alexei-led/pumba) [chaos](https://github.com/worstcase/blockade) ``` crystal src/cnf-conformance.cr network_chaos ``` -#### (WIP) To test if the CNF control layer uses [external retry logic](https://github.com/envoyproxy/envoy) +#### (To Do) To test if the CNF control layer uses [external retry logic](https://github.com/envoyproxy/envoy) ``` crystal src/cnf-conformance.cr external_retry ``` @@ -122,31 +122,31 @@ crystal src/cnf-conformance.cr external_retry crystal src/cnf-conformance.cr configuration_lifecycle ``` -#### (WIP) To test if the CNF is installed with a versioned Helm v3 Chart +#### (To Do) To test if the CNF is installed with a versioned Helm v3 Chart ``` crystal src/cnf-conformance.cr versioned_helm_chart ``` -#### :heavy_check_mark: To test if there are any (non-declarative) hardcoded IP addresses or subnet masks +#### (PoC) To test if there are any (non-declarative) hardcoded IP addresses or subnet masks ``` crystal src/cnf-conformance.cr ip_addresses ``` -#### :heavy_check_mark: To test if there is a liveness and readiness entry in the Helm chart +#### (PoC) To test if there is a liveness and readiness entry in the Helm chart ``` crystal src/cnf-conformance.cr liveness ``` -#### (WIP) Test starting a container without mounting a volume that has configuration files +#### (To Do) Test starting a container without mounting a volume that has configuration files ``` crystal src/cnf-conformance.cr no_volume_with_configuration ``` -#### (WIP) To test if the CNF responds properly [when being restarted](//https://github.com/litmuschaos/litmus) +#### (To Do) To test if the CNF responds properly [when being restarted](//https://github.com/litmuschaos/litmus) ``` crystal src/cnf-conformance.cr reset_cnf ``` -#### (WIP) To test if, when parent processes are restarted, the [child processes](https://github.com/falcosecurity/falco) are [reaped](https://github.com/draios/sysdig-inspect) +#### (To Do) To test if, when parent processes are restarted, the [child processes](https://github.com/falcosecurity/falco) are [reaped](https://github.com/draios/sysdig-inspect) ``` crystal src/cnf-conformance.cr check_reaped ``` -#### (WIP) To test if the CNF can perform a [rolling update](https://kubernetes.io/docs/tasks/run-application/rolling-update-replication-controller/) +#### (To Do) To test if the CNF can perform a [rolling update](https://kubernetes.io/docs/tasks/run-application/rolling-update-replication-controller/) ``` crystal src/cnf-conformance.cr rolling_update ``` @@ -156,23 +156,23 @@ crystal src/cnf-conformance.cr rolling_update ``` crystal src/cnf-conformance.cr observability ``` -#### (WIP) Test if there traffic to Fluentd +#### (To Do) Test if there traffic to Fluentd ``` crystal src/cnf-conformance.cr fluentd_traffic ``` -#### (WIP) Test if there is traffic to Jaeger +#### (To Do) Test if there is traffic to Jaeger ``` crystal src/cnf-conformance.cr jaeger_traffic ``` -#### (WIP) Test if there is traffic to Prometheus +#### (To Do) Test if there is traffic to Prometheus ``` crystal src/cnf-conformance.cr prometheus traffic ``` -#### (WIP) Test if tracing calls are compatible with [OpenTelemetry](https://opentracing.io/) +#### (To Do) Test if tracing calls are compatible with [OpenTelemetry](https://opentracing.io/) ``` crystal src/cnf-conformance.cr opentelemetry_compatible ``` -#### (WIP) Test are if the monitoring calls are compatible with [OpenMetric](https://github.com/OpenObservability/OpenMetrics) +#### (To Do) Test are if the monitoring calls are compatible with [OpenMetric](https://github.com/OpenObservability/OpenMetrics) ``` crystal src/cnf-conformance.cr openmetric_compatible ``` @@ -182,7 +182,7 @@ crystal src/cnf-conformance.cr openmetric_compatible ``` crystal src/cnf-conformance.cr installability ``` -#### :heavy_check_mark: Test if the install script uses [Helm v3](https://github.com/helm/) +#### (PoC) Test if the install script uses [Helm v3](https://github.com/helm/) ``` crystal src/cnf-conformance.cr install_script_helm ``` @@ -191,7 +191,7 @@ crystal src/cnf-conformance.cr install_script_helm ``` crystal src/cnf-conformance.cr helm_chart_valid ``` -#### (WIP) To test if the CNF can perform a [rolling update](https://kubernetes.io/docs/tasks/run-application/rolling-update-replication-controller/) +#### (To Do) To test if the CNF can perform a [rolling update](https://kubernetes.io/docs/tasks/run-application/rolling-update-replication-controller/) ``` crystal src/cnf-conformance.cr rolling_update ``` @@ -202,20 +202,21 @@ crystal src/cnf-conformance.cr rolling_update crystal src/cnf-conformance.cr hardware_affinity ``` -#### (WIP) Test if the CNF is accessing hardware in its configuration files +#### (To Do) Test if the CNF is accessing hardware in its configuration files ``` crystal src/cnf-conformance.cr static_accessing_hardware ``` -#### (WIP) Test if the CNF is accessing hardware directly during run-time (e.g. accessing the host /dev or /proc from a mount) +#### (To Do) Test if the CNF is accessing hardware directly during run-time (e.g. accessing the host /dev or /proc from a mount) ``` crystal src/cnf-conformance.cr dynamic_accessing_hardware ``` -#### (WIP) Test if the CNF is accessing hugepages directly instead of via [Kubernetes resources](https://github.com/cncf/cnf-testbed/blob/c4458634deca5e8ab73adf118eedde32904c8458/examples/use_case/external-packet-filtering-on-k8s-nsm-on-packet/gateway.yaml#L29) +#### (To Do) Test if the CNF is accessing hugepages directly instead of via [Kubernetes resources](https://github.com/cncf/cnf-testbed/blob/c4458634deca5e8ab73adf118eedde32904c8458/examples/use_case/external-packet-filtering-on-k8s-nsm-on-packet/gateway.yaml#L29) ``` crystal src/cnf-conformance.cr direct_hugepages ``` -#### (WIP) Test if the CNF Testbed performance output shows adequate throughput and sessions using the [CNF Testbed](https://github.com/cncf/cnf-testbed) (vendor neutral) hardware environment +#### (To Do) Test if the CNF Testbed performance output shows adequate throughput and sessions using the [CNF Testbed](https://github.com/cncf/cnf-testbed) (vendor neutral) hardware environment ``` crystal src/cnf-conformance.cr performance ``` + From fcd2961cbd6a62ab56dab9a2428737af8614ef38 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Thu, 12 Mar 2020 19:03:56 -0400 Subject: [PATCH 07/49] #cncf/cnf-conformance/issues/56 sample_setup and sample_setup_args now setup a generic cnf --- .../sample-coredns-cnf/cnf-conformance.yml | 1 + sample-cnfs/sample-generic-cnf/README.md | 39 +++++++++ .../sample-generic-cnf/cnf-conformance.yml | 10 +++ .../cnf-conformance.yml | 11 +++ spec/sample_coredns_setup_spec.cr | 49 +++++++---- src/tasks/utils.cr | 87 +++++++++++++------ 6 files changed, 151 insertions(+), 46 deletions(-) create mode 100644 sample-cnfs/sample-generic-cnf/README.md create mode 100644 sample-cnfs/sample-generic-cnf/cnf-conformance.yml create mode 100644 sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index ea4dc23e6..54431ae1d 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -1,5 +1,6 @@ --- helm_directory: cnfs/coredns/helm_chart/coredns +# helm_directory: helm_chart git_clone_url: https://github.com/coredns/coredns.git install_script: cnfs/coredns/Makefile release_name: coredns diff --git a/sample-cnfs/sample-generic-cnf/README.md b/sample-cnfs/sample-generic-cnf/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample-generic-cnf/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample-generic-cnf/cnf-conformance.yml b/sample-cnfs/sample-generic-cnf/cnf-conformance.yml new file mode 100644 index 000000000..e7d4a1d76 --- /dev/null +++ b/sample-cnfs/sample-generic-cnf/cnf-conformance.yml @@ -0,0 +1,10 @@ +--- +helm_directory: helm_chart +git_clone_url: https://github.com/coredns/coredns.git +install_script: cnfs/coredns/Makefile +release_name: coredns +deployment_name: coredns-coredns +application_deployment_names: [coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: coredns +white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml b/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml new file mode 100644 index 000000000..adcd43cbb --- /dev/null +++ b/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml @@ -0,0 +1,11 @@ +--- +# helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart +helm_directory: chart +git_clone_url: +install_script: cnfs/sample_privileged_cnf_setup_coredns/chart +release_name: privileged-coredns +deployment_name: privileged-coredns-coredns +application_deployment_names: [privileged-coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: privileged-coredns-coredns +white_list_helm_chart_container_names: [] diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 60c78b703..4f9a54247 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -27,8 +27,9 @@ describe "Utils" do after_each do `crystal src/cnf-conformance.cr cleanup` $?.success?.should be_true + sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end - + it "'wait_for_install' should wait for a cnf to be installed" do `crystal src/cnf-conformance.cr sample_coredns_setup` $?.success?.should be_true @@ -44,34 +45,44 @@ describe "Utils" do (current_replicas.to_i > 0).should be_true end - it "'sample_setup' should set up a sample cnf" do + it "'sample_setup' should set up a sample cnf", tags: "WIP" do args = Sam::Args.new - sample_setup(sample_dir: "sample-cnfs/sample-coredns-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "cnfs/coredns/helm_chart/coredns", git_clone_url: "https://github.com/coredns/coredns.git" ) + sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git" ) # check if directory exists - (Dir.exists? "sample-cnfs/sample-coredns-cnf").should be_true - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true - (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_true + (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true + (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/sample-generic-cnf/helm_chart/Chart.yaml")).should be_true + sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end - + # it "'sample_setup_args' should set up a sample cnf from a argument", tags: "WIP" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) # check if directory exists - (Dir.exists? "sample-cnfs/sample-coredns-cnf").should be_true - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_true - (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_true + (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true + (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true + (File.exists?("cnfs/sample-generic-cnf/helm_chart/Chart.yaml")).should be_true + sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end it "'sample_cleanup' should clean up a sample cnf from a argument", tags: "WIP" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) - sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) + # check if directory exists + (Dir.exists? "cnfs/sample-generic-cnf").should be_false + (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_false + (File.exists?("cnfs/sample-generic-cnf/helm_chart/Chart.yaml")).should be_false + end + + it "'sample_setup_args' should be able to deploy using a helm_directory" do + args = Sam::Args.new + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf_setup_coredns", deploy_with_chart: false, args: args, verbose: true ) # check if directory exists - (Dir.exists? "cnfs/coredns").should be_false - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_false - (File.exists?("cnfs/coredns/cnf-conformance.yml")).should be_false - (File.exists?("cnfs/coredns/helm_chart/coredns/Chart.yaml")).should be_false + (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns").should be_true + # should not clone + (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns/privileged-coredns").should be_false + (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml").should be_true + (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml").should be_true end end diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index bbf54f666..d01e1f8b7 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -52,37 +52,38 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) config = sample_conformance_yml(sample_dir) if args.named.keys.includes? "release_name" - release_name = args.named["release_name"] + release_name = "#{args.named["release_name"]}" else - release_name = config.get("release_name").as_s + release_name = "#{config.get("release_name").as_s?}" end puts "release_name: #{release_name}" if verbose if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] + deployment_name = "#{args.named["deployment_name"]}" else - deployment_name = config.get("deployment_name").as_s + deployment_name = "#{config.get("deployment_name").as_s?}" end puts "deployment_name: #{deployment_name}" if verbose if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] + helm_chart = "#{args.named["helm_chart"]}" else - helm_chart = config.get("helm_chart").as_s + helm_chart = "#{config.get("helm_chart").as_s?}" end puts "helm_chart: #{helm_chart}" if verbose if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] + helm_directory = "#{args.named["helm_directory"]}" else - helm_directory = config.get("helm_directory").as_s + helm_directory = "#{config.get("helm_directory").as_s?}" end puts "helm_directory: #{helm_directory}" if verbose if args.named.keys.includes? "git_clone_url" - git_clone_url = args.named["git_clone_url"] + git_clone_url = "#{args.named["git_clone_url"]}" else - git_clone_url = config.get("git_clone_url").as_s + # TODO check type (any) before doing .as_s + git_clone_url = "#{config.get("git_clone_url").as_s?}" end puts "git_clone_url: #{git_clone_url}" if verbose @@ -90,38 +91,61 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) end -def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url, deploy_with_chart=true, verbose=false) +def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false) current_dir = FileUtils.pwd puts current_dir if verbose + destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + puts "destination_cnf_dir: #{destination_cnf_dir}" if verbose + FileUtils.mkdir_p(destination_cnf_dir) # TODO enable recloning/fetching etc # TODO pass in block - git_clone = `git clone #{git_clone_url} #{current_dir}/#{CNF_DIR}/#{release_name}` + git_clone = `git clone #{git_clone_url} #{destination_cnf_dir}/#{release_name}` if git_clone_url.empty? == false puts git_clone if verbose # Copy the cnf-conformance.yml - yml_cp = `cp #{sample_dir}/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/#{release_name}` + # yml_cp = `cp #{sample_dir}/cnf-conformance.yml #{destination_cnf_dir}` + # Copy the sample + yml_cp = `cp -a #{sample_dir} #{CNF_DIR}` puts yml_cp if verbose + raise "Copy of #{sample_dir}/cnf-conformance.yml to #{destination_cnf_dir} failed!" unless $?.success? + begin helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" puts helm if verbose - helm_install = `#{helm} install #{release_name} #{helm_chart}` - puts helm_install if verbose - - - # Retrieve the helm chart source - FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart") - helm_pull = `#{helm} pull #{helm_chart}` - puts helm_pull if verbose - core_mv = `mv #{release_name}-*.tgz #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart` - puts core_mv if verbose - tar = `cd #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/#{release_name}/helm_chart/#{release_name}-*.tgz` - puts tar if verbose + if deploy_with_chart + puts "deploying with chart" if verbose + helm_install = `#{helm} install #{release_name} #{helm_chart}` + puts helm_install if verbose + + # Retrieve the helm chart source + FileUtils.mkdir_p("#{destination_cnf_dir}/#{helm_directory}") + helm_pull = `#{helm} pull #{helm_chart}` + puts helm_pull if verbose + # core_mv = `mv #{release_name}-*.tgz #{destination_cnf_dir}/#{helm_directory}` + # TODO helm_chart should be helm_chart_repo + puts "mv #{chart_name(helm_chart)}-*.tgz #{destination_cnf_dir}/#{helm_directory}" if verbose + core_mv = `mv #{chart_name(helm_chart)}-*.tgz #{destination_cnf_dir}/#{helm_directory}` + puts core_mv if verbose + + puts "cd #{destination_cnf_dir}/#{helm_directory}; tar -xvf #{destination_cnf_dir}/#{helm_directory}/#{chart_name(helm_chart)}-*.tgz" if verbose + tar = `cd #{destination_cnf_dir}/#{helm_directory}; tar -xvf #{destination_cnf_dir}/#{helm_directory}/#{chart_name(helm_chart)}-*.tgz` + puts tar if verbose + + puts "mv #{destination_cnf_dir}/#{helm_directory}/#{chart_name(helm_chart)}/* #{destination_cnf_dir}/#{helm_directory}" if verbose + move_chart = `mv #{destination_cnf_dir}/#{helm_directory}/#{chart_name(helm_chart)}/* #{destination_cnf_dir}/#{helm_directory}` + puts move_chart if verbose + else + puts "deploying with helm directory" if verbose + helm_install = `#{helm} install #{release_name} #{destination_cnf_dir}/#{helm_directory}` + puts helm_install if verbose + end + wait_for_install(deployment_name) - if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 + if helm_install.to_s.size > 0 # && helm_pull.to_s.size > 0 puts "Successfully setup #{release_name}".colorize(:green) end ensure @@ -139,6 +163,15 @@ def sample_cleanup(sample_dir, verbose=true) puts helm if verbose helm_uninstall = `#{helm} uninstall #{release_name}` puts helm_uninstall if verbose - rm = `rm -rf #{current_dir}/#{CNF_DIR}/#{release_name}` + destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + rm = `rm -rf #{destination_cnf_dir}` puts rm if verbose end + +def chart_name(helm_chart_repo) + helm_chart_repo.split("/").last +end + +def short_sample_dir(full_sample_dir) + full_sample_dir.split("/").last +end From 241d2ffb34d6822e80c656c0d553a61dab10aa6c Mon Sep 17 00:00:00 2001 From: wolfpack Date: Fri, 13 Mar 2020 19:28:03 -0400 Subject: [PATCH 08/49] #cncf/cnf-conformance/issues/56 sample_coredns now uses generic sample setup code --- .../sample-coredns-cnf-source/README.md | 39 ++++ .../cnf-conformance.yml | 11 ++ .../sample-coredns-cnf/cnf-conformance.yml | 4 +- .../sample_coredns/cnf-conformance.yml | 6 +- .../cnf-conformance.yml | 6 +- spec/cnf_conformance_spec.cr | 18 -- spec/configuration_lifecycle_spec.cr | 14 +- spec/installability_spec.cr | 59 ++++++ spec/sample_coredns_setup_spec.cr | 17 +- src/tasks/configuration_lifecycle.cr | 26 ++- src/tasks/installability.cr | 22 ++- src/tasks/sample_coredns_setup.cr | 184 ++---------------- src/tasks/utils.cr | 22 ++- 13 files changed, 217 insertions(+), 211 deletions(-) create mode 100644 sample-cnfs/sample-coredns-cnf-source/README.md create mode 100644 sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml create mode 100644 spec/installability_spec.cr diff --git a/sample-cnfs/sample-coredns-cnf-source/README.md b/sample-cnfs/sample-coredns-cnf-source/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf-source/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml new file mode 100644 index 000000000..5e16eb8f5 --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml @@ -0,0 +1,11 @@ +--- +helm_directory: cnfs/coredns/helm_chart/coredns +# helm_directory: helm_chart +git_clone_url: https://github.com/coredns/coredns.git +install_script: coredns/Makefile +release_name: coredns +deployment_name: coredns-coredns +application_deployment_names: [coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: coredns +white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index 54431ae1d..938751cb3 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -1,8 +1,8 @@ --- helm_directory: cnfs/coredns/helm_chart/coredns # helm_directory: helm_chart -git_clone_url: https://github.com/coredns/coredns.git -install_script: cnfs/coredns/Makefile +git_clone_url: +install_script: coredns/Makefile release_name: coredns deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index b3d011943..8abdde7b4 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/sample_coredns/chart -install_script: cnfs/sample_coredns/chart +helm_directory: chart +git_clone_url: +install_script: chart +release_name: coredns deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns diff --git a/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml index 88fb77cc6..9a3c51d33 100644 --- a/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/sample_coredns_bad_liveness/chart -install_script: cnfs/sample_coredns_bad_liveness/chart +helm_directory: chart +git_clone_url: +install_script: chart +release_name: bad-liveness deployment_name: bad-liveness-coredns application_deployment_names: [bad-liveness-coredns] helm_chart: stable/coredns diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_spec.cr index f8dc39a0a..f77c8cc23 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_spec.cr @@ -28,8 +28,6 @@ describe CnfConformance do $?.success?.should be_true (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil - (/FAILURE: Helm not found in install script/ =~ response_s).should_not be_nil - (/FAILURE: IP addresses found/ =~ response_s).should_not be_nil (/Lint Passed/ =~ response_s).should_not be_nil (/PASSED: Replicas increased to 3/ =~ response_s).should_not be_nil (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil @@ -45,21 +43,5 @@ describe CnfConformance do (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil end - it "'helm_chart_valid' should fail on a bad helm chart" do - # puts `pwd` - # puts `echo $KUBECONFIG` - `crystal src/cnf-conformance.cr sample_coredns_cleanup` - $?.success?.should be_true - `crystal src/cnf-conformance.cr bad_helm_cnf_setup` - $?.success?.should be_true - response_s = `crystal src/cnf-conformance.cr helm_chart_valid` - puts response_s - $?.success?.should be_true - (/Lint Failed/ =~ response_s).should_not be_nil - `crystal src/cnf-conformance.cr bad_helm_cnf_cleanup` - $?.success?.should be_true - `crystal src/cnf-conformance.cr sample_coredns_setup` - $?.success?.should be_true - end end diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index 3cc122f80..099b5db5b 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -12,6 +12,18 @@ describe CnfConformance do # `crystal src/cnf-conformance.cr setup` # $?.success?.should be_true end + it "'ip_addresses' should fail when ip addresses are found in source is set", tags: "liveness" do + begin + `crystal src/cnf-conformance.cr sample_coredns_source_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr ip_addresses verbose` + puts response_s + $?.success?.should be_true + (/FAILURE: IP addresses found/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr sample_coredns_source_cleanup verbose` + end + end it "'liveness' should pass when livenessProbe is set", tags: "liveness" do begin `crystal src/cnf-conformance.cr sample_coredns` @@ -21,7 +33,7 @@ describe CnfConformance do $?.success?.should be_true (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil ensure - `crystal src/cnf-conformance.cr cleanup_sample_coredns` + `crystal src/cnf-conformance.cr cleanup_sample_coredns verbose` end end it "'liveness' should fail when livenessProbe is not set", tags: "liveness" do diff --git a/spec/installability_spec.cr b/spec/installability_spec.cr new file mode 100644 index 000000000..263acc103 --- /dev/null +++ b/spec/installability_spec.cr @@ -0,0 +1,59 @@ +require "./spec_helper" +require "colorize" + +describe CnfConformance do + before_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true + + # `crystal src/cnf-conformance.cr setup` + # $?.success?.should be_true + end + + it "'install_script_helm' should fail if install script does not have helm" do + # puts `pwd` + # puts `echo $KUBECONFIG` + # `crystal src/cnf-conformance.cr cleanup` + # $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_source_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr install_script_helm` + puts response_s + $?.success?.should be_true + (/FAILURE: Helm not found in supplied install script/ =~ response_s).should_not be_nil + `crystal src/cnf-conformance.cr sample_coredns_source_cleanup` + end + + it "'helm_chart_valid' should pass on a good helm chart" do + # puts `pwd` + # puts `echo $KUBECONFIG` + # `crystal src/cnf-conformance.cr cleanup` + # $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr helm_chart_valid` + puts response_s + $?.success?.should be_true + (/Lint Passed/ =~ response_s).should_not be_nil + end + + it "'helm_chart_valid' should fail on a bad helm chart" do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr sample_coredns_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr bad_helm_cnf_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr helm_chart_valid` + puts response_s + $?.success?.should be_true + (/Lint Failed/ =~ response_s).should_not be_nil + `crystal src/cnf-conformance.cr bad_helm_cnf_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + end +end diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 4f9a54247..bb61d7db7 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -45,7 +45,7 @@ describe "Utils" do (current_replicas.to_i > 0).should be_true end - it "'sample_setup' should set up a sample cnf", tags: "WIP" do + it "'sample_setup' should set up a sample cnf" do args = Sam::Args.new sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git" ) # check if directory exists @@ -55,7 +55,7 @@ describe "Utils" do sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end # - it "'sample_setup_args' should set up a sample cnf from a argument", tags: "WIP" do + it "'sample_setup_args' should set up a sample cnf from a argument" do args = Sam::Args.new sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) # check if directory exists @@ -65,7 +65,7 @@ describe "Utils" do sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end - it "'sample_cleanup' should clean up a sample cnf from a argument", tags: "WIP" do + it "'sample_cleanup' should clean up a sample cnf from a argument" do args = Sam::Args.new sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) @@ -85,4 +85,15 @@ describe "Utils" do (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml").should be_true (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml").should be_true end + + it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do + args = Sam::Args.new + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + (cnf_conformance_dir).should eq("sample-generic-cnf") + end + + it "'sample_destination_dir' should return the full path of the potential destination cnf directory based on the source sample cnf directory", tags: "WIP" do + args = Sam::Args.new + sample_destination_dir("sample-generic-cnf").should contain("cnf-conformance/cnfs/sample-generic-cnf") + end end diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index a6e60dcf6..23f5b3b60 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -14,6 +14,7 @@ task "ip_addresses" do |_, args| cdir = FileUtils.pwd() response = String::Builder.new Dir.cd(CNF_DIR) + # TODO ignore *example*, *.md, *.txt Process.run("grep -rnw -E -o '([0-9]{1,3}[\.]){3}[0-9]{1,3}'", shell: true) do |proc| # Process.run("grep -rnw -E -o 'hithere'", shell: true) do |proc| while line = proc.output.gets @@ -49,8 +50,12 @@ task "liveness", ["retrieve_manifest"] do |_, args| puts "FAILURE: helm directory not found".colorize(:red) puts ex.message if check_args(args) end - puts "helm_directory: #{helm_directory}/manifest.yml" if check_verbose(args) - deployment = Totem.from_file "#{helm_directory}/manifest.yml" + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + puts "helm_directory: #{destination_cnf_dir}/#{helm_directory}/manifest.yml" if check_verbose(args) + deployment = Totem.from_file "#{destination_cnf_dir}/#{helm_directory}/manifest.yml" puts deployment.inspect if check_verbose(args) containers = deployment.get("spec").as_h["template"].as_h["spec"].as_h["containers"].as_a containers.each do |container| @@ -87,8 +92,12 @@ task "readiness", ["retrieve_manifest"] do |_, args| puts "FAILURE: helm directory not found".colorize(:red) puts ex.message if check_args(args) end - puts "helm_directory: #{helm_directory}/manifest.yml" if check_verbose(args) - deployment = Totem.from_file "#{helm_directory}/manifest.yml" + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + puts "helm_directory: #{destination_cnf_dir}/#{helm_directory}/manifest.yml" if check_verbose(args) + deployment = Totem.from_file "#{destination_cnf_dir}/#{helm_directory}/manifest.yml" puts deployment.inspect if check_verbose(args) containers = deployment.get("spec").as_h["template"].as_h["spec"].as_h["containers"].as_a containers.each do |container| @@ -115,10 +124,17 @@ end desc "Retrieve the manifest for the CNF's helm chart" task "retrieve_manifest" do |_, args| begin + puts "retrieve_manifest" if check_verbose(args) config = cnf_conformance_yml deployment_name = config.get("deployment_name").as_s + puts deployment_name if check_verbose(args) helm_directory = config.get("helm_directory").as_s - manifest = `kubectl get deployment #{deployment_name} -o yaml > #{helm_directory}/manifest.yml` + puts helm_directory if check_verbose(args) + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + manifest = `kubectl get deployment #{deployment_name} -o yaml > #{destination_cnf_dir}/#{helm_directory}/manifest.yml` puts manifest if check_verbose(args) rescue ex puts ex.message diff --git a/src/tasks/installability.cr b/src/tasks/installability.cr index d78f94c53..fd803acd5 100644 --- a/src/tasks/installability.cr +++ b/src/tasks/installability.cr @@ -15,9 +15,13 @@ task "install_script_helm" do |_, args| config = cnf_conformance_yml found = 0 + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) install_script = config.get("install_script").as_s response = String::Builder.new - content = File.open(install_script) do |file| + content = File.open("#{destination_cnf_dir}/#{install_script}") do |file| file.gets_to_end end # puts content @@ -25,9 +29,9 @@ task "install_script_helm" do |_, args| found = 1 end if found < 1 - puts "FAILURE: Helm not found in install script".colorize(:red) + puts "FAILURE: Helm not found in supplied install script".colorize(:red) else - puts "PASSED: Helm found in install script".colorize(:green) + puts "PASSED: Helm found in supplied install script".colorize(:green) end rescue ex puts ex.message @@ -53,14 +57,20 @@ task "helm_chart_valid", ["helm_local_install"] do |_, args| end puts "helm_directory: #{helm_directory}" if check_verbose(args) - ls_helm_directory = `ls -al #{helm_directory}` - puts "ls -al of helm_directory: #{ls_helm_directory}" if check_verbose(args) puts "helm_chart_repo: #{helm_chart_repo}" if check_verbose(args) current_dir = FileUtils.pwd puts current_dir if check_verbose(args) helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - helm_lint = `#{helm} lint #{helm_directory}` + + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + ls_helm_directory = `ls -al #{destination_cnf_dir}/#{helm_directory}` + puts "ls -al of helm_directory: #{ls_helm_directory}" if check_verbose(args) + + helm_lint = `#{helm} lint #{destination_cnf_dir}/#{helm_directory}` puts "helm_lint: #{helm_lint}" if check_verbose(args) # Process.run("helm lint #{helm_directory}", shell: true) do |proc| diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 358eb3dc7..925608e29 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -8,112 +8,17 @@ require "./utils.cr" desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Retrieve the cnf source - # TODO enable recloning/fetching etc - git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/coredns` - puts git_clone if check_verbose(args) - - # Copy the cnf-conformance.yml - yml_cp = `cp sample-cnfs/sample-coredns-cnf/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/coredns` - puts yml_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install coredns #{helm_chart}` - puts helm_install if check_verbose(args) - + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) +end - # Retrieve the helm chart source - FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/coredns/helm_chart") - helm_pull = `#{helm} pull #{helm_chart}` - puts helm_pull if check_verbose(args) - core_mv = `mv coredns-*.tgz #{current_dir}/#{CNF_DIR}/coredns/helm_chart` - puts core_mv if check_verbose(args) - tar = `cd #{current_dir}/#{CNF_DIR}/coredns/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/coredns/helm_chart/coredns-*.tgz` - puts tar if check_verbose(args) - # coredns-coredns deployment must exist before running the next line (must already be installed) - # # TODO get deployment name from previous install - # manifest = `kubectl get deployment coredns-coredns -o yaml > #{CNF_DIR}/coredns/helm_chart/coredns/manifest.yml` - # puts manifest if check_verbose(args) - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 - puts "Successfully setup coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end +desc "Sets up sample CoreDNS CNF with source" +task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) end task "sample_coredns", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns #{current_dir}/#{CNF_DIR}/` - puts chart_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_coredns new setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end @@ -285,74 +190,18 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns_bad_liveness #{current_dir}/#{CNF_DIR}/` - puts chart_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install bad-liveness #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_coredns_bad_liveness".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_coredns_bad_liveness" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) end task "sample_coredns_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) end task "cleanup_sample_coredns" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_coredns", verbose: true) end @@ -387,12 +236,9 @@ task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| end task "sample_coredns_bad_liveness_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall bad-liveness` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns_bad_liveness` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", verbose: true) +end +task "sample_coredns_source_cleanup" do |_, args| + sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf-source", verbose: true) end diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index d01e1f8b7..6cfebdbc6 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -25,6 +25,14 @@ def cnf_conformance_yml Totem.from_file "./#{cnf_conformance}" end +def cnf_conformance_dir + cnf_conformance = `find cnfs/* -name "cnf-conformance.yml"`.split("\n")[0] + if cnf_conformance.empty? + raise "No cnf_conformance.yml found! Did you run the setup task?" + end + cnf_conformance.split("/")[-2] +end + def sample_conformance_yml(sample_dir) cnf_conformance = `find #{sample_dir}/* -name "cnf-conformance.yml"`.split("\n")[0] if cnf_conformance.empty? @@ -48,7 +56,8 @@ def wait_for_install(deployment_name, wait_count=180) end end def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) - # # Parse the cnf-conformance.yml + puts "sample_setup_args" if verbose + config = sample_conformance_yml(sample_dir) if args.named.keys.includes? "release_name" @@ -82,7 +91,6 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) if args.named.keys.includes? "git_clone_url" git_clone_url = "#{args.named["git_clone_url"]}" else - # TODO check type (any) before doing .as_s git_clone_url = "#{config.get("git_clone_url").as_s?}" end puts "git_clone_url: #{git_clone_url}" if verbose @@ -91,12 +99,20 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) end +def sample_destination_dir(sample_source_dir) + current_dir = FileUtils.pwd + "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_source_dir)}" +end + def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false) + puts "sample_setup" if verbose current_dir = FileUtils.pwd puts current_dir if verbose - destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + # destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + destination_cnf_dir = sample_destination_dir(sample_dir) + puts "destination_cnf_dir: #{destination_cnf_dir}" if verbose FileUtils.mkdir_p(destination_cnf_dir) # TODO enable recloning/fetching etc From fd875bc6c44ae7728fe4ab21e80c7dd8c1cb1c64 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 00:54:27 -0400 Subject: [PATCH 09/49] #cncf/cnf-conformance/issues/56 bad helm sample cnf now generic --- .../chart/Chart.yaml | 23 +++ .../chart/README.md | 137 ++++++++++++ .../chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 124 +++++++++++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ .../chart/values.yaml | 195 ++++++++++++++++++ .../cnf-conformance.yml | 6 +- src/tasks/sample_coredns_setup.cr | 75 +------ src/tasks/utils.cr | 8 +- 22 files changed, 1136 insertions(+), 75 deletions(-) create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml new file mode 100755 index 000000000..77de0c04f --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.9.3 diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md new file mode 100755 index 000000000..dc00ca26b --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md @@ -0,0 +1,137 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml new file mode 100755 index 000000000..809cc660d --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ sdfskfsdf .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ sdfskfsdf .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ sdfskfsdf .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ sdfskfsdf .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ sdfskfsdf .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + privileged: true + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ sdfskfsdf .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml new file mode 100755 index 000000000..becf13564 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml @@ -0,0 +1,195 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml index b07726780..5b37d5d6c 100644 --- a/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/bad_helm_coredns/helm_chart/coredns -install_script: cnfs/bad_helm_coredns/Makefile +helm_directory: chart +git_clone_url: +install_script: +release_name: bad-helm-coredns-coredns deployment_name: bad-helm-coredns-coredns application_deployment_names: [bad-helm-coredns] helm_chart: stable/coredns diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 925608e29..66a39cf3d 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -16,74 +16,17 @@ task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) end +desc "Sets up an alternate sample CoreDNS CNF" task "sample_coredns", ["helm_local_install"] do |_, args| puts "sample_coredns new setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end +desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Retrieve the cnf source - # TODO enable recloning/fetching etc - git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts git_clone if check_verbose(args) - - # Copy the cnf-conformance.yml - yml_cp = `cp sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts yml_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - # name must be dns compliant (no underscores) - helm_install = `#{helm} install bad-helm-coredns #{helm_chart}` - puts helm_install if check_verbose(args) - - - # Retrieve the helm chart source - FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart") - helm_pull = `#{helm} pull #{helm_chart}` - puts helm_pull if check_verbose(args) - core_mv = `mv coredns-*.tgz #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart` - puts core_mv if check_verbose(args) - tar = `cd #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart/coredns-*.tgz` - puts tar if check_verbose(args) - - #sed command that changes helm chart to be bad - helm_sed = `sed -i s/"toYaml"/"sdfsdfsdf"/g #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart/coredns/templates/deployment.yaml` - puts helm_sed if check_verbose(args) - - wait_for_install(deployment_name) - - if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 - puts "Successfully setup coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "bad_helm_cnf_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 5 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| @@ -206,13 +149,7 @@ end task "bad_helm_cnf_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall bad-helm-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", verbose: true) end task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index 6cfebdbc6..8ee98af71 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -55,7 +55,7 @@ def wait_for_install(deployment_name, wait_count=180) second_count = second_count + 1 end end -def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) +def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false, wait_count=180) puts "sample_setup_args" if verbose config = sample_conformance_yml(sample_dir) @@ -95,7 +95,7 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) end puts "git_clone_url: #{git_clone_url}" if verbose - sample_setup(sample_dir: sample_dir, release_name: release_name, deployment_name: deployment_name, helm_chart: helm_chart, helm_directory: helm_directory, git_clone_url: git_clone_url, deploy_with_chart: deploy_with_chart, verbose: verbose ) + sample_setup(sample_dir: sample_dir, release_name: release_name, deployment_name: deployment_name, helm_chart: helm_chart, helm_directory: helm_directory, git_clone_url: git_clone_url, deploy_with_chart: deploy_with_chart, verbose: verbose, wait_count: wait_count ) end @@ -104,7 +104,7 @@ def sample_destination_dir(sample_source_dir) "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_source_dir)}" end -def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false) +def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false, wait_count=180) puts "sample_setup" if verbose current_dir = FileUtils.pwd @@ -160,7 +160,7 @@ def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_dir puts helm_install if verbose end - wait_for_install(deployment_name) + wait_for_install(deployment_name, wait_count) if helm_install.to_s.size > 0 # && helm_pull.to_s.size > 0 puts "Successfully setup #{release_name}".colorize(:green) end From da46d2e718bd3653bcbb6c19c5c8232aa2b1c9d7 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 01:26:42 -0400 Subject: [PATCH 10/49] #cncf/cnf-conformance/issues/56 whitelisted,privileged sample cnf now generic --- .../README.md | 39 ++++ .../chart/Chart.yaml | 23 +++ .../chart/README.md | 137 ++++++++++++ .../chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 124 +++++++++++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ .../chart/values.yaml | 195 ++++++++++++++++++ .../cnf-conformance.yml | 10 + src/tasks/sample_coredns_setup.cr | 58 +----- 22 files changed, 1174 insertions(+), 55 deletions(-) create mode 100644 sample-cnfs/sample_whitelisted_privileged_cnf/README.md create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml create mode 100644 sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/README.md b/sample-cnfs/sample_whitelisted_privileged_cnf/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml new file mode 100755 index 000000000..77de0c04f --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.9.3 diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md new file mode 100755 index 000000000..dc00ca26b --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md @@ -0,0 +1,137 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml new file mode 100755 index 000000000..02ed35fd8 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + privileged: true + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml new file mode 100755 index 000000000..becf13564 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml @@ -0,0 +1,195 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml b/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml new file mode 100644 index 000000000..50bae9ec2 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml @@ -0,0 +1,10 @@ +--- +helm_directory: chart +git_clone_url: +install_script: +release_name: privileged-coredns +deployment_name: privileged-coredns-coredns +application_deployment_names: [privileged-coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: privileged-coredns-coredns +white_list_helm_chart_container_names: [coredns] diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 66a39cf3d..4744800bf 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -30,54 +30,8 @@ task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` - puts chart_cp if check_verbose(args) - puts yml_mv if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install privileged-coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_whitelisted_setup".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_privileged_cnf_whitelisted_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end @@ -153,13 +107,7 @@ task "bad_helm_cnf_cleanup" do |_, args| end task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall privileged-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", verbose: true) end task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| From 62409683221984c406cfbf99d2c7c54ef4e997b9 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 03:27:06 -0400 Subject: [PATCH 11/49] #cncf/cnf-conformance/issues/56 all samples are now generic --- .../README.md | 0 .../chart/Chart.yaml | 0 .../chart/README.md | 0 .../chart/templates/NOTES.txt | 0 .../chart/templates/_helpers.tpl | 0 .../templates/clusterrole-autoscaler.yaml | 0 .../chart/templates/clusterrole.yaml | 0 .../clusterrolebinding-autoscaler.yaml | 0 .../chart/templates/clusterrolebinding.yaml | 0 .../chart/templates/configmap-autoscaler.yaml | 0 .../chart/templates/configmap.yaml | 0 .../templates/deployment-autoscaler.yaml | 0 .../chart/templates/deployment.yaml | 0 .../chart/templates/podsecuritypolicy.yaml | 0 .../chart/templates/service-metrics.yaml | 0 .../chart/templates/service.yaml | 0 .../templates/serviceaccount-autoscaler.yaml | 0 .../chart/templates/serviceaccount.yaml | 0 .../chart/templates/servicemonitor.yaml | 0 .../chart/values.yaml | 0 .../cnf-conformance.yml | 3 +- .../non-whitelisted-conformance.yml | 8 --- .../whitelisted-conformance.yml | 8 --- spec/sample_coredns_setup_spec.cr | 10 +-- src/tasks/cleanup.cr | 2 +- src/tasks/sample_coredns_setup.cr | 67 ++----------------- 26 files changed, 11 insertions(+), 87 deletions(-) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/README.md (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/Chart.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/README.md (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/NOTES.txt (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/_helpers.tpl (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrole-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrole.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrolebinding-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrolebinding.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/configmap-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/configmap.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/deployment-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/deployment.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/podsecuritypolicy.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/service-metrics.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/service.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/serviceaccount-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/serviceaccount.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/servicemonitor.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/values.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/cnf-conformance.yml (70%) delete mode 100644 sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml delete mode 100644 sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/README.md b/sample-cnfs/sample_privileged_cnf/README.md similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/README.md rename to sample-cnfs/sample_privileged_cnf/README.md diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml b/sample-cnfs/sample_privileged_cnf/chart/Chart.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml rename to sample-cnfs/sample_privileged_cnf/chart/Chart.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/README.md b/sample-cnfs/sample_privileged_cnf/chart/README.md similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/README.md rename to sample-cnfs/sample_privileged_cnf/chart/README.md diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/NOTES.txt b/sample-cnfs/sample_privileged_cnf/chart/templates/NOTES.txt similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/NOTES.txt rename to sample-cnfs/sample_privileged_cnf/chart/templates/NOTES.txt diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/_helpers.tpl b/sample-cnfs/sample_privileged_cnf/chart/templates/_helpers.tpl similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/_helpers.tpl rename to sample-cnfs/sample_privileged_cnf/chart/templates/_helpers.tpl diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/configmap-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/configmap-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/configmap.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/configmap.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/deployment-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/deployment-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/deployment.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/deployment.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/podsecuritypolicy.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/podsecuritypolicy.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/podsecuritypolicy.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service-metrics.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/service-metrics.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service-metrics.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/service-metrics.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/service.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/service.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/servicemonitor.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/servicemonitor.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/servicemonitor.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/values.yaml b/sample-cnfs/sample_privileged_cnf/chart/values.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/values.yaml rename to sample-cnfs/sample_privileged_cnf/chart/values.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml b/sample-cnfs/sample_privileged_cnf/cnf-conformance.yml similarity index 70% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml rename to sample-cnfs/sample_privileged_cnf/cnf-conformance.yml index adcd43cbb..6f42928a5 100644 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_privileged_cnf/cnf-conformance.yml @@ -1,8 +1,7 @@ --- -# helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart helm_directory: chart git_clone_url: -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart +install_script: release_name: privileged-coredns deployment_name: privileged-coredns-coredns application_deployment_names: [privileged-coredns-coredns] diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml b/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml deleted file mode 100644 index be15c9da8..000000000 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart -deployment_name: privileged-coredns-coredns -application_deployment_names: [privileged-coredns-coredns] -helm_chart: stable/coredns -helm_chart_container_name: privileged-coredns-coredns -white_list_helm_chart_container_names: [] diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml b/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml deleted file mode 100644 index 73d9f96ec..000000000 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart -deployment_name: privileged-coredns-coredns -application_deployment_names: [privileged-coredns-coredns] -helm_chart: stable/coredns -helm_chart_container_name: privileged-coredns-coredns -white_list_helm_chart_container_names: [coredns] diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index bb61d7db7..87e6ba01c 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -77,13 +77,13 @@ describe "Utils" do it "'sample_setup_args' should be able to deploy using a helm_directory" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf_setup_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) # check if directory exists - (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns").should be_true + (Dir.exists? "cnfs/sample_privileged_cnf").should be_true # should not clone - (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns/privileged-coredns").should be_false - (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml").should be_true - (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml").should be_true + (Dir.exists? "cnfs/sample_privileged_cnf/privileged-coredns").should be_false + (File.exists? "cnfs/sample_privileged_cnf/cnf-conformance.yml").should be_true + (File.exists? "cnfs/sample_privileged_cnf/chart/Chart.yaml").should be_true end it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index f2d922e21..bb9bc6673 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -7,7 +7,7 @@ desc "Cleans up the CNF Conformance test suite, the K8s cluster, and upstream pr task "cleanup", ["samples_cleanup"] do |_, args| end -task "samples_cleanup", ["sample_coredns_cleanup", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup"] do |_, args| +task "samples_cleanup", ["sample_coredns_cleanup", "cleanup_sample_coredns", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup", "sample_coredns_bad_liveness_cleanup", "sample_coredns_source_cleanup"] do |_, args| end task "tools_cleanup", ["helm_local_cleanup"] do |_, args| diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 4744800bf..6df3a1273 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -2,10 +2,8 @@ require "sam" require "file_utils" require "colorize" require "totem" -# require "commander" require "./utils.cr" - desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) @@ -22,11 +20,10 @@ task "sample_coredns", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end - desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| puts "bad_helm_cnf_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 5 ) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 2 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| @@ -34,56 +31,9 @@ task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, ar sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end - task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` - puts chart_cp if check_verbose(args) - puts yml_mv if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install privileged-coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_non_whitelisted_setup".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_privileged_cnf_non_whitelisted_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| @@ -91,17 +41,14 @@ task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) end - task "sample_coredns_cleanup" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) end - task "cleanup_sample_coredns" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample_coredns", verbose: true) end - task "bad_helm_cnf_cleanup" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", verbose: true) end @@ -111,13 +58,7 @@ task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| end task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall privileged-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_privileged_cnf", verbose: true) end task "sample_coredns_bad_liveness_cleanup" do |_, args| From cb2f0d9d50a3a6297f5ef98a78bce2640949972b Mon Sep 17 00:00:00 2001 From: wolfpack Date: Mon, 16 Mar 2020 13:09:57 -0400 Subject: [PATCH 12/49] Wait times now reduced for Sample CNFS --- spec/cnf_conformance_spec.cr | 4 ++-- spec/sample_coredns_setup_spec.cr | 15 +++++++-------- src/tasks/sample_coredns_setup.cr | 18 +++++++++++------- src/tasks/utils.cr | 6 ++++-- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_spec.cr index f77c8cc23..2b9dc4dd0 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_spec.cr @@ -9,6 +9,8 @@ describe CnfConformance do $?.success?.should be_true `crystal src/cnf-conformance.cr setup` $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_with_wait_setup` + $?.success?.should be_true end after_all do @@ -21,8 +23,6 @@ describe CnfConformance do it "'all' should run the whole test suite" do # puts `pwd` # puts `echo $KUBECONFIG` - `crystal src/cnf-conformance.cr sample_coredns_setup` - $?.success?.should be_true response_s = `crystal src/cnf-conformance.cr all verbose` puts response_s $?.success?.should be_true diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 87e6ba01c..349c6ce18 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -47,7 +47,7 @@ describe "Utils" do it "'sample_setup' should set up a sample cnf" do args = Sam::Args.new - sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git" ) + sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git", wait_count: 0 ) # check if directory exists (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true @@ -57,7 +57,7 @@ describe "Utils" do # it "'sample_setup_args' should set up a sample cnf from a argument" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) # check if directory exists (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true @@ -67,9 +67,9 @@ describe "Utils" do it "'sample_cleanup' should clean up a sample cnf from a argument" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) - sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) - # check if directory exists + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) + cleanup = sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) + (cleanup.success?).should be_true (Dir.exists? "cnfs/sample-generic-cnf").should be_false (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_false (File.exists?("cnfs/sample-generic-cnf/helm_chart/Chart.yaml")).should be_false @@ -77,8 +77,7 @@ describe "Utils" do it "'sample_setup_args' should be able to deploy using a helm_directory" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) - # check if directory exists + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) (Dir.exists? "cnfs/sample_privileged_cnf").should be_true # should not clone (Dir.exists? "cnfs/sample_privileged_cnf/privileged-coredns").should be_false @@ -88,7 +87,7 @@ describe "Utils" do it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) (cnf_conformance_dir).should eq("sample-generic-cnf") end diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 6df3a1273..c9930755c 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -6,39 +6,43 @@ require "./utils.cr" desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true, wait_count: 0 ) +end + +task "sample_coredns_with_wait_setup", ["helm_local_install"] do |_, args| + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true) end desc "Sets up sample CoreDNS CNF with source" task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true, wait_count: 0 ) end desc "Sets up an alternate sample CoreDNS CNF" task "sample_coredns", ["helm_local_install"] do |_, args| puts "sample_coredns new setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| puts "bad_helm_cnf_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 2 ) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| puts "sample_privileged_cnf_whitelisted_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| puts "sample_privileged_cnf_non_whitelisted_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| puts "sample_coredns_bad_liveness" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_coredns_cleanup" do |_, args| diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index 8ee98af71..c6c7c4ad8 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -177,11 +177,13 @@ def sample_cleanup(sample_dir, verbose=true) current_dir = FileUtils.pwd helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" puts helm if verbose - helm_uninstall = `#{helm} uninstall #{release_name}` - puts helm_uninstall if verbose destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" rm = `rm -rf #{destination_cnf_dir}` puts rm if verbose + helm_uninstall = `#{helm} uninstall #{release_name}` + ret = $? + puts helm_uninstall if verbose + ret end def chart_name(helm_chart_repo) From faa341d8b4caf54206a62d3ff08e8a068276265a Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 16:49:41 -0400 Subject: [PATCH 13/49] Add spec tests for rediness checks cncf/cnf-conformance#57 --- .../chart/templates/deployment.yaml | 9 ------- spec/configuration_lifecycle_spec.cr | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml index fd1d89aa8..edf22b182 100755 --- a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml @@ -85,15 +85,6 @@ spec: {{ toYaml .Values.resources | indent 10 }} ports: {{ include "coredns.containerPorts" . | indent 8 }} - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 volumes: - name: config-volume configMap: diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index 099b5db5b..bcfefab16 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -48,4 +48,28 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` end end + it "'readiness' should pass when readinessProbe is set", tags: "readiness" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr readiness verbose` + puts response_s + $?.success?.should be_true + (/PASSED: Helm readiness probe/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns verbose` + end + end + it "'readiness' should fail when readinessProbe is not set", tags: "readiness" do + begin + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr readiness verbose` + puts response_s + $?.success?.should be_true + (/FAILURE: No readinessProbe found/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` + end + end end From a118ba97006bba8b5e1bf8e32d03c78546e81376 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 18:35:55 -0400 Subject: [PATCH 14/49] Add setup task for sonobuoy cncf/cnf-conformance#104 --- src/tasks/sonobuoy_setup.cr | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 1e17831f5..4e84cdb53 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -3,14 +3,30 @@ require "file_utils" require "colorize" require "totem" require "./utils.cr" +require "http/client" desc "Sets up Sonobuoy in the K8s Cluster" task "install_sonobuoy" do |_, args| - response = String::Builder.new - Process.run("echo installing sonobuoy", shell: true) do |proc| - while line = proc.output.gets - response << line - puts "#{line}" if check_args(args) + TODO: Fetch version dynamically + # k8s_version = HTTP::Client.get("https://storage.googleapis.com/kubernetes-release/release/stable.txt").body.chomp.split(".")[0..1].join(".").gsub("v", "") + k8s_version = "0.17.2" + puts k8s_version if check_verbose(args) + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + unless Dir.exists?("#{current_dir}/#{TOOLS_DIR}/sonobuoy") + begin + puts "pwd? : #{current_dir}" if check_verbose(args) + puts "toolsdir : #{TOOLS_DIR}" if check_verbose(args) + puts "full path?: #{current_dir.to_s}/#{TOOLS_DIR}/sonobuoy" if check_verbose(args) + FileUtils.mkdir_p("#{current_dir}/#{TOOLS_DIR}/sonobuoy") + curl = `VERSION="#{k8s_version}" OS=linux ; curl -L "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" --output #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` + puts curl if check_verbose(args) + `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ + chmod +x #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy && \ + rm #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + puts sonobuoy if check_verbose(args) + puts `#{sonobuoy} version` if check_verbose(args) end end end From a2f15ecf9a2e2b5b51e98f7216c5a9273f3ea140 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 18:39:13 -0400 Subject: [PATCH 15/49] Add comment cncf/cnf-conformance#104 --- src/tasks/sonobuoy_setup.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 4e84cdb53..1048dc90f 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -7,7 +7,7 @@ require "http/client" desc "Sets up Sonobuoy in the K8s Cluster" task "install_sonobuoy" do |_, args| - TODO: Fetch version dynamically + #TODO: Fetch version dynamically # k8s_version = HTTP::Client.get("https://storage.googleapis.com/kubernetes-release/release/stable.txt").body.chomp.split(".")[0..1].join(".").gsub("v", "") k8s_version = "0.17.2" puts k8s_version if check_verbose(args) From 6ca1c35e5e8b245e4e7d7f79ce33e580e74f443a Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Tue, 17 Mar 2020 13:44:39 -0400 Subject: [PATCH 16/49] #cncf/cnf-conformance/issues/104 conformance tests and specs now run sonobuoy --- spec/installability_spec.cr | 2 +- spec/platform/platform_spec.cr | 11 +++++++++ src/cnf-conformance.cr | 2 +- src/tasks/cleanup.cr | 2 +- src/tasks/platform/platform.cr | 41 ++++++++++++++++++++++++++++++++++ src/tasks/sonobuoy_setup.cr | 12 +++++++++- 6 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 spec/platform/platform_spec.cr create mode 100644 src/tasks/platform/platform.cr diff --git a/spec/installability_spec.cr b/spec/installability_spec.cr index 263acc103..78f12f495 100644 --- a/spec/installability_spec.cr +++ b/spec/installability_spec.cr @@ -21,7 +21,7 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_source_setup` $?.success?.should be_true response_s = `crystal src/cnf-conformance.cr install_script_helm` - puts response_s + #puts response_s $?.success?.should be_true (/FAILURE: Helm not found in supplied install script/ =~ response_s).should_not be_nil `crystal src/cnf-conformance.cr sample_coredns_source_cleanup` diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr new file mode 100644 index 000000000..eb634af3d --- /dev/null +++ b/spec/platform/platform_spec.cr @@ -0,0 +1,11 @@ +require "./../spec_helper" +require "colorize" + +describe "Platform" do + it "'k8s_conformance' should pass if the sonobuoy tests pass" do + response_s = `crystal src/cnf-conformance.cr k8s_conformance` + puts response_s + (/PASSED: K8s conformance test has no failures/ =~ response_s).should_not be_nil + end +end + diff --git a/src/cnf-conformance.cr b/src/cnf-conformance.cr index abbcb261a..340e81906 100644 --- a/src/cnf-conformance.cr +++ b/src/cnf-conformance.cr @@ -2,7 +2,7 @@ require "sam" require "./tasks/**" desc "The CNF Conformance program enables interoperability of CNFs from multiple vendors running on top of Kubernetes supplied by different vendors. The goal is to provide an open source test suite to enable both open and closed source CNFs to demonstrate conformance and implementation of best practices." -task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity"] do |_, args| +task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity", "platform"] do |_, args| end Sam.help diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index bb9bc6673..6752f8d43 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -10,7 +10,7 @@ end task "samples_cleanup", ["sample_coredns_cleanup", "cleanup_sample_coredns", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup", "sample_coredns_bad_liveness_cleanup", "sample_coredns_source_cleanup"] do |_, args| end -task "tools_cleanup", ["helm_local_cleanup"] do |_, args| +task "tools_cleanup", ["helm_local_cleanup", "sonobuoy_cleanup"] do |_, args| end task "cleanup_all", ["cleanup_samples", "tools_cleanup"] do |_, args| diff --git a/src/tasks/platform/platform.cr b/src/tasks/platform/platform.cr new file mode 100644 index 000000000..b39a38823 --- /dev/null +++ b/src/tasks/platform/platform.cr @@ -0,0 +1,41 @@ +desc "Platform Tests" +task "platform", ["k8s_conformance"] do |_, args| +end + +desc "Does the platform pass the K8s conformance tests?" +task "k8s_conformance" do |_, args| + begin + #TODO enable full test with production mode + #sonobuoy = `sonobuoy run --wait` if PRODUCTION_MODE and not in test_mode + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + + # Clean up old results + delete = `#{sonobuoy} delete --wait` + puts delete if check_verbose(args) + + # Run the tests + testrun = `#{sonobuoy} run --wait --mode quick` + puts testrun if check_verbose(args) + + results = `results=$(#{sonobuoy} retrieve); #{sonobuoy} results $results` + puts results if check_verbose(args) + + # Grab the failed line from the results + failed_count = ((results.match(/Failed: (.*)/)).try &.[1]) + if failed_count.to_s.to_i > 0 + puts "FAILURE: K8s conformance test has #{failed_count} failure(s)!".colorize(:red) + else + puts "PASSED: K8s conformance test has no failures".colorize(:green) + end + rescue ex + puts ex.message + ex.backtrace.each do |x| + puts x + end + ensure + remove_tar = `rm *sonobuoy*.tar.gz` + puts remove_tar if check_verbose(args) + end +end diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 1048dc90f..77720f181 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -21,7 +21,7 @@ task "install_sonobuoy" do |_, args| FileUtils.mkdir_p("#{current_dir}/#{TOOLS_DIR}/sonobuoy") curl = `VERSION="#{k8s_version}" OS=linux ; curl -L "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" --output #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` puts curl if check_verbose(args) - `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ + `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ chmod +x #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy && \ rm #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" @@ -31,3 +31,13 @@ task "install_sonobuoy" do |_, args| end end +desc "Cleans up Sonobuoy" +task "sonobuoy_cleanup"do |_, args| + current_dir = FileUtils.pwd + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + delete = `#{sonobuoy} delete --wait` + puts delete if check_verbose(args) + rm = `rm -rf #{current_dir}/#{TOOLS_DIR}/sonobuoy` + puts rm if check_verbose(args) +end + From ad898fb7c7b9dd03d61b815b9b966799a815a489 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 02:44:21 -0400 Subject: [PATCH 17/49] rolling update conformance test usage: crystal src/cnf-conformance.cr rolling_update -- version_tag=latest where version_tags value is a valid release name for the image set in the helm chart's configuration i.e. for the coredns stable chart https://github.com/helm/charts/tree/master/stable/coredns#configuration the image is `coredns/coredns` and the version is `v1.6.7` but any valid version tag from docker hub https://hub.docker.com/r/coredns/coredns/tags can be used refs #7 --- src/tasks/configuration_lifecycle.cr | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 23f5b3b60..b350058c5 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -2,6 +2,7 @@ require "sam" require "file_utils" require "colorize" require "totem" +require "json" require "./utils.cr" desc "Configuration and lifecycle should be managed in a declarative manner, using ConfigMaps, Operators, or other declarative interfaces." @@ -143,3 +144,45 @@ task "retrieve_manifest" do |_, args| end end end + +desc "Test if the CNF can perform a rolling update" +task "rolling_update" do |_, args| + begin + puts "rolling_update" if check_verbose(args) + config = cnf_conformance_yml + + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" unless args.raw.includes? "version_tag" + + version_tag = args["version_tag"] + release_name = config.get("release_name").as_s + deployment_name = config.get("deployment_name").as_s + + helm_chart_values = JSON.parse(`helm get values #{release_name} -a -o json`) + puts "helm_chart_values" if check_verbose(args) + puts helm_chart_values if check_verbose(args) + image_name = helm_chart_values["image"]["repository"] + + puts "image_name: #{image_name}" if check_verbose(args) + + puts "rolling_update: setting new version" if check_verbose(args) + #do_update = `kubectl set image deployment/coredns-coredns coredns=coredns/coredns:latest --record` + puts "kubectl set image deployment/#{deployment_name} #{release_name}=#{image_name}:#{version_tag} --record" if check_verbose(args) + do_update = `kubectl set image deployment/#{deployment_name} #{release_name}=#{image_name}:#{version_tag} --record` + + # https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#rolling-update + puts "rolling_update: checking status new version" if check_verbose(args) + puts `kubectl rollout status deployment/#{deployment_name} --timeout=30s` + + if $?.success? + puts "PASSED: CNF #{deployment_name} Rolling Update Passed".colorize(:green) + else + puts "FAILURE: CNF #{deployment_name} Rolling Update Failed".colorize(:red) + end + + rescue ex + puts ex.message + ex.backtrace.each do |x| + puts x + end + end +end \ No newline at end of file From 7aa23327c09c9bc6297123db2c9c2cc06f96c5b4 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 03:11:24 -0400 Subject: [PATCH 18/49] fix bug in version check logic --- src/tasks/configuration_lifecycle.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index b350058c5..f4e728776 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -151,9 +151,11 @@ task "rolling_update" do |_, args| puts "rolling_update" if check_verbose(args) config = cnf_conformance_yml - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" unless args.raw.includes? "version_tag" + unless args.named.has_key? "version_tag" + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" + end - version_tag = args["version_tag"] + version_tag = args.named["version_tag"] release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s From d30bda36f99a037fb013aa91d8dfa3ce1a73a070 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 17:52:48 -0400 Subject: [PATCH 19/49] yml option --- .../sample-coredns-cnf/cnf-conformance.yml | 1 + src/tasks/configuration_lifecycle.cr | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index 938751cb3..b488ace90 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -8,4 +8,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns +helm_chart_image_repository_version: latest white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index f4e728776..11cac1bc9 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -151,11 +151,20 @@ task "rolling_update" do |_, args| puts "rolling_update" if check_verbose(args) config = cnf_conformance_yml - unless args.named.has_key? "version_tag" - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" + version_tag = nil + + if args.named.has_key? "version_tag" + version_tag = args.named["version_tag"] + end + + if config.has_key? "helm_chart_image_repository_version" + version_tag = config.get("helm_chart_image_repository_version").as_s + end + + unless version_tag + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" end - version_tag = args.named["version_tag"] release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s From f78da7437f40548ed4a4034113105b548b3a41a8 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:00:29 -0400 Subject: [PATCH 20/49] overide version from command line --- src/tasks/configuration_lifecycle.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 11cac1bc9..748a12fbd 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -153,14 +153,14 @@ task "rolling_update" do |_, args| version_tag = nil - if args.named.has_key? "version_tag" - version_tag = args.named["version_tag"] - end - if config.has_key? "helm_chart_image_repository_version" version_tag = config.get("helm_chart_image_repository_version").as_s end + if args.named.has_key? "version_tag" + version_tag = args.named["version_tag"] + end + unless version_tag raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" end From 7a1f8a067f5fabb1d55c5181d78c38d55f2c3cb4 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 25 Mar 2020 18:02:58 -0400 Subject: [PATCH 21/49] #cncf/cnf-conformance/issues/104 sonobuoy now not part of 'all' tests --- .travis.yml | 2 +- src/cnf-conformance.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6dd975b83..3c864b2a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,4 @@ jobs: # Set KUBECONFIG environment variable - export KUBECONFIG="$(kind get kubeconfig-path)" - script: crystal spec + script: crystal spec -v diff --git a/src/cnf-conformance.cr b/src/cnf-conformance.cr index 340e81906..abbcb261a 100644 --- a/src/cnf-conformance.cr +++ b/src/cnf-conformance.cr @@ -2,7 +2,7 @@ require "sam" require "./tasks/**" desc "The CNF Conformance program enables interoperability of CNFs from multiple vendors running on top of Kubernetes supplied by different vendors. The goal is to provide an open source test suite to enable both open and closed source CNFs to demonstrate conformance and implementation of best practices." -task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity", "platform"] do |_, args| +task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity"] do |_, args| end Sam.help From 25ecd3b7ade3930913edd1785f22274bf57d7ca6 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:36:49 -0400 Subject: [PATCH 22/49] tests --- .../sample_coredns/cnf-conformance.yml | 1 + spec/configuration_lifecycle_spec.cr | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index 8abdde7b4..b84d6da1b 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -7,4 +7,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns-coredns +helm_chart_image_repository_version: latest white_list_helm_chart_container_names: [] diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index bcfefab16..719fdb0ae 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -72,4 +72,28 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` end end + it "'rolling_update' should pass when valid version is given", tags: "rolling_update" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr rolling_update verbose` + puts response_s + $?.success?.should be_true + (/Rolling Update Passed/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns` + end + end + it "'rolling_update' should fail when invalid version is given", tags: "rolling_update" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr rolling_update verbose version_tag=this_is_not_real_version` + puts response_s + $?.success?.should be_true + (/Rolling Update Failed/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns` + end + end end From dcc94a171120b8630ec8f7a4a846d5b076489994 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 25 Mar 2020 18:41:12 -0400 Subject: [PATCH 23/49] #cncf/cnf-conformance/issues/104 sonobuoy now has a before task --- spec/platform/platform_spec.cr | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr index eb634af3d..5fb06cef6 100644 --- a/spec/platform/platform_spec.cr +++ b/spec/platform/platform_spec.cr @@ -2,6 +2,16 @@ require "./../spec_helper" require "colorize" describe "Platform" do + before_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr setup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_with_wait_setup` + $?.success?.should be_true + end it "'k8s_conformance' should pass if the sonobuoy tests pass" do response_s = `crystal src/cnf-conformance.cr k8s_conformance` puts response_s From 4590d76dd4ef32d7a984d0776b6764abc9bda8ca Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:46:05 -0400 Subject: [PATCH 24/49] change option name --- sample-cnfs/sample-coredns-cnf/cnf-conformance.yml | 2 +- sample-cnfs/sample_coredns/cnf-conformance.yml | 2 +- src/tasks/configuration_lifecycle.cr | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index b488ace90..5cf6e6ef8 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -8,5 +8,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns -helm_chart_image_repository_version: latest +cnf_image_version: latest white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index b84d6da1b..6ee417b27 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -7,5 +7,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns-coredns -helm_chart_image_repository_version: latest +cnf_image_version: latest white_list_helm_chart_container_names: [] diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 748a12fbd..cdf782fb5 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -153,8 +153,8 @@ task "rolling_update" do |_, args| version_tag = nil - if config.has_key? "helm_chart_image_repository_version" - version_tag = config.get("helm_chart_image_repository_version").as_s + if config.has_key? "cnf_image_version" + version_tag = config.get("cnf_image_version").as_s end if args.named.has_key? "version_tag" @@ -162,7 +162,7 @@ task "rolling_update" do |_, args| end unless version_tag - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'cnf_image_version'" end release_name = config.get("release_name").as_s From 3ba876b9b435e5bdce31c1129dff9cedc765bae0 Mon Sep 17 00:00:00 2001 From: William Harris Date: Thu, 26 Mar 2020 16:38:14 -0400 Subject: [PATCH 25/49] use helm installed by setup should fix test in travis and pair box issue was using the path helm which was version 2 versus the one the setup installs in tools. also made the output option backwards compatible w helm 2 anyway refs #71 --- src/tasks/configuration_lifecycle.cr | 2 +- src/tasks/utils.cr | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index cdf782fb5..43bda4b9a 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -168,7 +168,7 @@ task "rolling_update" do |_, args| release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s - helm_chart_values = JSON.parse(`helm get values #{release_name} -a -o json`) + helm_chart_values = JSON.parse(`#{tools_helm} get values #{release_name} -a --output json`) puts "helm_chart_values" if check_verbose(args) puts helm_chart_values if check_verbose(args) image_name = helm_chart_values["image"]["repository"] diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index c6c7c4ad8..26e388675 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -170,6 +170,11 @@ def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_dir end end +def tools_helm + current_dir = FileUtils.pwd + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" +end + def sample_cleanup(sample_dir, verbose=true) config = sample_conformance_yml(sample_dir) release_name = config.get("release_name").as_s From afa87cad44b5ff8bf9bac9b40e9ed7c738fc1561 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 1 Apr 2020 17:56:19 -0400 Subject: [PATCH 26/49] Don't run full sonobuoy test for spec cncf/cnf-conformance#104 --- src/tasks/platform/platform.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/tasks/platform/platform.cr b/src/tasks/platform/platform.cr index b39a38823..3bb3516b9 100644 --- a/src/tasks/platform/platform.cr +++ b/src/tasks/platform/platform.cr @@ -16,7 +16,14 @@ task "k8s_conformance" do |_, args| puts delete if check_verbose(args) # Run the tests - testrun = `#{sonobuoy} run --wait --mode quick` + #TODO when in test mode --mode quick, prod mode no quick + testrun = "" + puts ENV["CRYSTAL_ENV"]? if check_verbose(args) + if ENV["CRYSTAL_ENV"]? == "TEST" + testrun = `#{sonobuoy} run --wait --mode quick` + else + testrun = `#{sonobuoy} run --wait` + end puts testrun if check_verbose(args) results = `results=$(#{sonobuoy} retrieve); #{sonobuoy} results $results` From c78a124213308802d547e4b6212e27fa1e55ca40 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 1 Apr 2020 17:57:27 -0400 Subject: [PATCH 27/49] Set test mode when running spec cncf/cnf-conformance#104 --- spec/spec_helper.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 548dd0a20..35e7fa4ae 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,2 +1,4 @@ require "spec" require "../src/cnf_conformance" + +ENV["CRYSTAL_ENV"] = "TEST" From 3815cb5b0fee4277df0d524a8055842934799d2f Mon Sep 17 00:00:00 2001 From: wwatson Date: Thu, 2 Apr 2020 12:42:00 -0500 Subject: [PATCH 28/49] cncf/cnf-conformance#10 Platform test documentation cncf/cnf-conformance#10 Platform test documentation --- USAGE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/USAGE.md b/USAGE.md index 93e754463..17a08e4ab 100644 --- a/USAGE.md +++ b/USAGE.md @@ -218,5 +218,15 @@ crystal src/cnf-conformance.cr direct_hugepages ``` crystal src/cnf-conformance.cr performance ``` + +## Platform Tests +#### (PoC) Run all platform tests +``` +crystal src/cnf-conformance.cr platform +``` +#### (PoC) Run the K8s conformance tests +``` +crystal src/cnf-conformance.cr k8s_conformance +``` From 0fa63f98845fa25434135338ae6ff4a090c5524c Mon Sep 17 00:00:00 2001 From: wolfpack Date: Fri, 13 Mar 2020 19:28:03 -0400 Subject: [PATCH 29/49] #cncf/cnf-conformance/issues/56 sample_coredns now uses generic sample setup code --- .../sample-coredns-cnf-source/README.md | 39 ++++ .../cnf-conformance.yml | 11 ++ .../sample-coredns-cnf/cnf-conformance.yml | 4 +- .../sample_coredns/cnf-conformance.yml | 6 +- .../cnf-conformance.yml | 6 +- spec/cnf_conformance_spec.cr | 18 -- spec/configuration_lifecycle_spec.cr | 14 +- spec/installability_spec.cr | 59 ++++++ spec/sample_coredns_setup_spec.cr | 17 +- src/tasks/configuration_lifecycle.cr | 26 ++- src/tasks/installability.cr | 22 ++- src/tasks/sample_coredns_setup.cr | 184 ++---------------- src/tasks/utils.cr | 22 ++- 13 files changed, 217 insertions(+), 211 deletions(-) create mode 100644 sample-cnfs/sample-coredns-cnf-source/README.md create mode 100644 sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml create mode 100644 spec/installability_spec.cr diff --git a/sample-cnfs/sample-coredns-cnf-source/README.md b/sample-cnfs/sample-coredns-cnf-source/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf-source/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml new file mode 100644 index 000000000..5e16eb8f5 --- /dev/null +++ b/sample-cnfs/sample-coredns-cnf-source/cnf-conformance.yml @@ -0,0 +1,11 @@ +--- +helm_directory: cnfs/coredns/helm_chart/coredns +# helm_directory: helm_chart +git_clone_url: https://github.com/coredns/coredns.git +install_script: coredns/Makefile +release_name: coredns +deployment_name: coredns-coredns +application_deployment_names: [coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: coredns +white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index 54431ae1d..938751cb3 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -1,8 +1,8 @@ --- helm_directory: cnfs/coredns/helm_chart/coredns # helm_directory: helm_chart -git_clone_url: https://github.com/coredns/coredns.git -install_script: cnfs/coredns/Makefile +git_clone_url: +install_script: coredns/Makefile release_name: coredns deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index b3d011943..8abdde7b4 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/sample_coredns/chart -install_script: cnfs/sample_coredns/chart +helm_directory: chart +git_clone_url: +install_script: chart +release_name: coredns deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns diff --git a/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml index 88fb77cc6..9a3c51d33 100644 --- a/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns_bad_liveness/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/sample_coredns_bad_liveness/chart -install_script: cnfs/sample_coredns_bad_liveness/chart +helm_directory: chart +git_clone_url: +install_script: chart +release_name: bad-liveness deployment_name: bad-liveness-coredns application_deployment_names: [bad-liveness-coredns] helm_chart: stable/coredns diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_spec.cr index f8dc39a0a..f77c8cc23 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_spec.cr @@ -28,8 +28,6 @@ describe CnfConformance do $?.success?.should be_true (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil - (/FAILURE: Helm not found in install script/ =~ response_s).should_not be_nil - (/FAILURE: IP addresses found/ =~ response_s).should_not be_nil (/Lint Passed/ =~ response_s).should_not be_nil (/PASSED: Replicas increased to 3/ =~ response_s).should_not be_nil (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil @@ -45,21 +43,5 @@ describe CnfConformance do (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil end - it "'helm_chart_valid' should fail on a bad helm chart" do - # puts `pwd` - # puts `echo $KUBECONFIG` - `crystal src/cnf-conformance.cr sample_coredns_cleanup` - $?.success?.should be_true - `crystal src/cnf-conformance.cr bad_helm_cnf_setup` - $?.success?.should be_true - response_s = `crystal src/cnf-conformance.cr helm_chart_valid` - puts response_s - $?.success?.should be_true - (/Lint Failed/ =~ response_s).should_not be_nil - `crystal src/cnf-conformance.cr bad_helm_cnf_cleanup` - $?.success?.should be_true - `crystal src/cnf-conformance.cr sample_coredns_setup` - $?.success?.should be_true - end end diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index 3cc122f80..099b5db5b 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -12,6 +12,18 @@ describe CnfConformance do # `crystal src/cnf-conformance.cr setup` # $?.success?.should be_true end + it "'ip_addresses' should fail when ip addresses are found in source is set", tags: "liveness" do + begin + `crystal src/cnf-conformance.cr sample_coredns_source_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr ip_addresses verbose` + puts response_s + $?.success?.should be_true + (/FAILURE: IP addresses found/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr sample_coredns_source_cleanup verbose` + end + end it "'liveness' should pass when livenessProbe is set", tags: "liveness" do begin `crystal src/cnf-conformance.cr sample_coredns` @@ -21,7 +33,7 @@ describe CnfConformance do $?.success?.should be_true (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil ensure - `crystal src/cnf-conformance.cr cleanup_sample_coredns` + `crystal src/cnf-conformance.cr cleanup_sample_coredns verbose` end end it "'liveness' should fail when livenessProbe is not set", tags: "liveness" do diff --git a/spec/installability_spec.cr b/spec/installability_spec.cr new file mode 100644 index 000000000..263acc103 --- /dev/null +++ b/spec/installability_spec.cr @@ -0,0 +1,59 @@ +require "./spec_helper" +require "colorize" + +describe CnfConformance do + before_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true + + # `crystal src/cnf-conformance.cr setup` + # $?.success?.should be_true + end + + it "'install_script_helm' should fail if install script does not have helm" do + # puts `pwd` + # puts `echo $KUBECONFIG` + # `crystal src/cnf-conformance.cr cleanup` + # $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_source_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr install_script_helm` + puts response_s + $?.success?.should be_true + (/FAILURE: Helm not found in supplied install script/ =~ response_s).should_not be_nil + `crystal src/cnf-conformance.cr sample_coredns_source_cleanup` + end + + it "'helm_chart_valid' should pass on a good helm chart" do + # puts `pwd` + # puts `echo $KUBECONFIG` + # `crystal src/cnf-conformance.cr cleanup` + # $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr helm_chart_valid` + puts response_s + $?.success?.should be_true + (/Lint Passed/ =~ response_s).should_not be_nil + end + + it "'helm_chart_valid' should fail on a bad helm chart" do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr sample_coredns_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr bad_helm_cnf_setup` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr helm_chart_valid` + puts response_s + $?.success?.should be_true + (/Lint Failed/ =~ response_s).should_not be_nil + `crystal src/cnf-conformance.cr bad_helm_cnf_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_setup` + $?.success?.should be_true + end +end diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 4f9a54247..bb61d7db7 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -45,7 +45,7 @@ describe "Utils" do (current_replicas.to_i > 0).should be_true end - it "'sample_setup' should set up a sample cnf", tags: "WIP" do + it "'sample_setup' should set up a sample cnf" do args = Sam::Args.new sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git" ) # check if directory exists @@ -55,7 +55,7 @@ describe "Utils" do sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end # - it "'sample_setup_args' should set up a sample cnf from a argument", tags: "WIP" do + it "'sample_setup_args' should set up a sample cnf from a argument" do args = Sam::Args.new sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) # check if directory exists @@ -65,7 +65,7 @@ describe "Utils" do sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) end - it "'sample_cleanup' should clean up a sample cnf from a argument", tags: "WIP" do + it "'sample_cleanup' should clean up a sample cnf from a argument" do args = Sam::Args.new sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) @@ -85,4 +85,15 @@ describe "Utils" do (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml").should be_true (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml").should be_true end + + it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do + args = Sam::Args.new + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + (cnf_conformance_dir).should eq("sample-generic-cnf") + end + + it "'sample_destination_dir' should return the full path of the potential destination cnf directory based on the source sample cnf directory", tags: "WIP" do + args = Sam::Args.new + sample_destination_dir("sample-generic-cnf").should contain("cnf-conformance/cnfs/sample-generic-cnf") + end end diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index a6e60dcf6..23f5b3b60 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -14,6 +14,7 @@ task "ip_addresses" do |_, args| cdir = FileUtils.pwd() response = String::Builder.new Dir.cd(CNF_DIR) + # TODO ignore *example*, *.md, *.txt Process.run("grep -rnw -E -o '([0-9]{1,3}[\.]){3}[0-9]{1,3}'", shell: true) do |proc| # Process.run("grep -rnw -E -o 'hithere'", shell: true) do |proc| while line = proc.output.gets @@ -49,8 +50,12 @@ task "liveness", ["retrieve_manifest"] do |_, args| puts "FAILURE: helm directory not found".colorize(:red) puts ex.message if check_args(args) end - puts "helm_directory: #{helm_directory}/manifest.yml" if check_verbose(args) - deployment = Totem.from_file "#{helm_directory}/manifest.yml" + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + puts "helm_directory: #{destination_cnf_dir}/#{helm_directory}/manifest.yml" if check_verbose(args) + deployment = Totem.from_file "#{destination_cnf_dir}/#{helm_directory}/manifest.yml" puts deployment.inspect if check_verbose(args) containers = deployment.get("spec").as_h["template"].as_h["spec"].as_h["containers"].as_a containers.each do |container| @@ -87,8 +92,12 @@ task "readiness", ["retrieve_manifest"] do |_, args| puts "FAILURE: helm directory not found".colorize(:red) puts ex.message if check_args(args) end - puts "helm_directory: #{helm_directory}/manifest.yml" if check_verbose(args) - deployment = Totem.from_file "#{helm_directory}/manifest.yml" + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + puts "helm_directory: #{destination_cnf_dir}/#{helm_directory}/manifest.yml" if check_verbose(args) + deployment = Totem.from_file "#{destination_cnf_dir}/#{helm_directory}/manifest.yml" puts deployment.inspect if check_verbose(args) containers = deployment.get("spec").as_h["template"].as_h["spec"].as_h["containers"].as_a containers.each do |container| @@ -115,10 +124,17 @@ end desc "Retrieve the manifest for the CNF's helm chart" task "retrieve_manifest" do |_, args| begin + puts "retrieve_manifest" if check_verbose(args) config = cnf_conformance_yml deployment_name = config.get("deployment_name").as_s + puts deployment_name if check_verbose(args) helm_directory = config.get("helm_directory").as_s - manifest = `kubectl get deployment #{deployment_name} -o yaml > #{helm_directory}/manifest.yml` + puts helm_directory if check_verbose(args) + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + manifest = `kubectl get deployment #{deployment_name} -o yaml > #{destination_cnf_dir}/#{helm_directory}/manifest.yml` puts manifest if check_verbose(args) rescue ex puts ex.message diff --git a/src/tasks/installability.cr b/src/tasks/installability.cr index d78f94c53..fd803acd5 100644 --- a/src/tasks/installability.cr +++ b/src/tasks/installability.cr @@ -15,9 +15,13 @@ task "install_script_helm" do |_, args| config = cnf_conformance_yml found = 0 + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) install_script = config.get("install_script").as_s response = String::Builder.new - content = File.open(install_script) do |file| + content = File.open("#{destination_cnf_dir}/#{install_script}") do |file| file.gets_to_end end # puts content @@ -25,9 +29,9 @@ task "install_script_helm" do |_, args| found = 1 end if found < 1 - puts "FAILURE: Helm not found in install script".colorize(:red) + puts "FAILURE: Helm not found in supplied install script".colorize(:red) else - puts "PASSED: Helm found in install script".colorize(:green) + puts "PASSED: Helm found in supplied install script".colorize(:green) end rescue ex puts ex.message @@ -53,14 +57,20 @@ task "helm_chart_valid", ["helm_local_install"] do |_, args| end puts "helm_directory: #{helm_directory}" if check_verbose(args) - ls_helm_directory = `ls -al #{helm_directory}` - puts "ls -al of helm_directory: #{ls_helm_directory}" if check_verbose(args) puts "helm_chart_repo: #{helm_chart_repo}" if check_verbose(args) current_dir = FileUtils.pwd puts current_dir if check_verbose(args) helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - helm_lint = `#{helm} lint #{helm_directory}` + + current_cnf_dir_short_name = cnf_conformance_dir + puts current_cnf_dir_short_name if check_verbose(args) + destination_cnf_dir = sample_destination_dir(current_cnf_dir_short_name) + puts destination_cnf_dir if check_verbose(args) + ls_helm_directory = `ls -al #{destination_cnf_dir}/#{helm_directory}` + puts "ls -al of helm_directory: #{ls_helm_directory}" if check_verbose(args) + + helm_lint = `#{helm} lint #{destination_cnf_dir}/#{helm_directory}` puts "helm_lint: #{helm_lint}" if check_verbose(args) # Process.run("helm lint #{helm_directory}", shell: true) do |proc| diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 358eb3dc7..925608e29 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -8,112 +8,17 @@ require "./utils.cr" desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Retrieve the cnf source - # TODO enable recloning/fetching etc - git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/coredns` - puts git_clone if check_verbose(args) - - # Copy the cnf-conformance.yml - yml_cp = `cp sample-cnfs/sample-coredns-cnf/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/coredns` - puts yml_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install coredns #{helm_chart}` - puts helm_install if check_verbose(args) - + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) +end - # Retrieve the helm chart source - FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/coredns/helm_chart") - helm_pull = `#{helm} pull #{helm_chart}` - puts helm_pull if check_verbose(args) - core_mv = `mv coredns-*.tgz #{current_dir}/#{CNF_DIR}/coredns/helm_chart` - puts core_mv if check_verbose(args) - tar = `cd #{current_dir}/#{CNF_DIR}/coredns/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/coredns/helm_chart/coredns-*.tgz` - puts tar if check_verbose(args) - # coredns-coredns deployment must exist before running the next line (must already be installed) - # # TODO get deployment name from previous install - # manifest = `kubectl get deployment coredns-coredns -o yaml > #{CNF_DIR}/coredns/helm_chart/coredns/manifest.yml` - # puts manifest if check_verbose(args) - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 - puts "Successfully setup coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end +desc "Sets up sample CoreDNS CNF with source" +task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) end task "sample_coredns", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns #{current_dir}/#{CNF_DIR}/` - puts chart_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_coredns new setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end @@ -285,74 +190,18 @@ task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_ end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_coredns_bad_liveness #{current_dir}/#{CNF_DIR}/` - puts chart_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install bad-liveness #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_coredns_bad_liveness".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_coredns_bad_liveness" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) end task "sample_coredns_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) end task "cleanup_sample_coredns" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_coredns", verbose: true) end @@ -387,12 +236,9 @@ task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| end task "sample_coredns_bad_liveness_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall bad-liveness` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_coredns_bad_liveness` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", verbose: true) +end +task "sample_coredns_source_cleanup" do |_, args| + sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf-source", verbose: true) end diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index d01e1f8b7..6cfebdbc6 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -25,6 +25,14 @@ def cnf_conformance_yml Totem.from_file "./#{cnf_conformance}" end +def cnf_conformance_dir + cnf_conformance = `find cnfs/* -name "cnf-conformance.yml"`.split("\n")[0] + if cnf_conformance.empty? + raise "No cnf_conformance.yml found! Did you run the setup task?" + end + cnf_conformance.split("/")[-2] +end + def sample_conformance_yml(sample_dir) cnf_conformance = `find #{sample_dir}/* -name "cnf-conformance.yml"`.split("\n")[0] if cnf_conformance.empty? @@ -48,7 +56,8 @@ def wait_for_install(deployment_name, wait_count=180) end end def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) - # # Parse the cnf-conformance.yml + puts "sample_setup_args" if verbose + config = sample_conformance_yml(sample_dir) if args.named.keys.includes? "release_name" @@ -82,7 +91,6 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) if args.named.keys.includes? "git_clone_url" git_clone_url = "#{args.named["git_clone_url"]}" else - # TODO check type (any) before doing .as_s git_clone_url = "#{config.get("git_clone_url").as_s?}" end puts "git_clone_url: #{git_clone_url}" if verbose @@ -91,12 +99,20 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) end +def sample_destination_dir(sample_source_dir) + current_dir = FileUtils.pwd + "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_source_dir)}" +end + def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false) + puts "sample_setup" if verbose current_dir = FileUtils.pwd puts current_dir if verbose - destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + # destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" + destination_cnf_dir = sample_destination_dir(sample_dir) + puts "destination_cnf_dir: #{destination_cnf_dir}" if verbose FileUtils.mkdir_p(destination_cnf_dir) # TODO enable recloning/fetching etc From ffd7481e7a0fec8fdc44585d5829cb9091dc3a73 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 00:54:27 -0400 Subject: [PATCH 30/49] #cncf/cnf-conformance/issues/56 bad helm sample cnf now generic --- .../chart/Chart.yaml | 23 +++ .../chart/README.md | 137 ++++++++++++ .../chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 124 +++++++++++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ .../chart/values.yaml | 195 ++++++++++++++++++ .../cnf-conformance.yml | 6 +- src/tasks/sample_coredns_setup.cr | 75 +------ src/tasks/utils.cr | 8 +- 22 files changed, 1136 insertions(+), 75 deletions(-) create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml new file mode 100755 index 000000000..77de0c04f --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.9.3 diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md new file mode 100755 index 000000000..dc00ca26b --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/README.md @@ -0,0 +1,137 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml new file mode 100755 index 000000000..809cc660d --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ sdfskfsdf .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ sdfskfsdf .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ sdfskfsdf .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ sdfskfsdf .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ sdfskfsdf .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + privileged: true + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ sdfskfsdf .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml new file mode 100755 index 000000000..becf13564 --- /dev/null +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/chart/values.yaml @@ -0,0 +1,195 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml index b07726780..5b37d5d6c 100644 --- a/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml @@ -1,6 +1,8 @@ --- -helm_directory: cnfs/bad_helm_coredns/helm_chart/coredns -install_script: cnfs/bad_helm_coredns/Makefile +helm_directory: chart +git_clone_url: +install_script: +release_name: bad-helm-coredns-coredns deployment_name: bad-helm-coredns-coredns application_deployment_names: [bad-helm-coredns] helm_chart: stable/coredns diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 925608e29..66a39cf3d 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -16,74 +16,17 @@ task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) end +desc "Sets up an alternate sample CoreDNS CNF" task "sample_coredns", ["helm_local_install"] do |_, args| puts "sample_coredns new setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end +desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Retrieve the cnf source - # TODO enable recloning/fetching etc - git_clone = `git clone https://github.com/coredns/coredns.git #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts git_clone if check_verbose(args) - - # Copy the cnf-conformance.yml - yml_cp = `cp sample-cnfs/sample-bad_helm_coredns-cnf/cnf-conformance.yml #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts yml_cp if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - # name must be dns compliant (no underscores) - helm_install = `#{helm} install bad-helm-coredns #{helm_chart}` - puts helm_install if check_verbose(args) - - - # Retrieve the helm chart source - FileUtils.mkdir_p("#{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart") - helm_pull = `#{helm} pull #{helm_chart}` - puts helm_pull if check_verbose(args) - core_mv = `mv coredns-*.tgz #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart` - puts core_mv if check_verbose(args) - tar = `cd #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart; tar -xvf #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart/coredns-*.tgz` - puts tar if check_verbose(args) - - #sed command that changes helm chart to be bad - helm_sed = `sed -i s/"toYaml"/"sdfsdfsdf"/g #{current_dir}/#{CNF_DIR}/bad_helm_coredns/helm_chart/coredns/templates/deployment.yaml` - puts helm_sed if check_verbose(args) - - wait_for_install(deployment_name) - - if helm_install.to_s.size > 0 && helm_pull.to_s.size > 0 - puts "Successfully setup coredns".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "bad_helm_cnf_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 5 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| @@ -206,13 +149,7 @@ end task "bad_helm_cnf_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall bad-helm-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/bad_helm_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", verbose: true) end task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index 6cfebdbc6..8ee98af71 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -55,7 +55,7 @@ def wait_for_install(deployment_name, wait_count=180) second_count = second_count + 1 end end -def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) +def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false, wait_count=180) puts "sample_setup_args" if verbose config = sample_conformance_yml(sample_dir) @@ -95,7 +95,7 @@ def sample_setup_args(sample_dir, args, deploy_with_chart=true, verbose=false) end puts "git_clone_url: #{git_clone_url}" if verbose - sample_setup(sample_dir: sample_dir, release_name: release_name, deployment_name: deployment_name, helm_chart: helm_chart, helm_directory: helm_directory, git_clone_url: git_clone_url, deploy_with_chart: deploy_with_chart, verbose: verbose ) + sample_setup(sample_dir: sample_dir, release_name: release_name, deployment_name: deployment_name, helm_chart: helm_chart, helm_directory: helm_directory, git_clone_url: git_clone_url, deploy_with_chart: deploy_with_chart, verbose: verbose, wait_count: wait_count ) end @@ -104,7 +104,7 @@ def sample_destination_dir(sample_source_dir) "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_source_dir)}" end -def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false) +def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_directory, git_clone_url="", deploy_with_chart=true, verbose=false, wait_count=180) puts "sample_setup" if verbose current_dir = FileUtils.pwd @@ -160,7 +160,7 @@ def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_dir puts helm_install if verbose end - wait_for_install(deployment_name) + wait_for_install(deployment_name, wait_count) if helm_install.to_s.size > 0 # && helm_pull.to_s.size > 0 puts "Successfully setup #{release_name}".colorize(:green) end From d4db489d1047273cba0f024bf18171ec2c7c693c Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 01:26:42 -0400 Subject: [PATCH 31/49] #cncf/cnf-conformance/issues/56 whitelisted,privileged sample cnf now generic --- .../README.md | 39 ++++ .../chart/Chart.yaml | 23 +++ .../chart/README.md | 137 ++++++++++++ .../chart/templates/NOTES.txt | 30 +++ .../chart/templates/_helpers.tpl | 149 +++++++++++++ .../templates/clusterrole-autoscaler.yaml | 35 ++++ .../chart/templates/clusterrole.yaml | 38 ++++ .../clusterrolebinding-autoscaler.yaml | 28 +++ .../chart/templates/clusterrolebinding.yaml | 24 +++ .../chart/templates/configmap-autoscaler.yaml | 34 +++ .../chart/templates/configmap.yaml | 30 +++ .../templates/deployment-autoscaler.yaml | 77 +++++++ .../chart/templates/deployment.yaml | 124 +++++++++++ .../chart/templates/podsecuritypolicy.yaml | 57 +++++ .../chart/templates/service-metrics.yaml | 33 +++ .../chart/templates/service.yaml | 38 ++++ .../templates/serviceaccount-autoscaler.yaml | 21 ++ .../chart/templates/serviceaccount.yaml | 16 ++ .../chart/templates/servicemonitor.yaml | 33 +++ .../chart/values.yaml | 195 ++++++++++++++++++ .../cnf-conformance.yml | 10 + src/tasks/sample_coredns_setup.cr | 58 +----- 22 files changed, 1174 insertions(+), 55 deletions(-) create mode 100644 sample-cnfs/sample_whitelisted_privileged_cnf/README.md create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml create mode 100755 sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml create mode 100644 sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/README.md b/sample-cnfs/sample_whitelisted_privileged_cnf/README.md new file mode 100644 index 000000000..12981cc93 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/README.md @@ -0,0 +1,39 @@ +# Set up Sample CoreDNS CNF +./sample-cnfs/sample-coredns-cnf/readme.md +# Prerequistes +### Install helm +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh +``` +### Optional: Use a helm version manager +https://github.com/yuya-takeyama/helmenv +Check out helmenv into any path (here is ${HOME}/.helmenv) +``` +${HOME}/.helmenv) +$ git clone https://github.com/yuya-takeyama/helmenv.git ~/.helmenv +``` +Add ~/.helmenv/bin to your $PATH any way you like +``` +$ echo 'export PATH="$HOME/.helmenv/bin:$PATH"' >> ~/.bash_profile +``` +``` +helmenv versions +helmenv install +``` + +### core-dns installation +``` +helm install coredns stable/coredns +``` +### Pull down the helm chart code, untar it, and put it in the cnfs/coredns directory +``` +helm pull stable/coredns +``` +### Example cnf-conformance config file for sample-core-dns-cnf +In ./cnfs/sample-core-dns-cnf/cnf-conformance.yml +``` +--- +container_names: [coredns-coredns] +``` diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml new file mode 100755 index 000000000..77de0c04f --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +appVersion: 1.6.7 +description: CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS + Services +home: https://coredns.io +icon: https://coredns.io/images/CoreDNS_Colour_Horizontal.png +keywords: +- coredns +- dns +- kubedns +maintainers: +- email: hello@acale.ph + name: Acaleph +- email: shashidhara.huawei@gmail.com + name: shashidharatd +- email: andor44@gmail.com + name: andor44 +- email: manuel@rueg.eu + name: mrueg +name: coredns +sources: +- https://github.com/coredns/coredns +version: 1.9.3 diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md new file mode 100755 index 000000000..dc00ca26b --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/README.md @@ -0,0 +1,137 @@ +# CoreDNS + +[CoreDNS](https://coredns.io/) is a DNS server that chains plugins and provides DNS Services + +# TL;DR; + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +## Introduction + +This chart bootstraps a [CoreDNS](https://github.com/coredns/coredns) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. This chart will provide DNS Services and can be deployed in multiple configuration to support various scenarios listed below: + + - CoreDNS as a cluster dns service and a drop-in replacement for Kube/SkyDNS. This is the default mode and CoreDNS is deployed as cluster-service in kube-system namespace. This mode is chosen by setting `isClusterService` to true. + - CoreDNS as an external dns service. In this mode CoreDNS is deployed as any kubernetes app in user specified namespace. The CoreDNS service can be exposed outside the cluster by using using either the NodePort or LoadBalancer type of service. This mode is chosen by setting `isClusterService` to false. + - CoreDNS as an external dns provider for kubernetes federation. This is a sub case of 'external dns service' which uses etcd plugin for CoreDNS backend. This deployment mode as a dependency on `etcd-operator` chart, which needs to be pre-installed. + +## Prerequisites + +- Kubernetes 1.10 or later + +## Installing the Chart + +The chart can be installed as follows: + +```console +$ helm install --name coredns --namespace=kube-system stable/coredns +``` + +The command deploys CoreDNS on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists various ways to override default configuration during deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete coredns +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +| Parameter | Description | Default | +|:----------------------------------------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------| +| `image.repository` | The image repository to pull from | coredns/coredns | +| `image.tag` | The image tag to pull from | `v1.6.7` | +| `image.pullPolicy` | Image pull policy | IfNotPresent | +| `replicaCount` | Number of replicas | 1 | +| `resources.limits.cpu` | Container maximum CPU | `100m` | +| `resources.limits.memory` | Container maximum memory | `128Mi` | +| `resources.requests.cpu` | Container requested CPU | `100m` | +| `resources.requests.memory` | Container requested memory | `128Mi` | +| `serviceType` | Kubernetes Service type | `ClusterIP` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | {} | +| `prometheus.monitor.namespace` | Selector to select which namespaces the Endpoints objects are discovered from. | `""` | +| `service.clusterIP` | IP address to assign to service | `""` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.externalTrafficPolicy` | Enable client source IP preservation | `[]` | +| `service.annotations` | Annotations to add to service | `{prometheus.io/scrape: "true", prometheus.io/port: "9153"}`| +| `serviceAccount.create` | If true, create & use serviceAccount | false | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `rbac.create` | If true, create & use RBAC resources | true | +| `rbac.pspEnable` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `isClusterService` | Specifies whether chart should be deployed as cluster-service or normal k8s app. | true | +| `priorityClassName` | Name of Priority Class to assign pods | `""` | +| `servers` | Configuration for CoreDNS and plugins | See values.yml | +| `affinity` | Affinity settings for pod assignment | {} | +| `nodeSelector` | Node labels for pod assignment | {} | +| `tolerations` | Tolerations for pod assignment | [] | +| `zoneFiles` | Configure custom Zone files | [] | +| `extraSecrets` | Optional array of secrets to mount inside the CoreDNS container | [] | +| `customLabels` | Optional labels for Deployment(s), Pod, Service, ServiceMonitor objects | {} | +| `autoscaler.enabled` | Optionally enabled a cluster-proportional-autoscaler for CoreDNS | `false` | +| `autoscaler.coresPerReplica` | Number of cores in the cluster per CoreDNS replica | `256` | +| `autoscaler.nodesPerReplica` | Number of nodes in the cluster per CoreDNS replica | `16` | +| `autoscaler.image.repository` | The image repository to pull autoscaler from | k8s.gcr.io/cluster-proportional-autoscaler-amd64 | +| `autoscaler.image.tag` | The image tag to pull autoscaler from | `1.7.1` | +| `autoscaler.image.pullPolicy` | Image pull policy for the autoscaler | IfNotPresent | +| `autoscaler.priorityClassName` | Optional priority class for the autoscaler pod. `priorityClassName` used if not set. | `""` | +| `autoscaler.affinity` | Affinity settings for pod assignment for autoscaler | {} | +| `autoscaler.nodeSelector` | Node labels for pod assignment for autoscaler | {} | +| `autoscaler.tolerations` | Tolerations for pod assignment for autoscaler | [] | +| `autoscaler.resources.limits.cpu` | Container maximum CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.limits.memory` | Container maximum memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.resources.requests.cpu` | Container requested CPU for cluster-proportional-autoscaler | `20m` | +| `autoscaler.resources.requests.memory` | Container requested memory for cluster-proportional-autoscaler | `10Mi` | +| `autoscaler.configmap.annotations` | Annotations to add to autoscaler config map. For example to stop CI renaming them | {} | + +See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name coredns \ + --set rbac.create=false \ + stable/coredns +``` + +The above command disables automatic creation of RBAC rules. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name coredns -f values.yaml stable/coredns +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Caveats + +The chart will automatically determine which protocols to listen on based on +the protocols you define in your zones. This means that you could potentially +use both "TCP" and "UDP" on a single port. +Some cloud environments like "GCE" or "Azure container service" cannot +create external loadbalancers with both "TCP" and "UDP" protocols. So +When deploying CoreDNS with `serviceType="LoadBalancer"` on such cloud +environments, make sure you do not attempt to use both protocols at the same +time. + +## Autoscaling + +By setting `autoscaler.enabled = true` a +[cluster-proportional-autoscaler](https://github.com/kubernetes-incubator/cluster-proportional-autoscaler) +will be deployed. This will default to a coredns replica for every 256 cores, or +16 nodes in the cluster. These can be changed with `autoscaler.coresPerReplica` +and `autoscaler.nodesPerReplica`. When cluster is using large nodes (with more +cores), `coresPerReplica` should dominate. If using small nodes, +`nodesPerReplica` should dominate. + +This also creates a ServiceAccount, ClusterRole, and ClusterRoleBinding for +the autoscaler deployment. + +`replicaCount` is ignored if this is enabled. diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt new file mode 100755 index 000000000..3a1883b3a --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/NOTES.txt @@ -0,0 +1,30 @@ +{{- if .Values.isClusterService }} +CoreDNS is now running in the cluster as a cluster-service. +{{- else }} +CoreDNS is now running in the cluster. +It can be accessed using the below endpoint +{{- if contains "NodePort" .Values.serviceType }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "coredns.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "$NODE_IP:$NODE_PORT" +{{- else if contains "LoadBalancer" .Values.serviceType }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get svc -w {{ template "coredns.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "coredns.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo $SERVICE_IP +{{- else if contains "ClusterIP" .Values.serviceType }} + "{{ template "coredns.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" + from within the cluster +{{- end }} +{{- end }} + +It can be tested with the following: + +1. Launch a Pod with DNS tools: + +kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools + +2. Query the DNS server: + +/ # host kubernetes diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl new file mode 100755 index 000000000..a2efcb43e --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "coredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "coredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.servicePorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {port: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {port: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Generate the list of ports automatically from the server definitions +*/}} +{{- define "coredns.containerPorts" -}} + {{/* Set ports to be an empty dict */}} + {{- $ports := dict -}} + {{/* Iterate through each of the server blocks */}} + {{- range .Values.servers -}} + {{/* Capture port to avoid scoping awkwardness */}} + {{- $port := toString .port -}} + + {{/* If none of the server blocks has mentioned this port yet take note of it */}} + {{- if not (hasKey $ports $port) -}} + {{- $ports := set $ports $port (dict "istcp" false "isudp" false) -}} + {{- end -}} + {{/* Retrieve the inner dict that holds the protocols for a given port */}} + {{- $innerdict := index $ports $port -}} + + {{/* + Look at each of the zones and check which protocol they serve + At the moment the following are supported by CoreDNS: + UDP: dns:// + TCP: tls://, grpc:// + */}} + {{- range .zones -}} + {{- if has (default "" .scheme) (list "dns://") -}} + {{/* Optionally enable tcp for this service as well */}} + {{- if eq .use_tcp true }} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end }} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- end -}} + + {{- if has (default "" .scheme) (list "tls://" "grpc://") -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + {{- end -}} + + {{/* If none of the zones specify scheme, default to dns:// on both tcp & udp */}} + {{- if and (not (index $innerdict "istcp")) (not (index $innerdict "isudp")) -}} + {{- $innerdict := set $innerdict "isudp" true -}} + {{- $innerdict := set $innerdict "istcp" true -}} + {{- end -}} + + {{/* Write the dict back into the outer dict */}} + {{- $ports := set $ports $port $innerdict -}} + {{- end -}} + + {{/* Write out the ports according to the info collected above */}} + {{- range $port, $innerdict := $ports -}} + {{- if index $innerdict "isudp" -}} + {{- printf "- {containerPort: %v, protocol: UDP, name: udp-%s}\n" $port $port -}} + {{- end -}} + {{- if index $innerdict "istcp" -}} + {{- printf "- {containerPort: %v, protocol: TCP, name: tcp-%s}\n" $port $port -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "coredns.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "coredns.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml new file mode 100755 index 000000000..748c62bf7 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] +# Remove the configmaps rule once below issue is fixed: +# kubernetes-incubator/cluster-proportional-autoscaler#16 + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml new file mode 100755 index 000000000..029d13e27 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +{{- if .Values.rbac.pspEnable }} +- apiGroups: + - policy + - extensions + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "coredns.fullname" . }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml new file mode 100755 index 000000000..eafb38f9e --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }}-autoscaler +subjects: +- kind: ServiceAccount + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..49da9b548 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "coredns.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "coredns.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml new file mode 100755 index 000000000..50895ae5b --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap-autoscaler.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaler.enabled }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + {{- if .Values.customLabels }} + {{- toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- if .Values.autoscaler.configmap.annotations }} + annotations: + {{- toYaml .Values.autoscaler.configmap.annotations | nindent 4 }} + {{- end }} +data: + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + linear: |- + { + "coresPerReplica": {{ .Values.autoscaler.coresPerReplica | float64 }}, + "nodesPerReplica": {{ .Values.autoscaler.nodesPerReplica | float64 }}, + "preventSinglePointFailure": true + } +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml new file mode 100755 index 000000000..b7e1a667f --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +data: + Corefile: |- + {{ range .Values.servers }} + {{- range $idx, $zone := .zones }}{{ if $idx }} {{ else }}{{ end }}{{ default "" $zone.scheme }}{{ default "." $zone.zone }}{{ else }}.{{ end -}} + {{- if .port }}:{{ .port }} {{ end -}} + { + {{- range .plugins }} + {{ .name }}{{ if .parameters }} {{ .parameters }}{{ end }}{{ if .configBlock }} { +{{ .configBlock | indent 12 }} + }{{ end }} + {{- end }} + } + {{ end }} + {{- range .Values.zoneFiles }} + {{ .filename }}: {{ toYaml .contents | indent 4 }} + {{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml new file mode 100755 index 000000000..7ca185239 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment-autoscaler.yaml @@ -0,0 +1,77 @@ +{{- if .Values.autoscaler.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 8 }} + {{- end }} + annotations: + checksum/configmap: {{ include (print $.Template.BasePath "/configmap-autoscaler.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.fullname" . }}-autoscaler + {{- $priorityClassName := default .Values.priorityClassName .Values.autoscaler.priorityClassName }} + {{- if $priorityClassName }} + priorityClassName: {{ $priorityClassName | quote }} + {{- end }} + {{- if .Values.autoscaler.affinity }} + affinity: +{{ toYaml .Values.autoscaler.affinity | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.tolerations }} + tolerations: +{{ toYaml .Values.autoscaler.tolerations | indent 8 }} + {{- end }} + {{- if .Values.autoscaler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.autoscaler.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: autoscaler + image: "{{ .Values.autoscaler.image.repository }}:{{ .Values.autoscaler.image.tag }}" + imagePullPolicy: {{ .Values.autoscaler.image.pullPolicy }} + resources: +{{ toYaml .Values.autoscaler.resources | indent 10 }} + command: + - /cluster-proportional-autoscaler + - --namespace={{ .Release.Namespace }} + - --configmap={{ template "coredns.fullname" . }}-autoscaler + - --target=Deployment/{{ template "coredns.fullname" . }} + - --logtostderr=true + - --v=2 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml new file mode 100755 index 000000000..02ed35fd8 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + {{- if not .Values.autoscaler.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + template: + metadata: + labels: + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.isClusterService }} + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + {{- end }} + spec: + serviceAccountName: {{ template "coredns.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName | quote }} + {{- end }} + {{- if .Values.isClusterService }} + dnsPolicy: Default + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + containers: + - name: "coredns" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + privileged: true + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns +{{- range .Values.extraSecrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true +{{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: +{{ include "coredns.containerPorts" . | indent 8 }} + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + volumes: + - name: config-volume + configMap: + name: {{ template "coredns.fullname" . }} + items: + - key: Corefile + path: Corefile + {{ range .Values.zoneFiles }} + - key: {{ .filename }} + path: {{ .filename }} + {{ end }} +{{- range .Values.extraSecrets }} + - name: {{ .name }} + secret: + secretName: {{ .name }} + defaultMode: 400 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..754943fe5 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/podsecuritypolicy.yaml @@ -0,0 +1,57 @@ +{{- if .Values.rbac.pspEnable }} +{{ if .Capabilities.APIVersions.Has "policy/v1beta1" }} +apiVersion: policy/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: PodSecurityPolicy +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- else }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- end }} +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # Add back CAP_NET_BIND_SERVICE so that coredns can run on port 53 + allowedCapabilities: + - CAP_NET_BIND_SERVICE + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml new file mode 100755 index 000000000..ae213c043 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service-metrics.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }}-metrics + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + ports: + - name: metrics + port: 9153 + targetPort: 9153 +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml new file mode 100755 index 000000000..4098664bb --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "coredns.fullname" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: +{{ include "coredns.servicePorts" . | indent 2 -}} + type: {{ default "ClusterIP" .Values.serviceType }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml new file mode 100755 index 000000000..972c74612 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.autoscaler.enabled .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.fullname" . }}-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name }}-autoscaler + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }}-autoscaler +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml new file mode 100755 index 000000000..bced7ca3d --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "coredns.serviceAccountName" . }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml new file mode 100755 index 000000000..0a4ffb581 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "coredns.fullname" . }} + {{- if .Values.prometheus.monitor.namespace }} + namespace: {{ .Values.prometheus.monitor.namespace }} + {{- end }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- if .Values.isClusterService }} + k8s-app: {{ .Chart.Name | quote }} + {{- end }} + app.kubernetes.io/name: {{ template "coredns.name" . }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics +{{- end }} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml new file mode 100755 index 000000000..becf13564 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/chart/values.yaml @@ -0,0 +1,195 @@ +# Default values for coredns. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: coredns/coredns + tag: "1.6.7" + pullPolicy: IfNotPresent + +replicaCount: 1 + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +serviceType: "ClusterIP" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + +service: +# clusterIP: "" +# loadBalancerIP: "" +# externalTrafficPolicy: "" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9153" + +serviceAccount: + create: false + # The name of the ServiceAccount to use + # If not set and create is true, a name is generated using the fullname template + name: + +rbac: + # If true, create & use RBAC resources + create: true + # If true, create and use PodSecurityPolicy + pspEnable: false + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + # name: + +# isClusterService specifies whether chart should be deployed as cluster-service or normal k8s app. +isClusterService: true + +# Optional priority class to be used for the coredns pods. Used for autoscaler if autoscaler.priorityClassName not set. +priorityClassName: "" + +# Default zone is what Kubernetes recommends: +# https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options +servers: +- zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + +# Complete example with all the options: +# - zones: # the `zones` block can be left out entirely, defaults to "." +# - zone: hello.world. # optional, defaults to "." +# scheme: tls:// # optional, defaults to "" (which equals "dns://" in CoreDNS) +# - zone: foo.bar. +# scheme: dns:// +# use_tcp: true # set this parameter to optionally expose the port on tcp as well as udp for the DNS protocol +# # Note that this will not work if you are also exposing tls or grpc on the same server +# port: 12345 # optional, defaults to "" (which equals 53 in CoreDNS) +# plugins: # the plugins to use for this server block +# - name: kubernetes # name of plugin, if used multiple times ensure that the plugin supports it! +# parameters: foo bar # list of parameters after the plugin +# configBlock: |- # if the plugin supports extra block style config, supply it here +# hello world +# foo bar + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core +# for example: +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: foo.bar.com/role +# operator: In +# values: +# - master +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core +# for example: +# tolerations: +# - key: foo.bar.com/role +# operator: Equal +# value: master +# effect: NoSchedule +tolerations: [] + +# configure custom zone files as per https://coredns.io/2017/05/08/custom-dns-entries-for-kubernetes/ +zoneFiles: [] +# - filename: example.db +# domain: example.com +# contents: | +# example.com. IN SOA sns.dns.icann.com. noc.dns.icann.com. 2015082541 7200 3600 1209600 3600 +# example.com. IN NS b.iana-servers.net. +# example.com. IN NS a.iana-servers.net. +# example.com. IN A 192.168.99.102 +# *.example.com. IN A 192.168.99.102 + +# optional array of secrets to mount inside coredns container +# possible usecase: need for secure connection with etcd backend +extraSecrets: [] +# - name: etcd-client-certs +# mountPath: /etc/coredns/tls/etcd +# - name: some-fancy-secret +# mountPath: /etc/wherever + +# Custom labels to apply to Deployment, Pod, Service, ServiceMonitor. Including autoscaler if enabled. +customLabels: {} + +## Configue a cluster-proportional-autoscaler for coredns +# See https://github.com/kubernetes-incubator/cluster-proportional-autoscaler +autoscaler: + # Enabled the cluster-proportional-autoscaler + enabled: false + + # Number of cores in the cluster per coredns replica + coresPerReplica: 256 + # Number of nodes in the cluster per coredns replica + nodesPerReplica: 16 + + image: + repository: k8s.gcr.io/cluster-proportional-autoscaler-amd64 + tag: "1.7.1" + pullPolicy: IfNotPresent + + # Optional priority class to be used for the autoscaler pods. priorityClassName used if not set. + priorityClassName: "" + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#affinity-v1-core + affinity: {} + + # Node labels for pod assignment + # Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # expects input structure as per specification https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#toleration-v1-core + tolerations: [] + + # resources for autoscaler pod + resources: + requests: + cpu: "20m" + memory: "10Mi" + limits: + cpu: "20m" + memory: "10Mi" + + # Options for autoscaler configmap + configmap: + ## Annotations for the coredns-autoscaler configmap + # i.e. strategy.spinnaker.io/versioned: "false" to ensure configmap isn't renamed + annotations: {} diff --git a/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml b/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml new file mode 100644 index 000000000..50bae9ec2 --- /dev/null +++ b/sample-cnfs/sample_whitelisted_privileged_cnf/cnf-conformance.yml @@ -0,0 +1,10 @@ +--- +helm_directory: chart +git_clone_url: +install_script: +release_name: privileged-coredns +deployment_name: privileged-coredns-coredns +application_deployment_names: [privileged-coredns-coredns] +helm_chart: stable/coredns +helm_chart_container_name: privileged-coredns-coredns +white_list_helm_chart_container_names: [coredns] diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 66a39cf3d..4744800bf 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -30,54 +30,8 @@ task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` - puts chart_cp if check_verbose(args) - puts yml_mv if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install privileged-coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_whitelisted_setup".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_privileged_cnf_whitelisted_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end @@ -153,13 +107,7 @@ task "bad_helm_cnf_cleanup" do |_, args| end task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall privileged-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", verbose: true) end task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| From 5230063ed79896686b3fd7831933fd415f73a7ed Mon Sep 17 00:00:00 2001 From: wolfpack Date: Sun, 15 Mar 2020 03:27:06 -0400 Subject: [PATCH 32/49] #cncf/cnf-conformance/issues/56 all samples are now generic --- .../README.md | 0 .../chart/Chart.yaml | 0 .../chart/README.md | 0 .../chart/templates/NOTES.txt | 0 .../chart/templates/_helpers.tpl | 0 .../templates/clusterrole-autoscaler.yaml | 0 .../chart/templates/clusterrole.yaml | 0 .../clusterrolebinding-autoscaler.yaml | 0 .../chart/templates/clusterrolebinding.yaml | 0 .../chart/templates/configmap-autoscaler.yaml | 0 .../chart/templates/configmap.yaml | 0 .../templates/deployment-autoscaler.yaml | 0 .../chart/templates/deployment.yaml | 0 .../chart/templates/podsecuritypolicy.yaml | 0 .../chart/templates/service-metrics.yaml | 0 .../chart/templates/service.yaml | 0 .../templates/serviceaccount-autoscaler.yaml | 0 .../chart/templates/serviceaccount.yaml | 0 .../chart/templates/servicemonitor.yaml | 0 .../chart/values.yaml | 0 .../cnf-conformance.yml | 3 +- .../non-whitelisted-conformance.yml | 8 --- .../whitelisted-conformance.yml | 8 --- spec/sample_coredns_setup_spec.cr | 10 +-- src/tasks/cleanup.cr | 2 +- src/tasks/sample_coredns_setup.cr | 67 ++----------------- 26 files changed, 11 insertions(+), 87 deletions(-) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/README.md (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/Chart.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/README.md (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/NOTES.txt (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/_helpers.tpl (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrole-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrole.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrolebinding-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/clusterrolebinding.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/configmap-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/configmap.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/deployment-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/deployment.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/podsecuritypolicy.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/service-metrics.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/service.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/serviceaccount-autoscaler.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/serviceaccount.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/templates/servicemonitor.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/chart/values.yaml (100%) rename sample-cnfs/{sample_privileged_cnf_setup_coredns => sample_privileged_cnf}/cnf-conformance.yml (70%) delete mode 100644 sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml delete mode 100644 sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/README.md b/sample-cnfs/sample_privileged_cnf/README.md similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/README.md rename to sample-cnfs/sample_privileged_cnf/README.md diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml b/sample-cnfs/sample_privileged_cnf/chart/Chart.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml rename to sample-cnfs/sample_privileged_cnf/chart/Chart.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/README.md b/sample-cnfs/sample_privileged_cnf/chart/README.md similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/README.md rename to sample-cnfs/sample_privileged_cnf/chart/README.md diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/NOTES.txt b/sample-cnfs/sample_privileged_cnf/chart/templates/NOTES.txt similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/NOTES.txt rename to sample-cnfs/sample_privileged_cnf/chart/templates/NOTES.txt diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/_helpers.tpl b/sample-cnfs/sample_privileged_cnf/chart/templates/_helpers.tpl similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/_helpers.tpl rename to sample-cnfs/sample_privileged_cnf/chart/templates/_helpers.tpl diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrole.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrole.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/clusterrolebinding.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/clusterrolebinding.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/configmap-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/configmap-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/configmap.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/configmap.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/configmap.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/deployment-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/deployment-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/deployment.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/deployment.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/deployment.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/podsecuritypolicy.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/podsecuritypolicy.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/podsecuritypolicy.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/podsecuritypolicy.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service-metrics.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/service-metrics.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service-metrics.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/service-metrics.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/service.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/service.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/service.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount-autoscaler.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount-autoscaler.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount-autoscaler.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/serviceaccount.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/serviceaccount.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/servicemonitor.yaml b/sample-cnfs/sample_privileged_cnf/chart/templates/servicemonitor.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/templates/servicemonitor.yaml rename to sample-cnfs/sample_privileged_cnf/chart/templates/servicemonitor.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/chart/values.yaml b/sample-cnfs/sample_privileged_cnf/chart/values.yaml similarity index 100% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/chart/values.yaml rename to sample-cnfs/sample_privileged_cnf/chart/values.yaml diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml b/sample-cnfs/sample_privileged_cnf/cnf-conformance.yml similarity index 70% rename from sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml rename to sample-cnfs/sample_privileged_cnf/cnf-conformance.yml index adcd43cbb..6f42928a5 100644 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_privileged_cnf/cnf-conformance.yml @@ -1,8 +1,7 @@ --- -# helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart helm_directory: chart git_clone_url: -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart +install_script: release_name: privileged-coredns deployment_name: privileged-coredns-coredns application_deployment_names: [privileged-coredns-coredns] diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml b/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml deleted file mode 100644 index be15c9da8..000000000 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart -deployment_name: privileged-coredns-coredns -application_deployment_names: [privileged-coredns-coredns] -helm_chart: stable/coredns -helm_chart_container_name: privileged-coredns-coredns -white_list_helm_chart_container_names: [] diff --git a/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml b/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml deleted file mode 100644 index 73d9f96ec..000000000 --- a/sample-cnfs/sample_privileged_cnf_setup_coredns/whitelisted-conformance.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -helm_directory: cnfs/sample_privileged_cnf_setup_coredns/chart -install_script: cnfs/sample_privileged_cnf_setup_coredns/chart -deployment_name: privileged-coredns-coredns -application_deployment_names: [privileged-coredns-coredns] -helm_chart: stable/coredns -helm_chart_container_name: privileged-coredns-coredns -white_list_helm_chart_container_names: [coredns] diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index bb61d7db7..87e6ba01c 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -77,13 +77,13 @@ describe "Utils" do it "'sample_setup_args' should be able to deploy using a helm_directory" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf_setup_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) # check if directory exists - (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns").should be_true + (Dir.exists? "cnfs/sample_privileged_cnf").should be_true # should not clone - (Dir.exists? "cnfs/sample_privileged_cnf_setup_coredns/privileged-coredns").should be_false - (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/cnf-conformance.yml").should be_true - (File.exists? "cnfs/sample_privileged_cnf_setup_coredns/chart/Chart.yaml").should be_true + (Dir.exists? "cnfs/sample_privileged_cnf/privileged-coredns").should be_false + (File.exists? "cnfs/sample_privileged_cnf/cnf-conformance.yml").should be_true + (File.exists? "cnfs/sample_privileged_cnf/chart/Chart.yaml").should be_true end it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index f2d922e21..bb9bc6673 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -7,7 +7,7 @@ desc "Cleans up the CNF Conformance test suite, the K8s cluster, and upstream pr task "cleanup", ["samples_cleanup"] do |_, args| end -task "samples_cleanup", ["sample_coredns_cleanup", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup"] do |_, args| +task "samples_cleanup", ["sample_coredns_cleanup", "cleanup_sample_coredns", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup", "sample_coredns_bad_liveness_cleanup", "sample_coredns_source_cleanup"] do |_, args| end task "tools_cleanup", ["helm_local_cleanup"] do |_, args| diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 4744800bf..6df3a1273 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -2,10 +2,8 @@ require "sam" require "file_utils" require "colorize" require "totem" -# require "commander" require "./utils.cr" - desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) @@ -22,11 +20,10 @@ task "sample_coredns", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) end - desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| puts "bad_helm_cnf_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 5 ) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 2 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| @@ -34,56 +31,9 @@ task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, ar sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end - task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| - current_dir = FileUtils.pwd - puts current_dir if check_verbose(args) - - # Copy the chart into the cnfs directory and use the correct cnf-conformance.yml - chart_cp = `cp -r #{current_dir}/sample-cnfs/sample_privileged_cnf_setup_coredns #{current_dir}/#{CNF_DIR}/` - yml_mv = `mv #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/non-whitelisted-conformance.yml #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns/cnf-conformance.yml` - puts chart_cp if check_verbose(args) - puts yml_mv if check_verbose(args) - - # Parse the cnf-conformance.yml - config = cnf_conformance_yml - - if args.named.keys.includes? "deployment_name" - deployment_name = args.named["deployment_name"] - else - deployment_name = config.get("deployment_name").as_s - end - puts "deployment_name: #{deployment_name}" if check_verbose(args) - - if args.named.keys.includes? "helm_chart" - helm_chart = args.named["helm_chart"] - else - helm_chart = config.get("helm_chart").as_s - end - puts "helm_chart: #{helm_chart}" if check_verbose(args) - - if args.named.keys.includes? "helm_directory" - helm_directory = args.named["helm_directory"] - else - helm_directory = config.get("helm_directory").as_s - end - puts "helm_directory: #{helm_directory}" if check_verbose(args) - - begin - - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_install = `#{helm} install privileged-coredns #{helm_directory}` - puts helm_install if check_verbose(args) - - wait_for_install(deployment_name) - if helm_install.to_s.size > 0 - puts "Successfully setup sample_privileged_cnf_non_whitelisted_setup".colorize(:green) - end - ensure - cd = `cd #{current_dir}` - puts cd if check_verbose(args) - end + puts "sample_privileged_cnf_non_whitelisted_setup" if check_verbose(args) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| @@ -91,17 +41,14 @@ task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) end - task "sample_coredns_cleanup" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample-coredns-cnf", verbose: true) end - task "cleanup_sample_coredns" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample_coredns", verbose: true) end - task "bad_helm_cnf_cleanup" do |_, args| sample_cleanup(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", verbose: true) end @@ -111,13 +58,7 @@ task "sample_privileged_cnf_whitelisted_cleanup" do |_, args| end task "sample_privileged_cnf_non_whitelisted_cleanup" do |_, args| - current_dir = FileUtils.pwd - helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" - puts helm if check_verbose(args) - helm_uninstall = `#{helm} uninstall privileged-coredns` - puts helm_uninstall if check_verbose(args) - rm = `rm -rf #{current_dir}/#{CNF_DIR}/sample_privileged_cnf_setup_coredns` - puts rm if check_verbose(args) + sample_cleanup(sample_dir: "sample-cnfs/sample_privileged_cnf", verbose: true) end task "sample_coredns_bad_liveness_cleanup" do |_, args| From a40ef89e5190591d4bab4f2346f4c72a82c2f4f4 Mon Sep 17 00:00:00 2001 From: wolfpack Date: Mon, 16 Mar 2020 13:09:57 -0400 Subject: [PATCH 33/49] Wait times now reduced for Sample CNFS --- spec/cnf_conformance_spec.cr | 4 ++-- spec/sample_coredns_setup_spec.cr | 15 +++++++-------- src/tasks/sample_coredns_setup.cr | 18 +++++++++++------- src/tasks/utils.cr | 6 ++++-- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_spec.cr index f77c8cc23..2b9dc4dd0 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_spec.cr @@ -9,6 +9,8 @@ describe CnfConformance do $?.success?.should be_true `crystal src/cnf-conformance.cr setup` $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_with_wait_setup` + $?.success?.should be_true end after_all do @@ -21,8 +23,6 @@ describe CnfConformance do it "'all' should run the whole test suite" do # puts `pwd` # puts `echo $KUBECONFIG` - `crystal src/cnf-conformance.cr sample_coredns_setup` - $?.success?.should be_true response_s = `crystal src/cnf-conformance.cr all verbose` puts response_s $?.success?.should be_true diff --git a/spec/sample_coredns_setup_spec.cr b/spec/sample_coredns_setup_spec.cr index 87e6ba01c..349c6ce18 100644 --- a/spec/sample_coredns_setup_spec.cr +++ b/spec/sample_coredns_setup_spec.cr @@ -47,7 +47,7 @@ describe "Utils" do it "'sample_setup' should set up a sample cnf" do args = Sam::Args.new - sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git" ) + sample_setup(sample_dir: "sample-cnfs/sample-generic-cnf", release_name: "coredns", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "helm_chart", git_clone_url: "https://github.com/coredns/coredns.git", wait_count: 0 ) # check if directory exists (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true @@ -57,7 +57,7 @@ describe "Utils" do # it "'sample_setup_args' should set up a sample cnf from a argument" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) # check if directory exists (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_true @@ -67,9 +67,9 @@ describe "Utils" do it "'sample_cleanup' should clean up a sample cnf from a argument" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) - sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) - # check if directory exists + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) + cleanup = sample_cleanup(sample_dir: "sample-cnfs/sample-generic-cnf", verbose: true) + (cleanup.success?).should be_true (Dir.exists? "cnfs/sample-generic-cnf").should be_false (File.exists?("cnfs/sample-generic-cnf/cnf-conformance.yml")).should be_false (File.exists?("cnfs/sample-generic-cnf/helm_chart/Chart.yaml")).should be_false @@ -77,8 +77,7 @@ describe "Utils" do it "'sample_setup_args' should be able to deploy using a helm_directory" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) - # check if directory exists + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) (Dir.exists? "cnfs/sample_privileged_cnf").should be_true # should not clone (Dir.exists? "cnfs/sample_privileged_cnf/privileged-coredns").should be_false @@ -88,7 +87,7 @@ describe "Utils" do it "'cnf_conformance_yml' should return the short name of the destination cnf directory", tags: "WIP" do args = Sam::Args.new - sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) (cnf_conformance_dir).should eq("sample-generic-cnf") end diff --git a/src/tasks/sample_coredns_setup.cr b/src/tasks/sample_coredns_setup.cr index 6df3a1273..c9930755c 100644 --- a/src/tasks/sample_coredns_setup.cr +++ b/src/tasks/sample_coredns_setup.cr @@ -6,39 +6,43 @@ require "./utils.cr" desc "Sets up sample CoreDNS CNF" task "sample_coredns_setup", ["helm_local_install"] do |_, args| - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true, wait_count: 0 ) +end + +task "sample_coredns_with_wait_setup", ["helm_local_install"] do |_, args| + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf", args: args, verbose: true) end desc "Sets up sample CoreDNS CNF with source" task "sample_coredns_source_setup", ["helm_local_install"] do |_, args| - sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample-coredns-cnf-source", args: args, verbose: true, wait_count: 0 ) end desc "Sets up an alternate sample CoreDNS CNF" task "sample_coredns", ["helm_local_install"] do |_, args| puts "sample_coredns new setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end desc "Sets up a Bad helm CNF Setup" task "bad_helm_cnf_setup", ["helm_local_install"] do |_, args| puts "bad_helm_cnf_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 2 ) + sample_setup_args(sample_dir: "sample-cnfs/sample-bad_helm_coredns-cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_privileged_cnf_whitelisted_setup", ["helm_local_install"] do |_, args| puts "sample_privileged_cnf_whitelisted_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_whitelisted_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_privileged_cnf_non_whitelisted_setup", ["helm_local_install"] do |_, args| puts "sample_privileged_cnf_non_whitelisted_setup" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_coredns_bad_liveness", ["helm_local_install"] do |_, args| puts "sample_coredns_bad_liveness" if check_verbose(args) - sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true ) + sample_setup_args(sample_dir: "sample-cnfs/sample_coredns_bad_liveness", deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) end task "sample_coredns_cleanup" do |_, args| diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index 8ee98af71..c6c7c4ad8 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -177,11 +177,13 @@ def sample_cleanup(sample_dir, verbose=true) current_dir = FileUtils.pwd helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" puts helm if verbose - helm_uninstall = `#{helm} uninstall #{release_name}` - puts helm_uninstall if verbose destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{short_sample_dir(sample_dir)}" rm = `rm -rf #{destination_cnf_dir}` puts rm if verbose + helm_uninstall = `#{helm} uninstall #{release_name}` + ret = $? + puts helm_uninstall if verbose + ret end def chart_name(helm_chart_repo) From 5dfdbe3578ae73a8eeeb78b12ca74f27d6ea845a Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 16:49:41 -0400 Subject: [PATCH 34/49] Add spec tests for rediness checks cncf/cnf-conformance#57 --- .../chart/templates/deployment.yaml | 9 ------- spec/configuration_lifecycle_spec.cr | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml index fd1d89aa8..edf22b182 100755 --- a/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml +++ b/sample-cnfs/sample_coredns_bad_liveness/chart/templates/deployment.yaml @@ -85,15 +85,6 @@ spec: {{ toYaml .Values.resources | indent 10 }} ports: {{ include "coredns.containerPorts" . | indent 8 }} - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 volumes: - name: config-volume configMap: diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index 099b5db5b..bcfefab16 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -48,4 +48,28 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` end end + it "'readiness' should pass when readinessProbe is set", tags: "readiness" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr readiness verbose` + puts response_s + $?.success?.should be_true + (/PASSED: Helm readiness probe/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns verbose` + end + end + it "'readiness' should fail when readinessProbe is not set", tags: "readiness" do + begin + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr readiness verbose` + puts response_s + $?.success?.should be_true + (/FAILURE: No readinessProbe found/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` + end + end end From 5e9279d077170c9dd4e8d1453bd5f560b9eeadaf Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 18:35:55 -0400 Subject: [PATCH 35/49] Add setup task for sonobuoy cncf/cnf-conformance#104 --- src/tasks/sonobuoy_setup.cr | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 1e17831f5..4e84cdb53 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -3,14 +3,30 @@ require "file_utils" require "colorize" require "totem" require "./utils.cr" +require "http/client" desc "Sets up Sonobuoy in the K8s Cluster" task "install_sonobuoy" do |_, args| - response = String::Builder.new - Process.run("echo installing sonobuoy", shell: true) do |proc| - while line = proc.output.gets - response << line - puts "#{line}" if check_args(args) + TODO: Fetch version dynamically + # k8s_version = HTTP::Client.get("https://storage.googleapis.com/kubernetes-release/release/stable.txt").body.chomp.split(".")[0..1].join(".").gsub("v", "") + k8s_version = "0.17.2" + puts k8s_version if check_verbose(args) + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + unless Dir.exists?("#{current_dir}/#{TOOLS_DIR}/sonobuoy") + begin + puts "pwd? : #{current_dir}" if check_verbose(args) + puts "toolsdir : #{TOOLS_DIR}" if check_verbose(args) + puts "full path?: #{current_dir.to_s}/#{TOOLS_DIR}/sonobuoy" if check_verbose(args) + FileUtils.mkdir_p("#{current_dir}/#{TOOLS_DIR}/sonobuoy") + curl = `VERSION="#{k8s_version}" OS=linux ; curl -L "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" --output #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` + puts curl if check_verbose(args) + `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ + chmod +x #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy && \ + rm #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + puts sonobuoy if check_verbose(args) + puts `#{sonobuoy} version` if check_verbose(args) end end end From c8036209196233bce3a380fd9f83e8e099251a7f Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 16 Mar 2020 18:39:13 -0400 Subject: [PATCH 36/49] Add comment cncf/cnf-conformance#104 --- src/tasks/sonobuoy_setup.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 4e84cdb53..1048dc90f 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -7,7 +7,7 @@ require "http/client" desc "Sets up Sonobuoy in the K8s Cluster" task "install_sonobuoy" do |_, args| - TODO: Fetch version dynamically + #TODO: Fetch version dynamically # k8s_version = HTTP::Client.get("https://storage.googleapis.com/kubernetes-release/release/stable.txt").body.chomp.split(".")[0..1].join(".").gsub("v", "") k8s_version = "0.17.2" puts k8s_version if check_verbose(args) From 2cb5cb49497745474d2b492daa95311c1513e4d8 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Tue, 17 Mar 2020 13:44:39 -0400 Subject: [PATCH 37/49] #cncf/cnf-conformance/issues/104 conformance tests and specs now run sonobuoy --- spec/installability_spec.cr | 2 +- spec/platform/platform_spec.cr | 11 +++++++++ src/cnf-conformance.cr | 2 +- src/tasks/cleanup.cr | 2 +- src/tasks/platform/platform.cr | 41 ++++++++++++++++++++++++++++++++++ src/tasks/sonobuoy_setup.cr | 12 +++++++++- 6 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 spec/platform/platform_spec.cr create mode 100644 src/tasks/platform/platform.cr diff --git a/spec/installability_spec.cr b/spec/installability_spec.cr index 263acc103..78f12f495 100644 --- a/spec/installability_spec.cr +++ b/spec/installability_spec.cr @@ -21,7 +21,7 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_source_setup` $?.success?.should be_true response_s = `crystal src/cnf-conformance.cr install_script_helm` - puts response_s + #puts response_s $?.success?.should be_true (/FAILURE: Helm not found in supplied install script/ =~ response_s).should_not be_nil `crystal src/cnf-conformance.cr sample_coredns_source_cleanup` diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr new file mode 100644 index 000000000..eb634af3d --- /dev/null +++ b/spec/platform/platform_spec.cr @@ -0,0 +1,11 @@ +require "./../spec_helper" +require "colorize" + +describe "Platform" do + it "'k8s_conformance' should pass if the sonobuoy tests pass" do + response_s = `crystal src/cnf-conformance.cr k8s_conformance` + puts response_s + (/PASSED: K8s conformance test has no failures/ =~ response_s).should_not be_nil + end +end + diff --git a/src/cnf-conformance.cr b/src/cnf-conformance.cr index abbcb261a..340e81906 100644 --- a/src/cnf-conformance.cr +++ b/src/cnf-conformance.cr @@ -2,7 +2,7 @@ require "sam" require "./tasks/**" desc "The CNF Conformance program enables interoperability of CNFs from multiple vendors running on top of Kubernetes supplied by different vendors. The goal is to provide an open source test suite to enable both open and closed source CNFs to demonstrate conformance and implementation of best practices." -task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity"] do |_, args| +task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity", "platform"] do |_, args| end Sam.help diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index bb9bc6673..6752f8d43 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -10,7 +10,7 @@ end task "samples_cleanup", ["sample_coredns_cleanup", "cleanup_sample_coredns", "bad_helm_cnf_cleanup", "sample_privileged_cnf_non_whitelisted_cleanup", "sample_privileged_cnf_whitelisted_cleanup", "sample_coredns_bad_liveness_cleanup", "sample_coredns_source_cleanup"] do |_, args| end -task "tools_cleanup", ["helm_local_cleanup"] do |_, args| +task "tools_cleanup", ["helm_local_cleanup", "sonobuoy_cleanup"] do |_, args| end task "cleanup_all", ["cleanup_samples", "tools_cleanup"] do |_, args| diff --git a/src/tasks/platform/platform.cr b/src/tasks/platform/platform.cr new file mode 100644 index 000000000..b39a38823 --- /dev/null +++ b/src/tasks/platform/platform.cr @@ -0,0 +1,41 @@ +desc "Platform Tests" +task "platform", ["k8s_conformance"] do |_, args| +end + +desc "Does the platform pass the K8s conformance tests?" +task "k8s_conformance" do |_, args| + begin + #TODO enable full test with production mode + #sonobuoy = `sonobuoy run --wait` if PRODUCTION_MODE and not in test_mode + current_dir = FileUtils.pwd + puts current_dir if check_verbose(args) + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + + # Clean up old results + delete = `#{sonobuoy} delete --wait` + puts delete if check_verbose(args) + + # Run the tests + testrun = `#{sonobuoy} run --wait --mode quick` + puts testrun if check_verbose(args) + + results = `results=$(#{sonobuoy} retrieve); #{sonobuoy} results $results` + puts results if check_verbose(args) + + # Grab the failed line from the results + failed_count = ((results.match(/Failed: (.*)/)).try &.[1]) + if failed_count.to_s.to_i > 0 + puts "FAILURE: K8s conformance test has #{failed_count} failure(s)!".colorize(:red) + else + puts "PASSED: K8s conformance test has no failures".colorize(:green) + end + rescue ex + puts ex.message + ex.backtrace.each do |x| + puts x + end + ensure + remove_tar = `rm *sonobuoy*.tar.gz` + puts remove_tar if check_verbose(args) + end +end diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index 1048dc90f..77720f181 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -21,7 +21,7 @@ task "install_sonobuoy" do |_, args| FileUtils.mkdir_p("#{current_dir}/#{TOOLS_DIR}/sonobuoy") curl = `VERSION="#{k8s_version}" OS=linux ; curl -L "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" --output #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` puts curl if check_verbose(args) - `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ + `tar -xzf #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz -C #{current_dir}/#{TOOLS_DIR}/sonobuoy/ && \ chmod +x #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy && \ rm #{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy.tar.gz` sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" @@ -31,3 +31,13 @@ task "install_sonobuoy" do |_, args| end end +desc "Cleans up Sonobuoy" +task "sonobuoy_cleanup"do |_, args| + current_dir = FileUtils.pwd + sonobuoy = "#{current_dir}/#{TOOLS_DIR}/sonobuoy/sonobuoy" + delete = `#{sonobuoy} delete --wait` + puts delete if check_verbose(args) + rm = `rm -rf #{current_dir}/#{TOOLS_DIR}/sonobuoy` + puts rm if check_verbose(args) +end + From 7b86489a6a3660a8403b68f14b68c4aeb41ae997 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 02:44:21 -0400 Subject: [PATCH 38/49] rolling update conformance test usage: crystal src/cnf-conformance.cr rolling_update -- version_tag=latest where version_tags value is a valid release name for the image set in the helm chart's configuration i.e. for the coredns stable chart https://github.com/helm/charts/tree/master/stable/coredns#configuration the image is `coredns/coredns` and the version is `v1.6.7` but any valid version tag from docker hub https://hub.docker.com/r/coredns/coredns/tags can be used refs #7 --- src/tasks/configuration_lifecycle.cr | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 23f5b3b60..b350058c5 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -2,6 +2,7 @@ require "sam" require "file_utils" require "colorize" require "totem" +require "json" require "./utils.cr" desc "Configuration and lifecycle should be managed in a declarative manner, using ConfigMaps, Operators, or other declarative interfaces." @@ -143,3 +144,45 @@ task "retrieve_manifest" do |_, args| end end end + +desc "Test if the CNF can perform a rolling update" +task "rolling_update" do |_, args| + begin + puts "rolling_update" if check_verbose(args) + config = cnf_conformance_yml + + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" unless args.raw.includes? "version_tag" + + version_tag = args["version_tag"] + release_name = config.get("release_name").as_s + deployment_name = config.get("deployment_name").as_s + + helm_chart_values = JSON.parse(`helm get values #{release_name} -a -o json`) + puts "helm_chart_values" if check_verbose(args) + puts helm_chart_values if check_verbose(args) + image_name = helm_chart_values["image"]["repository"] + + puts "image_name: #{image_name}" if check_verbose(args) + + puts "rolling_update: setting new version" if check_verbose(args) + #do_update = `kubectl set image deployment/coredns-coredns coredns=coredns/coredns:latest --record` + puts "kubectl set image deployment/#{deployment_name} #{release_name}=#{image_name}:#{version_tag} --record" if check_verbose(args) + do_update = `kubectl set image deployment/#{deployment_name} #{release_name}=#{image_name}:#{version_tag} --record` + + # https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#rolling-update + puts "rolling_update: checking status new version" if check_verbose(args) + puts `kubectl rollout status deployment/#{deployment_name} --timeout=30s` + + if $?.success? + puts "PASSED: CNF #{deployment_name} Rolling Update Passed".colorize(:green) + else + puts "FAILURE: CNF #{deployment_name} Rolling Update Failed".colorize(:red) + end + + rescue ex + puts ex.message + ex.backtrace.each do |x| + puts x + end + end +end \ No newline at end of file From e322ef7c186a0bac391b823c7c1bdf06e2fee258 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 03:11:24 -0400 Subject: [PATCH 39/49] fix bug in version check logic --- src/tasks/configuration_lifecycle.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index b350058c5..f4e728776 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -151,9 +151,11 @@ task "rolling_update" do |_, args| puts "rolling_update" if check_verbose(args) config = cnf_conformance_yml - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" unless args.raw.includes? "version_tag" + unless args.named.has_key? "version_tag" + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" + end - version_tag = args["version_tag"] + version_tag = args.named["version_tag"] release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s From ee3fade3cbca7d939e72121e45df5ec7d5f13d5e Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 17:52:48 -0400 Subject: [PATCH 40/49] yml option --- .../sample-coredns-cnf/cnf-conformance.yml | 1 + src/tasks/configuration_lifecycle.cr | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index 938751cb3..b488ace90 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -8,4 +8,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns +helm_chart_image_repository_version: latest white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index f4e728776..11cac1bc9 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -151,11 +151,20 @@ task "rolling_update" do |_, args| puts "rolling_update" if check_verbose(args) config = cnf_conformance_yml - unless args.named.has_key? "version_tag" - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag" + version_tag = nil + + if args.named.has_key? "version_tag" + version_tag = args.named["version_tag"] + end + + if config.has_key? "helm_chart_image_repository_version" + version_tag = config.get("helm_chart_image_repository_version").as_s + end + + unless version_tag + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" end - version_tag = args.named["version_tag"] release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s From 907cc274a503cded9276405718b3a7bc1c825c00 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:00:29 -0400 Subject: [PATCH 41/49] overide version from command line --- src/tasks/configuration_lifecycle.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 11cac1bc9..748a12fbd 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -153,14 +153,14 @@ task "rolling_update" do |_, args| version_tag = nil - if args.named.has_key? "version_tag" - version_tag = args.named["version_tag"] - end - if config.has_key? "helm_chart_image_repository_version" version_tag = config.get("helm_chart_image_repository_version").as_s end + if args.named.has_key? "version_tag" + version_tag = args.named["version_tag"] + end + unless version_tag raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" end From 7f0371338703e923379af7668799df8d40562da9 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 25 Mar 2020 18:02:58 -0400 Subject: [PATCH 42/49] #cncf/cnf-conformance/issues/104 sonobuoy now not part of 'all' tests --- .travis.yml | 2 +- src/cnf-conformance.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6dd975b83..3c864b2a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,4 @@ jobs: # Set KUBECONFIG environment variable - export KUBECONFIG="$(kind get kubeconfig-path)" - script: crystal spec + script: crystal spec -v diff --git a/src/cnf-conformance.cr b/src/cnf-conformance.cr index 340e81906..abbcb261a 100644 --- a/src/cnf-conformance.cr +++ b/src/cnf-conformance.cr @@ -2,7 +2,7 @@ require "sam" require "./tasks/**" desc "The CNF Conformance program enables interoperability of CNFs from multiple vendors running on top of Kubernetes supplied by different vendors. The goal is to provide an open source test suite to enable both open and closed source CNFs to demonstrate conformance and implementation of best practices." -task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity", "platform"] do |_, args| +task "all", ["compatibility","stateless", "security", "scalability", "configuration_lifecycle", "observability", "installability", "hardware_affinity"] do |_, args| end Sam.help From 4039f04e6775428cab26704fbbe95deac62c59f4 Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:36:49 -0400 Subject: [PATCH 43/49] tests --- .../sample_coredns/cnf-conformance.yml | 1 + spec/configuration_lifecycle_spec.cr | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index 8abdde7b4..b84d6da1b 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -7,4 +7,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns-coredns +helm_chart_image_repository_version: latest white_list_helm_chart_container_names: [] diff --git a/spec/configuration_lifecycle_spec.cr b/spec/configuration_lifecycle_spec.cr index bcfefab16..719fdb0ae 100644 --- a/spec/configuration_lifecycle_spec.cr +++ b/spec/configuration_lifecycle_spec.cr @@ -72,4 +72,28 @@ describe CnfConformance do `crystal src/cnf-conformance.cr sample_coredns_bad_liveness_cleanup` end end + it "'rolling_update' should pass when valid version is given", tags: "rolling_update" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr rolling_update verbose` + puts response_s + $?.success?.should be_true + (/Rolling Update Passed/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns` + end + end + it "'rolling_update' should fail when invalid version is given", tags: "rolling_update" do + begin + `crystal src/cnf-conformance.cr sample_coredns` + $?.success?.should be_true + response_s = `crystal src/cnf-conformance.cr rolling_update verbose version_tag=this_is_not_real_version` + puts response_s + $?.success?.should be_true + (/Rolling Update Failed/ =~ response_s).should_not be_nil + ensure + `crystal src/cnf-conformance.cr cleanup_sample_coredns` + end + end end From c90e601aca05bf3dcae4bccf5e994d0b50915ce1 Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 25 Mar 2020 18:41:12 -0400 Subject: [PATCH 44/49] #cncf/cnf-conformance/issues/104 sonobuoy now has a before task --- spec/platform/platform_spec.cr | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr index eb634af3d..5fb06cef6 100644 --- a/spec/platform/platform_spec.cr +++ b/spec/platform/platform_spec.cr @@ -2,6 +2,16 @@ require "./../spec_helper" require "colorize" describe "Platform" do + before_all do + # puts `pwd` + # puts `echo $KUBECONFIG` + `crystal src/cnf-conformance.cr samples_cleanup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr setup` + $?.success?.should be_true + `crystal src/cnf-conformance.cr sample_coredns_with_wait_setup` + $?.success?.should be_true + end it "'k8s_conformance' should pass if the sonobuoy tests pass" do response_s = `crystal src/cnf-conformance.cr k8s_conformance` puts response_s From 57e8f8a0c943e68fa5d528617d192e6d835650ff Mon Sep 17 00:00:00 2001 From: William Harris Date: Wed, 25 Mar 2020 18:46:05 -0400 Subject: [PATCH 45/49] change option name --- sample-cnfs/sample-coredns-cnf/cnf-conformance.yml | 2 +- sample-cnfs/sample_coredns/cnf-conformance.yml | 2 +- src/tasks/configuration_lifecycle.cr | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml index b488ace90..5cf6e6ef8 100644 --- a/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml +++ b/sample-cnfs/sample-coredns-cnf/cnf-conformance.yml @@ -8,5 +8,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns -helm_chart_image_repository_version: latest +cnf_image_version: latest white_list_helm_chart_container_names: [falco, nginx, coredns, calico-node, kube-proxy, nginx-proxy] diff --git a/sample-cnfs/sample_coredns/cnf-conformance.yml b/sample-cnfs/sample_coredns/cnf-conformance.yml index b84d6da1b..6ee417b27 100644 --- a/sample-cnfs/sample_coredns/cnf-conformance.yml +++ b/sample-cnfs/sample_coredns/cnf-conformance.yml @@ -7,5 +7,5 @@ deployment_name: coredns-coredns application_deployment_names: [coredns-coredns] helm_chart: stable/coredns helm_chart_container_name: coredns-coredns -helm_chart_image_repository_version: latest +cnf_image_version: latest white_list_helm_chart_container_names: [] diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index 748a12fbd..cdf782fb5 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -153,8 +153,8 @@ task "rolling_update" do |_, args| version_tag = nil - if config.has_key? "helm_chart_image_repository_version" - version_tag = config.get("helm_chart_image_repository_version").as_s + if config.has_key? "cnf_image_version" + version_tag = config.get("cnf_image_version").as_s end if args.named.has_key? "version_tag" @@ -162,7 +162,7 @@ task "rolling_update" do |_, args| end unless version_tag - raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'helm_chart_image_repository_version'" + raise "FAILURE: please specify a version of the CNF's release's image with the option version_tag or with cnf_conformance_yml option 'cnf_image_version'" end release_name = config.get("release_name").as_s From 8dc9d8d2c24993d74fc7360a913332ede657daff Mon Sep 17 00:00:00 2001 From: William Harris Date: Thu, 26 Mar 2020 16:38:14 -0400 Subject: [PATCH 46/49] use helm installed by setup should fix test in travis and pair box issue was using the path helm which was version 2 versus the one the setup installs in tools. also made the output option backwards compatible w helm 2 anyway refs #71 --- src/tasks/configuration_lifecycle.cr | 2 +- src/tasks/utils.cr | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tasks/configuration_lifecycle.cr b/src/tasks/configuration_lifecycle.cr index cdf782fb5..43bda4b9a 100644 --- a/src/tasks/configuration_lifecycle.cr +++ b/src/tasks/configuration_lifecycle.cr @@ -168,7 +168,7 @@ task "rolling_update" do |_, args| release_name = config.get("release_name").as_s deployment_name = config.get("deployment_name").as_s - helm_chart_values = JSON.parse(`helm get values #{release_name} -a -o json`) + helm_chart_values = JSON.parse(`#{tools_helm} get values #{release_name} -a --output json`) puts "helm_chart_values" if check_verbose(args) puts helm_chart_values if check_verbose(args) image_name = helm_chart_values["image"]["repository"] diff --git a/src/tasks/utils.cr b/src/tasks/utils.cr index c6c7c4ad8..26e388675 100644 --- a/src/tasks/utils.cr +++ b/src/tasks/utils.cr @@ -170,6 +170,11 @@ def sample_setup(sample_dir, release_name, deployment_name, helm_chart, helm_dir end end +def tools_helm + current_dir = FileUtils.pwd + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" +end + def sample_cleanup(sample_dir, verbose=true) config = sample_conformance_yml(sample_dir) release_name = config.get("release_name").as_s From 9f704f3785824e3014d366505c4bfea9fa24775e Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 1 Apr 2020 17:56:19 -0400 Subject: [PATCH 47/49] Don't run full sonobuoy test for spec cncf/cnf-conformance#104 --- src/tasks/platform/platform.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/tasks/platform/platform.cr b/src/tasks/platform/platform.cr index b39a38823..3bb3516b9 100644 --- a/src/tasks/platform/platform.cr +++ b/src/tasks/platform/platform.cr @@ -16,7 +16,14 @@ task "k8s_conformance" do |_, args| puts delete if check_verbose(args) # Run the tests - testrun = `#{sonobuoy} run --wait --mode quick` + #TODO when in test mode --mode quick, prod mode no quick + testrun = "" + puts ENV["CRYSTAL_ENV"]? if check_verbose(args) + if ENV["CRYSTAL_ENV"]? == "TEST" + testrun = `#{sonobuoy} run --wait --mode quick` + else + testrun = `#{sonobuoy} run --wait` + end puts testrun if check_verbose(args) results = `results=$(#{sonobuoy} retrieve); #{sonobuoy} results $results` From b0e5e294c21b9f2616e045a668db2e10bcb175cf Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Wed, 1 Apr 2020 17:57:27 -0400 Subject: [PATCH 48/49] Set test mode when running spec cncf/cnf-conformance#104 --- spec/spec_helper.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 548dd0a20..35e7fa4ae 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,2 +1,4 @@ require "spec" require "../src/cnf_conformance" + +ENV["CRYSTAL_ENV"] = "TEST" From 5de8bc357767c668da7a289984240f01e18133c0 Mon Sep 17 00:00:00 2001 From: wwatson Date: Thu, 2 Apr 2020 12:42:00 -0500 Subject: [PATCH 49/49] cncf/cnf-conformance#10 Platform test documentation cncf/cnf-conformance#10 Platform test documentation --- USAGE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/USAGE.md b/USAGE.md index 93e754463..17a08e4ab 100644 --- a/USAGE.md +++ b/USAGE.md @@ -218,5 +218,15 @@ crystal src/cnf-conformance.cr direct_hugepages ``` crystal src/cnf-conformance.cr performance ``` + +## Platform Tests +#### (PoC) Run all platform tests +``` +crystal src/cnf-conformance.cr platform +``` +#### (PoC) Run the K8s conformance tests +``` +crystal src/cnf-conformance.cr k8s_conformance +```