feat(test): better bats tests, install from latest tag and then update from master

.devcontainer/devcontainer.json

@@ -24,6 +24,13 @@
             "version": "latest"
         },
         "ghcr.io/brokenpip3/devcontainers-bats/bats-libs:0": {
+        },
+        "ghcr.io/devcontainers/features/nix:1": {
+            "multiUser": "false",
+            "extraNixConfig": "experimental-features = nix-command flakes"
+        },
+        "ghcr.io/devcontainers/features/hugo:1": {
+            "version": "v0.99.1"
         }
     },
     // "forwardPorts": [],

.github/workflows/auto-tests-bats.yaml

@@ -19,6 +19,7 @@ on:
       - 'backup/**'
       - '*.md'
 
+#TODO: create a matrix per earch bats file
 jobs:
   run-tests:
     if: github.event.pull_request.draft == false

.gitignore

@@ -101,3 +101,4 @@ result
 ### website
 website/node_modules
 website/public
+website/.hugo_build.lock

Makefile

@@ -181,6 +181,10 @@ update-lts-version: ## Update the latest lts version
 	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' test/e2e/test_utility.go
 	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' test/helm/helm_test.go
 	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' pkg/constants/constants.go
+	#TODO: source the version from config.base.env for bats test, no need of hardcoded version
+	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' test/bats/1-deploy.bats
+	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' test/bats/2-deploy-with-more-options.bats
+	sed -i 's|jenkins/jenkins:[0-9]\+.[0-9]\+.[0-9]\+|jenkins/jenkins:$(LATEST_LTS_VERSION)|g' test/bats/3-deploy-with-webhook.bats
 
 .PHONY: run
 run: export WATCH_NAMESPACE = $(NAMESPACE)

flake.nix

@@ -42,6 +42,14 @@
             })
             go_15_pkgs.go
             golangci_pkgs.golangci-lint
+
+            pkgs.kind
+            (pkgs.bats.withLibraries (p: [
+            p.bats-support
+            p.bats-assert
+            p.bats-file
+            p.bats-detik
+            ]))
           ];
           shellHook = ''
             echo Operator Version ${operatorVersion}

test/bats/1-deploy.bats

@@ -7,15 +7,24 @@ diag() {
   echo "# DEBUG $@" >&3
 }
 
-#bats test_tags=phase:setup
-@test "1.0 Create namespace" {
+#bats test_tags=phase:setup,scenario:vanilla
+@test "1.0  Init: create namespace" {
   ${KUBECTL} get ns ${DETIK_CLIENT_NAMESPACE} && skip "Namespace ${DETIK_CLIENT_NAMESPACE} already exists"
   run ${KUBECTL} create ns ${DETIK_CLIENT_NAMESPACE}
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.1 Vanilla install helm chart" {
+#bats test_tags=phase:setup,scenario:vanilla
+@test "1.1  Init: add helm chart repo" {
+    ${HELM} repo list|grep -qc jenkins-operator && skip "Jenkins repo already exists"
+    upstream_url="https://raw.githubusercontent.com/jenkinsci/kubernetes-operator/master/chart"
+    run ${HELM} repo add jenkins-operator $upstream_url
+    assert_success
+    assert_output '"jenkins-operator" has been added to your repositories'
+}
+
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.2  Helm: vanilla install helm chart latest tagged version" {
   run echo ${DETIK_CLIENT_NAMESPACE}
   run echo ${OPERATOR_IMAGE}
   ${HELM} status default && skip "Helm release 'default' already exists"
@@ -24,15 +33,16 @@ diag() {
     --set namespace=${DETIK_CLIENT_NAMESPACE} \
     --set operator.image=${OPERATOR_IMAGE} \
     --set jenkins.latestPlugins=true \
+    --set jenkins.image="jenkins/jenkins:2.440.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=false \
-    chart/jenkins-operator
+    jenkins-operator/jenkins-operator --version=$(cat ../../VERSION.txt | sed 's/v//')
   assert_success
   assert ${HELM} status default
   touch "chart/jenkins-operator/deploy.tmp"
 }
 
-#bats test_tags=phase:helm
-@test "1.2 Helm: check Jenkins operator pods status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.2  Helm: check Jenkins operator pods status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there is 1 deployment named 'default-jenkins-operator'"
   assert_success
@@ -44,8 +54,8 @@ diag() {
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.3 Helm: check Jenkins Pod status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.3  Helm: check Jenkins Pod status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run try "at most 20 times every 10s to get pods named 'jenkins-jenkins' and verify that '.status.containerStatuses[?(@.name==\"jenkins-master\")].ready' is 'true'"
   assert_success
@@ -54,8 +64,8 @@ diag() {
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.4 Helm: check Jenkins service status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.4  Helm: check Jenkins service status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there is 1 service named 'jenkins-operator-http-jenkins'"
   assert_success
@@ -64,8 +74,8 @@ diag() {
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.5 Helm: check Jenkins configmaps created" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.5  Helm: check Jenkins configmaps created" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there is 1 configmap named 'jenkins-operator-base-configuration-jenkins'"
   assert_success
@@ -75,44 +85,99 @@ diag() {
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.6 Helm: check Jenkins operator role status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.6  Helm: check Jenkins operator role status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there are 2 role named 'jenkins-operator*'"
   assert_success
   run verify "there is 1 role named 'leader-election-role'"
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.7 Helm: check Jenkins operator role binding status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.7  Helm: check Jenkins operator role binding status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there is 1 rolebinding named 'jenkins-operator-jenkins'"
   assert_success
   run verify "there is 1 rolebinding named 'leader-election-rolebinding'"
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.8 Helm: check Jenkins operator service account status" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.8  Helm: check Jenkins operator service account status" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there are 2 serviceaccount named 'jenkins-operator*'"
   assert_success
 }
 
-#bats test_tags=phase:helm
-@test "1.9 Helm: check Jenkins crd" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.9  Helm: check Jenkins crd" {
   [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
   run verify "there is 1 crd named 'jenkins.jenkins.io'"
   assert_success
 }
 
-@test "1.9 Helm: Clean" {
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.10 Helm: upgrade from main branch same values" {
+  run echo ${DETIK_CLIENT_NAMESPACE}
+  run echo ${OPERATOR_IMAGE}
+  run ${HELM} upgrade default \
+    --set jenkins.namespace=${DETIK_CLIENT_NAMESPACE} \
+    --set namespace=${DETIK_CLIENT_NAMESPACE} \
+    --set operator.image=${OPERATOR_IMAGE} \
+    --set jenkins.latestPlugins=true \
+    --set jenkins.image="jenkins/jenkins:2.440.1-lts" \
+    --set jenkins.backup.makeBackupBeforePodDeletion=false \
+    chart/jenkins-operator
+  assert_success
+  assert ${HELM} status default
+}
+
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.11 Helm: check Jenkins operator pods status again" {
+  [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
+  run verify "there is 1 deployment named 'default-jenkins-operator'"
+  assert_success
+
+  run verify "there is 1 pod named 'default-jenkins-operator-'"
+  assert_success
+
+  run try "at most 20 times every 10s to get pods named 'default-jenkins-operator-' and verify that '.status.containerStatuses[?(@.name==\"jenkins-operator\")].ready' is 'true'"
+  assert_success
+}
+
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.12 Helm: check Jenkins operator pods status" {
+  [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
+  run verify "there is 1 deployment named 'default-jenkins-operator'"
+  assert_success
+
+  run verify "there is 1 pod named 'default-jenkins-operator-'"
+  assert_success
+
+  run try "at most 20 times every 10s to get pods named 'default-jenkins-operator-' and verify that '.status.containerStatuses[?(@.name==\"jenkins-operator\")].ready' is 'true'"
+  assert_success
+}
+
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.13 Helm: check Jenkins Pod status" {
+  [[ ! -f "chart/jenkins-operator/deploy.tmp" ]] && skip "Jenkins helm chart have not been deployed correctly"
+  run try "at most 20 times every 10s to get pods named 'jenkins-jenkins' and verify that '.status.containerStatuses[?(@.name==\"jenkins-master\")].ready' is 'true'"
+  assert_success
+
+  run try "at most 20 times every 5s to get pods named 'jenkins-jenkins' and verify that '.status.containerStatuses[?(@.name==\"jenkins-master\")].ready' is 'true'"
+  assert_success
+}
+
+#bats test_tags=phase:helm,scenario:vanilla
+@test "1.14 Helm: clean" {
   run ${HELM} uninstall default
   assert_success
   # Wait for the complete removal
   sleep 30
+
   run verify "there is 0 pvc named 'jenkins backup'"
   assert_success
+
   rm "chart/jenkins-operator/deploy.tmp"
 }