diff --git a/.github/workflows/auto-tests-bats.yaml b/.github/workflows/auto-tests-bats.yaml index a2293bde0..cb8a894fc 100644 --- a/.github/workflows/auto-tests-bats.yaml +++ b/.github/workflows/auto-tests-bats.yaml @@ -1,4 +1,4 @@ -name: Run bats tests +name: Tests BATS on: push: branches: @@ -22,7 +22,7 @@ on: jobs: run-tests: if: github.event.pull_request.draft == false - name: Run automated bats tests + name: BATS Run tests runs-on: ubuntu-latest steps: - name: Check out code diff --git a/.github/workflows/auto-tests-e2e.yaml b/.github/workflows/auto-tests-e2e.yaml index ad987843c..63671cabd 100644 --- a/.github/workflows/auto-tests-e2e.yaml +++ b/.github/workflows/auto-tests-e2e.yaml @@ -1,4 +1,4 @@ -name: Run e2e tests +name: Tests E2E on: push: branches: @@ -20,20 +20,29 @@ on: - '*.md' jobs: - run-tests: - if: github.event.pull_request.draft == false - name: Run automated tests + create-e2e-list: + name: E2E Create tests list runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} steps: - - name: Check out code - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - id: matrix + run: | + script=$(./test/make_matrix_ginkgo.sh e2e) + echo "matrix=${script}" >> $GITHUB_OUTPUT + + verify-code: + name: E2E Verify code before tests + runs-on: ubuntu-latest + needs: [create-e2e-list] + steps: + - uses: actions/checkout@v4 - name: Set up env vars run: | echo "GO111MODULE=on" >> $GITHUB_ENV echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV - echo "HELM_VERSION=v$(sed -n 's/HELM_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV - echo "KIND_CLUSTER_NAME=$(sed -n 's/KIND_CLUSTER_NAME=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV echo "GOPATH=/home/runner/go" >> $GITHUB_ENV - name: Prepare go environment @@ -47,6 +56,33 @@ jobs: - name: Verify code formatting run: make verify + run-e2e-tests: + runs-on: ubuntu-latest + needs: [create-e2e-list, verify-code] + if: github.event.pull_request.draft == false + name: E2E ${{ matrix.test }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.create-e2e-list.outputs.matrix) }} + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up env vars + run: | + echo "GO111MODULE=on" >> $GITHUB_ENV + echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV + echo "KIND_CLUSTER_NAME=$(sed -n 's/KIND_CLUSTER_NAME=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV + echo "GOPATH=/home/runner/go" >> $GITHUB_ENV + + - name: Prepare go environment + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Ensure Golang runtime dependencies + run: make go-dependencies + - name: Kind setup uses: helm/kind-action@v1.9.0 with: @@ -55,16 +91,19 @@ jobs: - name: Prepare environment for e2e run: | - sudo apt-get update - sudo apt-get install socat + sudo apt-get update && sudo apt-get install -y socat sudo mkdir -p $HOME/.kube sudo chown -R $USER $HOME/.kube - - name: Jenkins Operator - e2e - list tests - run: make e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.dryRun' - - - name: Jenkins Operator - e2e - run: make e2e E2E_TEST_ARGS='-ginkgo.v' + - name: Jenkins Operator - e2e Chart tests + env: + TNAME: ${{ matrix.test }} + TFILE: ${{ matrix.file }} + TLINE: ${{ matrix.line }} + run: | + git reset --hard + printf "\n \n > Running test: %s from file: $s line: %s\n" "${TNAME}" "${TFILE}" "${TLINE}" + make e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.focus="${TNAME}"' - name: Debug if: failure() diff --git a/.github/workflows/auto-tests-helm.yaml b/.github/workflows/auto-tests-helm.yaml index 6c75fa195..a79b199c3 100644 --- a/.github/workflows/auto-tests-helm.yaml +++ b/.github/workflows/auto-tests-helm.yaml @@ -1,4 +1,4 @@ -name: Run Helm e2e tests +name: Tests HELM on: push: branches: @@ -20,10 +20,50 @@ on: - '*.md' jobs: - run-tests: - if: github.event.pull_request.draft == false - name: Run automated tests + create-helm-list: + name: HELM Create tests list + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - id: matrix + run: | + script=$(./test/make_matrix_ginkgo.sh helm) + echo "matrix=${script}" >> $GITHUB_OUTPUT + + verify-code: + name: HELM Verify code before tests runs-on: ubuntu-latest + needs: [create-helm-list] + steps: + - uses: actions/checkout@v4 + + - name: Set up env vars + run: | + echo "GO111MODULE=on" >> $GITHUB_ENV + echo "GO_VERSION=v$(sed -n 's/GO_VERSION=//p' config.base.env | tr -d '\n' | tr -d '"')" >> $GITHUB_ENV + echo "GOPATH=/home/runner/go" >> $GITHUB_ENV + + - name: Prepare go environment + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Ensure Golang runtime dependencies + run: make go-dependencies + + - name: Verify code formatting + run: make verify + + run-helm-tests: + runs-on: ubuntu-latest + needs: [create-helm-list, verify-code] + if: github.event.pull_request.draft == false + name: HELM ${{ matrix.test }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.create-helm-list.outputs.matrix) }} steps: - name: Check out code uses: actions/checkout@v4 @@ -44,9 +84,6 @@ jobs: - name: Ensure Golang runtime dependencies run: make go-dependencies - - name: Verify code formatting - run: make verify - - name: Kind setup uses: helm/kind-action@v1.9.0 with: @@ -55,13 +92,26 @@ jobs: - name: Prepare environment for e2e run: | - sudo apt-get update - sudo apt-get install socat + sudo apt-get update && sudo apt-get install -y socat sudo mkdir -p $HOME/.kube sudo chown -R $USER $HOME/.kube - name: Jenkins Operator - Helm Chart tests + env: + TNAME: ${{ matrix.test }} + TFILE: ${{ matrix.file }} + TLINE: ${{ matrix.line }} run: | git reset --hard make helm-lint - make helm-e2e E2E_TEST_ARGS='-ginkgo.v' + printf "\n \n > Running test: %s from file: $s line: %s\n" "${TNAME}" "${TFILE}" "${TLINE}" + make helm-e2e E2E_TEST_ARGS='-ginkgo.v -ginkgo.focus="${TNAME}"' + + - name: Debug + if: failure() + shell: bash + continue-on-error: true + run: | + randomns=$(kubectl get ns| grep -i 'ns[0-9]\+' |cut -d ' ' -f 1) + kubectl get pods -n ${randomns} + kubectl get events -n ${randomns} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 333b17e16..374688161 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/sirosen/check-jsonschema - rev: 0.22.0 + rev: 0.28.0 hooks: - id: check-github-workflows - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/flake.nix b/flake.nix index 8d1648cf9..655d01edc 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,11 @@ pkgs.gnumake pkgs.wget pkgs.helm-docs + (pkgs.writeShellApplication { + name = "make_matrix"; + runtimeInputs = with pkgs; [ bash gnugrep gawk ]; + text = builtins.readFile ./test/make_matrix_ginkgo.sh; + }) go_15_pkgs.go golangci_pkgs.golangci-lint ]; diff --git a/test/e2e/jenkins_configuration_test.go b/test/e2e/jenkins_configuration_test.go index 8ce3443f7..4d01594ed 100644 --- a/test/e2e/jenkins_configuration_test.go +++ b/test/e2e/jenkins_configuration_test.go @@ -90,7 +90,7 @@ var _ = Describe("Jenkins controller configuration", func() { }) Context("when deploying CR to cluster", func() { - It("creates Jenkins instance and configures it", func() { + It("creates vanilla Jenkins instance and configures it", func() { WaitForJenkinsBaseConfigurationToComplete(jenkins) verifyJenkinsMasterPodAttributes(jenkins) verifyServices(jenkins) @@ -138,7 +138,7 @@ var _ = Describe("Jenkins controller priority class", func() { }) Context("when deploying CR with priority class to cluster", func() { - It("creates Jenkins instance and configures it", func() { + It("creates Jenkins instance with priority class and configures it", func() { WaitForJenkinsBaseConfigurationToComplete(jenkins) verifyJenkinsMasterPodAttributes(jenkins) }) diff --git a/test/e2e/jenkins_pod_restart_test.go b/test/e2e/jenkins_pod_restart_test.go index 70846c949..4d4743297 100644 --- a/test/e2e/jenkins_pod_restart_test.go +++ b/test/e2e/jenkins_pod_restart_test.go @@ -47,7 +47,7 @@ var _ = Describe("Jenkins controller", func() { }) Context("when restarting Jenkins master pod", func() { - It("new Jenkins Master pod should be created", func() { + It("new Jenkins pod should be created after a restart", func() { WaitForJenkinsBaseConfigurationToComplete(jenkins) restartJenkinsMasterPod(jenkins) waitForRecreateJenkinsMasterPod(jenkins) @@ -96,19 +96,16 @@ var _ = Describe("Jenkins controller", func() { }) Context("when running Jenkins safe restart", func() { - It("authorization strategy is not overwritten", func() { + It("authorization strategy is not overwritten after a restart", func() { // TODO: @brokenpip3 temporary disable this flaky test Skip("Temporary skipping this test") - WaitForJenkinsBaseConfigurationToComplete(jenkins) WaitForJenkinsUserConfigurationToComplete(jenkins) jenkinsClient, cleanUpFunc := verifyJenkinsAPIConnection(jenkins, namespace.Name) defer cleanUpFunc() checkIfAuthorizationStrategyUnsecuredIsSet(jenkinsClient) - err := jenkinsClient.SafeRestart() Expect(err).NotTo(HaveOccurred()) waitForJenkinsSafeRestart(jenkinsClient) - checkIfAuthorizationStrategyUnsecuredIsSet(jenkinsClient) }) }) diff --git a/test/e2e/jenkins_test.go b/test/e2e/jenkins_test.go index 1543c465f..96d4431ad 100644 --- a/test/e2e/jenkins_test.go +++ b/test/e2e/jenkins_test.go @@ -143,8 +143,8 @@ func createJenkinsCRSafeRestart(name, namespace string, seedJob *[]v1alpha2.Seed }, SeedJobs: seedJobs, Service: v1alpha2.Service{ - Type: corev1.ServiceTypeNodePort, - Port: constants.DefaultHTTPPortInt32, + Type: corev1.ServiceTypeNodePort, + Port: constants.DefaultHTTPPortInt32, NodePort: 30303, }, }, @@ -240,13 +240,14 @@ func verifyJenkinsAPIConnection(jenkins *v1alpha2.Jenkins, namespace string) (je func restartJenkinsMasterPod(jenkins *v1alpha2.Jenkins) { _, _ = fmt.Fprintf(GinkgoWriter, "Restarting Jenkins master pod\n") jenkinsPod := getJenkinsMasterPod(jenkins) - _, _ = fmt.Fprintf(GinkgoWriter, "Jenkins pod: %+v\n", jenkinsPod) + initialCreationTimestamp := jenkinsPod.CreationTimestamp.DeepCopy() + + _, _ = fmt.Fprintf(GinkgoWriter, "Jenkins pod: %+v\n", jenkinsPod.Status.Phase) Expect(K8sClient.Delete(context.TODO(), jenkinsPod)).Should(Succeed()) Eventually(func() (bool, error) { jenkinsPod = getJenkinsMasterPod(jenkins) - fmt.Printf("Jenkins pod deletion timestamp: %v\n", jenkinsPod.DeletionTimestamp) - return jenkinsPod.DeletionTimestamp != nil, nil + return !jenkinsPod.CreationTimestamp.Equal(initialCreationTimestamp), nil }, 45*retryInterval, retryInterval).Should(BeTrue()) _, _ = fmt.Fprintf(GinkgoWriter, "Jenkins master pod has been restarted\n") diff --git a/test/e2e/restorebackup_test.go b/test/e2e/restorebackup_test.go index cb8958537..aa90f6ae5 100644 --- a/test/e2e/restorebackup_test.go +++ b/test/e2e/restorebackup_test.go @@ -210,8 +210,8 @@ func createJenkinsWithBackupAndRestoreConfigured(name, namespace string) *v1alph }, }, Service: v1alpha2.Service{ - Type: corev1.ServiceTypeNodePort, - Port: constants.DefaultHTTPPortInt32, + Type: corev1.ServiceTypeNodePort, + Port: constants.DefaultHTTPPortInt32, NodePort: 30303, }, }, diff --git a/test/e2e/test_utility.go b/test/e2e/test_utility.go index 7b62003f4..bce1ee0dd 100644 --- a/test/e2e/test_utility.go +++ b/test/e2e/test_utility.go @@ -165,8 +165,8 @@ func RenderJenkinsCR(name, namespace string, seedJob *[]v1alpha2.SeedJob, groovy }, SeedJobs: seedJobs, Service: v1alpha2.Service{ - Type: corev1.ServiceTypeNodePort, - Port: constants.DefaultHTTPPortInt32, + Type: corev1.ServiceTypeNodePort, + Port: constants.DefaultHTTPPortInt32, NodePort: 30303, }, Roles: []rbacv1.RoleRef{ diff --git a/test/helm/helm_test.go b/test/helm/helm_test.go index 5e65ed394..b446a29a8 100644 --- a/test/helm/helm_test.go +++ b/test/helm/helm_test.go @@ -36,7 +36,7 @@ var _ = Describe("Jenkins Controller", func() { }) Context("Deploys jenkins operator with helm charts with default values", func() { - It("Deploys Jenkins operator and configures default Jenkins instance", func() { + It("Deploys Jenkins operator and configures the default Jenkins instance", func() { jenkins := &v1alpha2.Jenkins{ TypeMeta: v1alpha2.JenkinsTypeMeta(), ObjectMeta: metav1.ObjectMeta{ @@ -99,7 +99,7 @@ var _ = Describe("Jenkins Controller with security validator", func() { }) Context("When Jenkins CR contains plugins with security warnings", func() { - It("Denies creating a jenkins CR with a warning", func() { + It("Denies creating a jenkins CR with a plugin contains security warning", func() { By("Deploying the operator along with webhook and cert-manager") cmd := exec.Command("../../bin/helm", "upgrade", "jenkins", "../../chart/jenkins-operator", "--namespace", namespace.Name, "--debug", "--set-string", fmt.Sprintf("jenkins.namespace=%s", namespace.Name), @@ -121,7 +121,7 @@ var _ = Describe("Jenkins Controller with security validator", func() { }) }) Context("When Jenkins CR doesn't contain plugins with security warnings", func() { - It("Jenkins instance is successfully created", func() { + It("Permit creating a jenkins CR without security warning in plugins", func() { By("Deploying the operator along with webhook and cert-manager") cmd := exec.Command("../../bin/helm", "upgrade", "jenkins", "../../chart/jenkins-operator", "--namespace", namespace.Name, "--debug", "--set-string", fmt.Sprintf("jenkins.namespace=%s", namespace.Name), diff --git a/test/make_matrix_ginkgo.sh b/test/make_matrix_ginkgo.sh new file mode 100755 index 000000000..3477a8028 --- /dev/null +++ b/test/make_matrix_ginkgo.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +TESTDIR="${TESTDIR:-test}" + +json_output(){ + # Make shellcheck happy, + # declare local before assign + local lastl + local line + local grep_info + local f + local l + local t + + lastl=$(echo "${1}" | wc -l) + line=0 + printf '{\"include\":[' + while read -r test; do + line=$((line + 1)) + grep_info=$(echo "${test}"|awk -F '"' '{print $1}') + f=$(echo "${grep_info}"|cut -d ':' -f 1) + l=$(echo "${grep_info}"|cut -d ':' -f 2) + t=$(echo "${test}"|awk -F '"' '{print $2}') + printf '{\"file\":\"%s\",\"line\":\"%s\",\"test\":\"%s\"}' "$f" "$l" "$t" + [[ $line -ne $lastl ]] && printf "," + done <<< "${1}" + printf "]}" +} + +parse(){ + grep -nrE 'It\([^)]+\)' "$1" +} + +tests_list=$(parse "${TESTDIR}"/"${1}") +json_output "${tests_list}"