conformance.yml

# ------------------------------------------------------------
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------

name: Components Conformance Tests

on:
  repository_dispatch:
    types: [conformance-test]
  workflow_dispatch:
  schedule:
    - cron: '0 */8 * * *'
  push:
    branches:
      - 'release-*'
      - 'gh-readonly-queue/master/*'
  pull_request:
    branches:
      - 'master'
      - 'release-*'
      - 'gh-readonly-queue/master/*'
  merge_group:

jobs:
  # Based on whether this is a PR or a scheduled run, we will run a different
  # subset of the conformance tests. This allows all the tests not requiring
  # secrets to be executed on pull requests.
  generate-matrix:
    runs-on: ubuntu-22.04
    steps:
    - name: Parse repository_dispatch payload
      if: github.event_name == 'repository_dispatch'
      working-directory: ${{ github.workspace }}
      run: |
        if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
          echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
          echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
        fi

    - name: Check out code
      uses: actions/checkout@v3
      with:
        repository: ${{ env.CHECKOUT_REPO }}
        ref: ${{ env.CHECKOUT_REF }}

    - name: Generate test matrix
      id: generate-matrix
      env:
        VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
      run: |
        if [ -z "$VAULT_NAME" ]; then
          # Do not include cloud tests when credentials are not available
          node .github/scripts/test-info.mjs conformance false
        else
          # Include cloud tests
          node .github/scripts/test-info.mjs conformance true
        fi

    - name: Create PR comment
      if: env.PR_NUMBER != ''
      uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
      with:
        header: ${{ github.run_id }}
        number: ${{ env.PR_NUMBER }}
        GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
        message: |
          # Components conformance test

          🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**

          Commit ref: ${{ env.CHECKOUT_REF }}

    outputs:
      test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }}

  conformance:
    name: ${{ matrix.component }} conformance

    # Add "id-token" with the intended permissions.
    # Needed by the 'Authenticate to Google Cloud' step.
    permissions:
      contents: 'read'
      id-token: 'write'

    runs-on: ubuntu-22.04
    env:
      UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}}
      GOCOV_VER: "v1.1.0"
      GOTESTSUM_VER: "v1.9.0"
    defaults:
      run:
        shell: bash

    needs: 
      - generate-matrix

    strategy:
      fail-fast: false # Keep running even if one component fails
      matrix: 
        include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }}

    steps:
    - name: Set default payload repo and ref
      working-directory: ${{ github.workspace }}
      run: |
        echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
        echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV

    - name: Parse repository_dispatch payload
      if: github.event_name == 'repository_dispatch'
      working-directory: ${{ github.workspace }}
      run: |
        if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
          echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
          echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
        fi

    - name: Check out code
      uses: actions/checkout@v3
      with:
        repository: ${{ env.CHECKOUT_REPO }}
        ref: ${{ env.CHECKOUT_REF }}

    - name: Setup test environment
      run: |
        # Output file
        echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV

        # Current time (used by Terraform)
        echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV}

    - name: Configure conformance test and source path
      run: |
        TEST_COMPONENT=$(echo ${{ matrix.component }} | sed -E 's/\./\//g')
        export TEST_PATH="tests/certification/${TEST_COMPONENT}"
        echo "TEST_PATH=$TEST_PATH" >> $GITHUB_ENV
        export SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
        echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
        # converts slashes to dots in this string, so that it doesn't consider them sub-folders
        export SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g')
        echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV

    - uses: Azure/login@v1
      if: matrix.required-secrets != ''
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}

    # Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
    #    az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
    # Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated
    - name: Setup secrets
      if: matrix.required-secrets != ''
      env:
        VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
      run: |
        secrets="${{ matrix.required-secrets }}"
        for secretName in $(echo -n $secrets | tr ',' ' '); do
          value=$(az keyvault secret show \
            --name $secretName \
            --vault-name $VAULT_NAME \
            --query value \
            --output tsv)
          echo "::add-mask::$value"
          echo "$secretName=$value" >> $GITHUB_OUTPUT
          echo "$secretName=$value" >> $GITHUB_ENV
        done

    # Authenticate with GCP Workload Identity Pool
    # Exports GCP ENV Vars:
    #    - GCP_PROJECT
    #    - GOOGLE_APPLICATION_CREDENTIALS
    - id: 'auth'
      if: matrix.require-gcp-credentials == 'true'
      name: 'Authenticate to Google Cloud'
      uses: 'google-github-actions/auth@v1'
      with:
        token_format: 'access_token'
        workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }}
        service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
        create_credentials_file: true
        export_environment_variables: true
        cleanup_credentials: true

    # Download the required certificates into files, and set env var pointing to their names
    - name: Setup certs
      if: matrix.required-certs != ''
      run: |
        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(mktemp --suffix .pfx)
          echo "Downloading cert $CERT_NAME into file $CERT_FILE"
          rm $CERT_FILE && \
            az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
          echo 'Setting $CERT_NAME to' "$CERT_FILE"
          echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
        done

    - name: Setup Terraform
      if: matrix.require-terraform == 'true'
      uses: hashicorp/setup-terraform@v2.0.3

    - name: Set Cloudflare env vars
      if: matrix.require-cloudflare-credentials == 'true'
      run: |
        echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV
        echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV

    - name: Set AWS env vars
      if: matrix.require-aws-credentials == 'true'
      run: |
        echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV
        echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV

    - name: Configure AWS Credentials
      if: matrix.require-aws-credentials == 'true'
      uses: aws-actions/configure-aws-credentials@v1
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
        aws-region: us-west-1

    - name: Start MongoDB
      if: matrix.mongodb-version != ''
      uses: supercharge/mongodb-github-action@1.8.0
      with:
        mongodb-version: ${{ matrix.mongodb-version }}
        mongodb-replica-set: test-rs

    - name: Set up Go
      uses: actions/setup-go@v3
      with:
        go-version-file: 'go.mod'

    - name: Install Node.js ${{ matrix.nodejs-version }}
      if: matrix.nodejs-version != ''
      uses: actions/setup-node@v3
      with:
        node-version: ${{ matrix.nodejs-version }}

    - name: Start KinD
      uses: helm/kind-action@v1.5.0
      if: matrix.require-kind == 'true'

    - name: Download Go dependencies
      run: |
        go mod download
        go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }}
        go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }}

    - name: Run setup script
      if: matrix.setup-script != ''
      run: .github/scripts/components-scripts/${{ matrix.setup-script }}

    - name: Catch setup failures
      if: failure()
      run: |
        echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV

    - name: Run tests
      continue-on-error: true
      run: |
        set -e 
        KIND=$(echo ${{ matrix.component }} | cut -d. -f1)
        NAME=$(echo ${{ matrix.component }} | cut -d. -f2-)
        KIND_UPPER="$(tr '[:lower:]' '[:upper:]' <<< ${KIND:0:1})${KIND:1}"

        if [ "${KIND}" = "secretstores" ]; then
          KIND_UPPER=SecretStore
        fi

        echo "Running tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "
        echo "Source Pacakge: " ${{ matrix.source-pkg }}

        set +e
        gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json \
          --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.xml --format standard-verbose -- \
          -p 2 -count=1 -timeout=15m -tags=conftests ./tests/conformance --run="Test${KIND_UPPER}Conformance/${NAME}" -coverprofile=cover.out \
          -covermode=set -coverpkg=${{ matrix.source-pkg }}

        status=$?
        echo "Completed tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "
        if test $status -ne 0; then
          echo "Setting CONFORMANCE_FAILURE"
          echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV
        fi
        set -e

        # Fail the step if we found no test to run
        if grep -q "warning: no tests to run" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json ; then
          echo "::error:: No test was found for component ${{ matrix.component }}"
          exit -1
        fi

    - name: Delete downloaded up certs
      if: always() && matrix.required-certs != ''
      run: |
        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(printenv $CERT_NAME)

          echo "Cleaning up the certificate file $CERT_FILE..."
          rm $CERT_FILE || true
        done

    - name: Check conformance test passed
      continue-on-error: false
      run: |
        echo "CONFORMANCE_FAILURE=$CONFORMANCE_FAILURE"
        if [[ -v CONFORMANCE_FAILURE ]]; then
          exit 1
        fi

    - name: Prepare test result info
      if: always()
      run: |
        mkdir -p tmp/result_files
        echo "Writing to tmp/result_files/${{ matrix.component }}.txt"
        if [[ "${{ env.CONFORMANCE_FAILURE }}" == "true" ]]; then
          echo "0" >> "tmp/result_files/${{ matrix.component }}.txt"
        else
          echo "1" >> "tmp/result_files/${{ matrix.component }}.txt"
        fi

    - name: Upload result files
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: result_files
        path: tmp/result_files
        retention-days: 1
    
    - name: Prepare coverage report file to upload
      if: github.event_name == 'schedule'
      run: |
        mkdir -p tmp/conf_code_cov
        cp cover.out tmp/conf_code_cov/${{ env.SOURCE_PATH_LINEAR }}.out
    
    - name: Upload coverage report file
      uses: actions/upload-artifact@v3
      if: github.event_name == 'schedule'
      with:
        name: conf_code_cov
        path: tmp/conf_code_cov
        retention-days: 7

    # Upload logs for test analytics to consume
    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@master
      with:
        name: ${{ matrix.component }}_conformance_test
        path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.*

    - name: Run destroy script
      if: always() && matrix.destroy-script != ''
      run: .github/scripts/components-scripts/${{ matrix.destroy-script }}

  post_job:
    name: Post-completion
    runs-on: ubuntu-22.04
    if: always()
    needs:
      - conformance
      - generate-matrix
    steps:
      - name: Parse repository_dispatch payload
        if: github.event_name == 'repository_dispatch'
        working-directory: ${{ github.workspace }}
        run: |
          if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
            echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
            echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
          fi

      - name: Download test result artifact
        if: always() && env.PR_NUMBER != ''
        uses: actions/download-artifact@v3
        continue-on-error: true
        id: testresults
        with:
          name: result_files
          path: tmp/result_files

      - name: Build message
        if: always() && env.PR_NUMBER != ''
        # Abusing of the github-script action to be able to write this in JS
        uses: actions/github-script@v6
        with:
          script: |
            const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}')
            const basePath = '${{ steps.testresults.outputs.download-path }}'
            const testType = 'conformance'

            const fs = require('fs')
            const path = require('path')

            let message = `# Components ${testType} test

            🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**

            Commit ref: ${{ env.CHECKOUT_REF }}`

            let allSuccess = true
            let allFound = true
            let notSuccess = []
            let notFound = []
            for (let i = 0; i < allComponents.length; i++) {
                let component = allComponents[i]
                if (!component) {
                    continue
                }
                if (typeof component == 'object') {
                    component = component.component
                }
                let found = false
                let success = false
                try {
                    let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8')
                    read = read.split('\n')[0]
                    switch (read) {
                        case '1':
                            found = true
                            success = true
                            break
                        case '0':
                            found = true
                            success = false
                    }
                } catch (e) {
                    // ignore errors, leave found = false
                }

                if (!found) {
                    allFound = false
                    notFound.push(component)
                }
                if (!success) {
                    allSuccess = false
                    notSuccess.push(component)
                }
            }

            if (allSuccess) {
                if (allFound) {
                    message += '\n\n' + `# ✅ All ${testType} tests passed

            All tests have reported a successful status` + '\n\n'
                } else {
                    message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status

            Although there were no failures reported, some tests did not report a status:` + '\n\n'
                    for (let i = 0; i < notFound.length; i++) {
                        message += '- ' + notFound[i] + '\n'
                    }
                    message += '\n'
                }
            } else {
                message += '\n\n' + `# ❌ Some ${testType} tests failed

            These tests failed:` + '\n\n'
                for (let i = 0; i < notSuccess.length; i++) {
                    message += '- ' + notSuccess[i] + '\n'
                }
                message += '\n'

                if (!allFound) {
                    message += 'Additionally, some tests did not report a status:\n\n'
                    for (let i = 0; i < notFound.length; i++) {
                        message += '- ' + notFound[i] + '\n'
                    }
                    message += '\n'
                }
            }

            fs.writeFileSync('message.txt', message)

      - name: Replace PR comment
        if: env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          path: message.txt