test.yaml

name: Tests
on:
  pull_request:
    branches: [ "master", "release/**", "core-snap-security-release/**", "security-release/**" ]
  push:
    branches: [ "master", "release/**", "core-snap-security-release/**", "security-release/**" ]

concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  snap-builds:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        toolchain:
          - default
          - FIPS
        version:
          # test version is a build of snapd with test keys and should
          # only be installed by test runners. The pristine versions
          # are the build that should be installed by human users.
          - pristine
          - test
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Select Go toolchain
      run: |
        case "${{ matrix.toolchain }}" in
            default)
                rm -f fips-build
                ;;
            FIPS)
                touch fips-build
                ;;
            *)
                echo "unknown toolchain ${{ matrix.toolchain }}"
                exit 1
                ;;
        esac
        case "${{ matrix.version }}" in
            pristine)
                rm -f test-build
                ;;
            test)
                touch test-build
                ;;
        esac

    - name: Build snapd snap
      uses: snapcore/action-build@v1
      with:
        snapcraft-channel: 8.x/stable

    - name: Build test snapd snap
      uses: snapcore/action-build@v1
      with:
        snapcraft-channel: 8.x/stable

    - name: Check built artifact
      run: |
        unsquashfs snapd*.snap meta/snap.yaml usr/lib/snapd/
        if cat squashfs-root/meta/snap.yaml | grep -q "version:.*dirty.*"; then
          echo "PR produces dirty snapd snap version"
          cat squashfs-root/usr/lib/snapd/dirty-git-tree-info.txt
          exit 1
        elif cat squashfs-root/usr/lib/snapd/info | grep -q "VERSION=.*dirty.*"; then
          echo "PR produces dirty internal snapd info version"
          cat squashfs-root/usr/lib/snapd/info
          cat squashfs-root/usr/lib/snapd/dirty-git-tree-info.txt
          exit 1
        fi

    - name: Uploading snapd snap artifact
      uses: actions/upload-artifact@v3
      with:
        name: snap-files-${{ matrix.toolchain }}-${{ matrix.version }}
        path: "*.snap"

  cache-build-deps:
    runs-on: ubuntu-20.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
        # golang latest ensures things work on the edge

    - name: Download Debian dependencies
      run: |
          sudo apt clean
          sudo apt update
          sudo apt build-dep -d -y ${{ github.workspace }}/src/github.com/snapcore/snapd
          # for indent
          sudo apt install texinfo autopoint

    - name: Copy dependencies
      run: |
        sudo tar cvf cached-apt.tar /var/cache/apt

    - name: upload Debian dependencies
      uses: actions/upload-artifact@v3
      with:
        name: debian-dependencies
        path: ./cached-apt.tar

  static-checks:
    runs-on: ubuntu-latest
    needs: [cache-build-deps]
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
      GITHUB_PULL_REQUEST_TITLE: ${{ github.event.pull_request.title }}

    strategy:
      # we cache successful runs so it's fine to keep going
      fail-fast: false      
      matrix:
        gochannel:
          - 1.18
          - latest/stable

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}

    - name: Download Debian dependencies
      uses: actions/download-artifact@v3
      with:
        name: debian-dependencies
        path: ./debian-deps/

    - name: Copy dependencies
      run: |
        test -f ./debian-deps/cached-apt.tar
        sudo tar xvf ./debian-deps/cached-apt.tar -C /

    - name: Install Debian dependencies
      run: |
          sudo apt update
          sudo apt build-dep -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    # golang latest ensures things work on the edge
    - name: Install the go snap
      run: |
          sudo snap install --classic --channel=${{ matrix.gochannel }} go

    - name: Install ShellCheck as a snap
      run: |
          sudo apt-get remove --purge shellcheck
          sudo snap install shellcheck

    - name: Get C vendoring
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd/c-vendor && ./vendor.sh

    - name: golangci-lint
      if: ${{ matrix.gochannel == 'latest/stable' }}
      uses: golangci/golangci-lint-action@v3
      with:
        # version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest`
        # to use the latest version
        version: v1.60.1
        working-directory: ./src/github.com/snapcore/snapd
        # show only new issues
        # use empty path prefix to make annotations work
        args: --new-from-rev=${{ github.base_ref }} --path-prefix=
        # skip all additional steps
        skip-pkg-cache: true
        skip-build-cache: true
        # XXX: does no work with working-directory
        # only-new-issues: true

    - name: Get changed files
      id: changed-files
      uses: tj-actions/changed-files@v41.0.0
      with:
        path: ./src/github.com/snapcore/snapd

    - name: Save changes files
      run: |
          CHANGED_FILES="${{ steps.changed-files.outputs.all_changed_files }}"
          echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
          echo "The changed files found are: $CHANGED_FILES"

    - name: Run static checks
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
          # run gofmt checks only with Go 1.18
          if [ "${{ matrix.gochannel }}" != "1.18" ]; then
              export SKIP_GOFMT=1
              echo "Formatting checks will be skipped due to the use of Go version ${{ matrix.gochannel }}"
          fi
          sudo apt-get install -y python3-yamlordereddictloader
          ./run-checks --static

    - name: Cache prebuilt indent
      id: cache-indent-bin
      uses: actions/cache@v3
      with:
        path: indent-bin
        key: ${{ runner.os }}-indent-2.2.13

    # build indent 2.2.13 which has this patch
    # https://git.savannah.gnu.org/cgit/indent.git/commit/?id=22b83d68e9a8b429590f42920e9f473a236123cf
    - name: Build indent 2.2.13
      if: steps.cache-indent-bin.outputs.cache-hit != 'true'
      run: |
          sudo apt install texinfo autopoint
          curl -O https://ftp.gnu.org/gnu/indent/indent-2.2.13.tar.xz
          tar xvf indent-2.2.13.tar.xz
          cd indent-2.2.13
          autoreconf -if
          # set prefix in case we want to pack to tar/extract into system
          ./configure --prefix=/opt/indent
          make -j
          make install DESTDIR=${{ github.workspace }}/indent-bin
          find ${{ github.workspace }}/indent-bin -ls

    - name: Check C source code formatting
      run: |
          set -x
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/
          ./autogen.sh
          # apply formatting
          PATH=${{ github.workspace }}/indent-bin/opt/indent/bin:$PATH make fmt
          set +x
          if [ -n "$(git diff --stat)" ]; then
              git diff
              echo "C files are not fomratted correctly, run 'make fmt'"
              echo "make sure to have clang-format and indent 2.2.13+ installed"
              exit 1
          fi

  branch-static-checks:
    runs-on: ubuntu-latest
    needs: [cache-build-deps]
    if: github.ref != 'refs/heads/master'
    steps:

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # needed for git commit history
        fetch-depth: 0

    - name: check-branch-ubuntu-daily-spread
      run: |
        # Compare the daily system in master and in the current branch
        wget -q -O test_master.yaml https://raw.githubusercontent.com/snapcore/snapd/master/.github/workflows/test.yaml
        system_daily="$(yq '.jobs.spread.strategy.matrix.include.[] | select(.group == "ubuntu-daily") | .systems'  test_master.yaml)"
        current_daily="$(yq '.jobs.spread.strategy.matrix.include.[] | select(.group == "ubuntu-daily") | .systems'  .github/workflows/test.yaml)"
        test "$system_daily" == "$current_daily"
      shell: bash

  unit-tests:
    needs: [static-checks]
    runs-on: ubuntu-22.04
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
    strategy:
      # we cache successful runs so it's fine to keep going
      fail-fast: false      
      matrix:
        gochannel:
          - 1.18
          - latest/stable
        unit-scenario:
          - normal

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}

    - name: Download Debian dependencies
      uses: actions/download-artifact@v3
      with:
        name: debian-dependencies
        path: ./debian-deps/

    - name: Copy dependencies
      run: |
        test -f ./debian-deps/cached-apt.tar
        sudo tar xvf ./debian-deps/cached-apt.tar -C /

    - name: Install Debian dependencies
      run: |
          sudo apt update
          sudo apt build-dep -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    # golang latest ensures things work on the edge
    - name: Install the go snap
      run: |
          sudo snap install --classic --channel=${{ matrix.gochannel }} go

    - name: Get deps
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/ && ./get-deps.sh

    - name: Build C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/
          ./autogen.sh
          make -j$(nproc)

    - name: Build Go
      run: |
          go build github.com/snapcore/snapd/...

    - name: Test C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/ && make check

    - name: Reset code coverage data
      run: |
          rm -rf ${{ github.workspace }}/.coverage/

    - name: Test Go
      if: ${{ matrix.unit-scenario == 'normal' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        ./run-checks --unit

    - name: Upload the coverage results
      if: ${{ matrix.gochannel != 'latest/stable' }}
      uses: actions/upload-artifact@v3
      with:
        name: coverage-files
        path: "${{ github.workspace }}/src/github.com/snapcore/snapd/.coverage/coverage*.cov"

  # TODO run unit tests of C code
  unit-tests-special:
    needs: [static-checks]
    runs-on: ubuntu-22.04
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
    strategy:
      # we cache successful runs so it's fine to keep going
      fail-fast: false
      matrix:
        gochannel:
          - 1.18
          - latest/stable
        unit-scenario:
          - snapd_debug
          - withbootassetstesting
          - nosecboot
          - faultinject
          - race

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}

    - name: Download Debian dependencies
      uses: actions/download-artifact@v3
      with:
        name: debian-dependencies
        path: ./debian-deps/

    - name: Copy dependencies
      run: |
        test -f ./debian-deps/cached-apt.tar
        sudo tar xvf ./debian-deps/cached-apt.tar -C /

    - name: Install Debian dependencies
      run: |
          sudo apt update
          sudo apt build-dep -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    # golang latest ensures things work on the edge
    - name: Install the go snap
      run: |
          sudo snap install --classic --channel=${{ matrix.gochannel }} go

    - name: Get deps
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/ && ./get-deps.sh

    - name: Build C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/
          ./autogen.sh
          make -j$(nproc)

    - name: Build Go
      run: |
          go build github.com/snapcore/snapd/...

    - name: Test C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/ && make check

    - name: Reset code coverage data
      run: |
          rm -rf ${{ github.workspace }}/.coverage/

    - name: Test Go (SNAPD_DEBUG=1)
      if: ${{ matrix.unit-scenario == 'snapd_debug' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 SNAPD_DEBUG=1 ./run-checks --unit

    - name: Test Go (withbootassetstesting)
      if: ${{ matrix.unit-scenario == 'withbootassetstesting' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=withbootassetstesting ./run-checks --unit

    - name: Test Go (nosecboot)
      if: ${{ matrix.unit-scenario == 'nosecboot' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        echo "Dropping github.com/snapcore/secboot"
        # use govendor remove so that a subsequent govendor sync does not
        # install secboot again
        # ${{ github.workspace }}/bin/govendor remove github.com/snapcore/secboot
        # ${{ github.workspace }}/bin/govendor remove +unused
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=nosecboot ./run-checks --unit

    - name: Test Go (faultinject)
      if: ${{ matrix.unit-scenario == 'faultinject' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=faultinject ./run-checks --unit

    - name: Test Go (-race)
      if: ${{ matrix.unit-scenario == 'race' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 GO_TEST_RACE=1 SKIP_COVERAGE=1 ./run-checks --unit

    - name: Upload the coverage results
      if: ${{ matrix.gochannel != 'latest/stable' }}
      uses: actions/upload-artifact@v3
      with:
        name: coverage-files
        path: "${{ github.workspace }}/src/github.com/snapcore/snapd/.coverage/coverage*.cov"


  unit-tests-cross-distro:
    needs: [static-checks]
    env:
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /usr/sbin:/usr/bin:/sbin:/bin

    strategy:
      fail-fast: false
      matrix:
        distro:
          # TODO add arch?
          - fedora:39
          - opensuse/tumbleweed

    runs-on: ubuntu-latest
    container: ${{ matrix.distro }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        # needed for git commit history
        fetch-depth: 0

    - name: Install dependencies
      run: |
        # approximation to handle both typical foo:bar (tagged) and foo/bar
        # (with implicit :latest)
        distroname="$(echo "${{ matrix.distro }}" | tr : - | tr / -)"
        case "${{ matrix.distro }}" in
        fedora:*)
            dnf install -y rpmdevtools
            dnf install -y $(rpmspec -q --buildrequires "./packaging/$distroname/snapd.spec")
            # TODO these are needed only by cmd/snap-seccomp unit tests, and
            # should be added to BuildRequires
            dnf install -y glibc-devel.i686 glibc-static.i686
            ;;
        opensuse/*)
            zypper --non-interactive install -y rpmdevtools rpm-build git
            zypper --non-interactive install -y $(rpmspec -q --buildrequires "./packaging/$distroname/snapd.spec")
            ;;
        *)
            echo "Unsupported distribution variant ${{ matrix.distro }}"
            exit 1
            ;;
        esac
    - name: Set up test user
      run: |
        useradd -U -m test-user
        chown -R test-user:test-user $PWD

    - name: Unit tests (Go)
      run: |
        su test-user sh -c "SKIP_DIRTY_CHECK=1 ./run-checks --unit"

    - name: Unit tests (C)
      run: |
        su test-user sh -c "./mkversion.sh 1337-git && cd ./cmd && ./autogen.sh && make -j && make check"


  code-coverage:
    needs: [unit-tests, unit-tests-special]
    runs-on: ubuntu-20.04
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
    steps:
    - name: Download the coverage files
      uses: actions/download-artifact@v3
      with:
        name: coverage-files
        path: .coverage/

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      # uploading to codecov occasionally fails, so continue running the test
      # workflow regardless of the upload
      continue-on-error: true
      with:
        fail_ci_if_error: true
        flags: unittests
        name: codecov-umbrella
        files: .coverage/coverage-*.cov
        verbose: true

  spread:
    needs: [unit-tests, snap-builds]
    name: ${{ matrix.group }}
    runs-on: [self-hosted, spread-enabled]
    strategy:
      # FIXME: enable fail-fast mode once spread can cancel an executing job.
      # Disable fail-fast mode as it doesn't function with spread. It seems
      # that cancelling tasks requires short, interruptible actions and
      # interrupting spread, notably, does not work today. As such disable
      # fail-fast while we tackle that problem upstream.
      fail-fast: false
      matrix:
        include:
          - group: amazon-linux
            backend: google-distro-1
            systems: 'amazon-linux-2-64 amazon-linux-2023-64'
            tests: 'tests/...'
          - group: arch-linux
            backend: google-distro-2
            systems: 'arch-linux-64'
            tests: 'tests/...'
          - group: centos
            backend: google-distro-2
            systems: 'centos-9-64'
            tests: 'tests/...'
          - group: debian-req
            backend: google-distro-1
            systems: 'debian-11-64'
            tests: 'tests/...'
          - group: debian-not-req
            backend: google-distro-1
            systems: 'debian-12-64 debian-sid-64'
            tests: 'tests/...'
          - group: fedora
            backend: google-distro-1
            systems: 'fedora-38-64 fedora-39-64'
            tests: 'tests/...'
          - group: fedora-os
            backend: openstack
            systems: 'fedora-40-64'
            tests: 'tests/...'
          - group: opensuse
            backend: google-distro-2
            systems: 'opensuse-15.5-64 opensuse-15.6-64 opensuse-tumbleweed-64'
            tests: 'tests/...'
          - group: ubuntu-trusty
            backend: google
            systems: 'ubuntu-14.04-64'
            tests: 'tests/smoke/ tests/main/canonical-livepatch tests/main/canonical-livepatch-14.04'
          - group: ubuntu-xenial-bionic
            backend: google
            systems: 'ubuntu-16.04-64 ubuntu-18.04-64'
            tests: 'tests/...'
          - group: ubuntu-focal-jammy
            backend: google
            systems: 'ubuntu-20.04-64 ubuntu-22.04-64'
            tests: 'tests/...'
          - group: ubuntu-noble
            backend: google
            systems: 'ubuntu-24.04-64'
            tests: 'tests/...'
          - group: ubuntu-no-lts
            backend: google
            systems: ''
            tests: 'tests/...'
          - group: ubuntu-daily
            backend: google
            systems: 'ubuntu-24.10-64'
            tests: 'tests/...'
          - group: ubuntu-core-16
            backend: google-core
            systems: 'ubuntu-core-16-64'
            tests: 'tests/...'
          - group: ubuntu-core-18
            backend: google-core
            systems: 'ubuntu-core-18-64'
            tests: 'tests/...'
          - group: ubuntu-core-20
            backend: google-core
            systems: 'ubuntu-core-20-64'
            tests: 'tests/...'
          - group: ubuntu-core-22
            backend: google-core
            systems: 'ubuntu-core-22-64'
            tests: 'tests/...'
          - group: ubuntu-core-24
            backend: google-core
            systems: 'ubuntu-core-24-64'
            tests: 'tests/...'
          - group: ubuntu-arm
            backend: google-arm
            systems: 'ubuntu-20.04-arm-64 ubuntu-core-22-arm-64'
            tests: 'tests/...'
          - group: ubuntu-secboot
            backend: google
            systems: 'ubuntu-secboot-20.04-64'
            tests: 'tests/...'

          - group: nested-ubuntu-16.04
            backend: google-nested
            systems: 'ubuntu-16.04-64'
            tests: 'tests/nested/...'
          - group: nested-ubuntu-18.04
            backend: google-nested
            systems: 'ubuntu-18.04-64'
            tests: 'tests/nested/...'
          - group: nested-ubuntu-20.04
            backend: google-nested
            systems: 'ubuntu-20.04-64'
            tests: 'tests/nested/...'
          - group: nested-ubuntu-22.04
            backend: google-nested
            systems: 'ubuntu-22.04-64'
            tests: 'tests/nested/...'
          - group: nested-ubuntu-24.04
            backend: google-nested
            systems: 'ubuntu-24.04-64'
            tests: 'tests/nested/...'
    steps:
    - name: Cleanup job workspace
      id: cleanup-job-workspace
      run: |
          rm -rf "${{ github.workspace }}"
          mkdir "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v3
      with:
        # spread uses tags as delta reference
        fetch-depth: 0

    - name: Get previous attempt
      id: get-previous-attempt
      run: |
        echo "previous_attempt=$(( ${{ github.run_attempt }} - 1 ))" >> $GITHUB_OUTPUT
      shell: bash

    - name: Get previous cache
      uses: actions/cache@v3
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.group }}-${{ steps.get-previous-attempt.outputs.previous_attempt }}"

    - name: Prepare test results env and vars
      id: prepare-test-results-env
      run: |
          # Create test results directories and save vars
          TEST_RESULTS_DIR="${{ github.workspace }}/.test-results"
          echo "TEST_RESULTS_DIR=$TEST_RESULTS_DIR" >> $GITHUB_ENV

          # Save the var with the failed tests file
          echo "FAILED_TESTS_FILE=$TEST_RESULTS_DIR/failed-tests" >> $GITHUB_ENV

          # Make sure the test results dirs are created
          # This step has to be after the cache is restored
          mkdir -p "$TEST_RESULTS_DIR"        

    - name: Check failed tests to run
      if: "!contains(github.event.pull_request.labels.*.name, 'Run all')"
      run: |
          # Save previous failed test results in FAILED_TESTS env var
          FAILED_TESTS=""
          if [ -f "$FAILED_TESTS_FILE" ]; then
              echo "Failed tests file found"
              FAILED_TESTS="$(cat $FAILED_TESTS_FILE)"
              if [ -n "$FAILED_TESTS" ]; then
                  echo "Failed tests to run: $FAILED_TESTS"
                  echo "FAILED_TESTS=$FAILED_TESTS" >> $GITHUB_ENV
              fi
          fi

    - name: Setup run tests variable
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      run: |
          RUN_TESTS=""
          # Save previous failed test results in FAILED_TESTS env var
          if [ -n "$FAILED_TESTS" ]; then
              RUN_TESTS="$FAILED_TESTS"
          else
              for SYSTEM in ${{ matrix.systems }}; do
                  for TESTS in ${{ matrix.tests }}; do
                      RUN_TESTS="$RUN_TESTS ${{ matrix.backend }}:$SYSTEM:$TESTS"
                  done
              done
          fi
          echo RUN_TESTS="$RUN_TESTS"  >> $GITHUB_ENV

    - name: Setup grafana parameters
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      run: |
          # Configure parameters to filter logs (these logs are sent read by grafana agent)
          CHANGE_ID="${{ github.event.number }}"
          if [ -z "$CHANGE_ID" ]; then
            CHANGE_ID="main"
          fi
          FILTERED_LOG_FILE="spread_${CHANGE_ID}_n${{ github.run_attempt }}.filtered.log"
          # The log-filter tool is used to filter the spread logs to be stored
          echo FILTER_PARAMS="-o $FILTERED_LOG_FILE -e Debug -e WARNING: -f Failed=NO_LINES -f Error=NO_LINES"  >> $GITHUB_ENV
          echo FILTERED_LOG_FILE="$FILTERED_LOG_FILE"  >> $GITHUB_ENV

          # Add start line to filtered log
          echo "GRAFANA START: pr ${CHANGE_ID} attempt ${{ github.run_attempt }} run ${{ github.run_id }} group ${{ matrix.group }}" > "$FILTERED_LOG_FILE"

    - name: Download built snap
      uses: actions/download-artifact@v3
      with:
        name: snap-files-default-test
        pattern: snapd_1337.*.snap
        path: "${{ github.workspace }}/built-snap"

    - name: Rename imported snap
      run: |
        for snap in built-snap/snapd_1337.*.snap; do
          mv "${snap}" "${snap}.keep"
        done

    - name: Run spread tests
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread') && ( !startsWith(matrix.group, 'nested-') || contains(github.event.pull_request.labels.*.name, 'Run nested') )"
      env:
          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
      run: |
          # Register a problem matcher to highlight spread failures
          echo "::add-matcher::.github/spread-problem-matcher.json"
          set -x
          SPREAD=spread
          if [[ "${{ matrix.group }}" =~ nested- ]]; then
            export NESTED_BUILD_SNAPD_FROM_CURRENT=true
            export NESTED_ENABLE_KVM=true
          fi

          case "${{ matrix.systems }}" in
            *-arm-*)
              SPREAD_USE_PREBUILT_SNAPD_SNAP=false
              ;;
            *)
              SPREAD_USE_PREBUILT_SNAPD_SNAP=true
              ;;
          esac
          export SPREAD_USE_PREBUILT_SNAPD_SNAP

          if [[ "${{ matrix.systems }}" =~ amazon-linux-2023 ]]; then
              # Amazon Linux 2023 has no xdelta, however we cannot disable
              # xdelta on a per-target basis as it's used in the repack section
              # of spread.yaml, which is shared by all targets, so all systems
              # in this batch will not use delta for transferring project data
              echo "Disabling xdelta support"
              export NO_DELTA=1
          fi

          # Add openstack backend definition to spread.yaml
          if [ "${{ matrix.backend }}" = openstack ]; then
              ./tests/lib/spread/add-backend tests/lib/spread/backend.openstack.yaml spread.yaml
          fi

          # This coud be the case when either there are not systems for a group or 
          # the list of tests to run is empty
          if [ -z "$RUN_TESTS" ]; then
            echo "No tests to run, skiping..."
            exit 0
          fi

          # Run spread tests
          # "pipefail" ensures that a non-zero status from the spread is
          # propagated; and we use a subshell as this option could trigger
          # undesired changes elsewhere
          echo "Running command: $SPREAD $RUN_TESTS"
          (set -o pipefail; $SPREAD -no-debug-output -logs .logs $RUN_TESTS | PYTHONDONTWRITEBYTECODE=1 ./tests/lib/external/snapd-testing-tools/utils/log-filter $FILTER_PARAMS | tee spread.log)


    - name: Uploading spread logs
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: spread-logs-${{ matrix.systems }}
        path: ".logs/*.log"

    - name: Discard spread workers
      if: always()
      run: |
        shopt -s nullglob;
        for r in .spread-reuse.*.yaml; do
          spread -discard -reuse-pid="$(echo "$r" | grep -o -E '[0-9]+')";
        done

    - name: Report spread errors
      if: always()
      run: |
        if [ -e spread.log ]; then
            echo "Running spread log analyzer"
            ACTIONS_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
            PYTHONDONTWRITEBYTECODE=1 ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json --cut 1 >/dev/null
            while IFS= read -r line; do
                if [ ! -z "$line" ]; then
                    echo "Adding failed test line to filtered log"
                    echo "GRAFANA FAILED: $line $ACTIONS_URL" | tee -a "$FILTERED_LOG_FILE"
                fi
            done <<< $(jq -r '.[] | select( .type == "info" ) | select( .info_type == "Error" ) | "\(.verb) \(.task)"' spread-results.json)
        else
            echo "No spread log found, skipping errors reporting"
        fi

    - name: Analyze spread test results
      if: always()
      run: |
          if [ -f spread.log ]; then
              echo "Running spread log parser"
              PYTHONDONTWRITEBYTECODE=1 ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json

              # Add openstack backend definition to spread.yaml
              if [ "${{ matrix.backend }}" = openstack ]; then
                  ./tests/lib/spread/add-backend tests/lib/spread/backend.openstack.yaml spread.yaml
              fi

              echo "Running spread log analyzer"
              PYTHONDONTWRITEBYTECODE=1 ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"

              echo "List of failed tests saved"
              cat "$FAILED_TESTS_FILE"
          else
              echo "No spread log found, saving empty list of failed tests"
              touch "$FAILED_TESTS_FILE"
          fi

    - name: Save spread test results to cache
      if: always()
      uses: actions/cache/save@v3
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.group }}-${{ github.run_attempt }}"