ci.yml

name: CI

on: [push, pull_request]

#
# Make sure to update the copy of this in `iptfs-linux` as well.
#

jobs:
  build-fast-kernel:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Checkout kernel repository
        uses: actions/checkout@v4
        with:
          repository: LabNConsulting/iptfs-linux
          path: linux

      - name: Install build depencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils

      - name: Kernel build cache
        id: linux-cache
        uses: actions/cache@v4
        with:
          path: output-linux.tar.xz
          # We need a unique key name in order for new versions to save
          key: ${{ runner.os }}-kernel-${{ hashFiles('linux-fast.config', 'output-linux.tar.xz') }}
          restore-keys: |
            key: ${{ runner.os }}-kernel-

      - name: Build kernel
        run: |
          set -x
          rm -rf output-linux
          if [ -f output-linux.tar.xz ]; then
            echo "Cache found, extracting"
            tar -xpJf output-linux.tar.xz
            # If the kernel config is changed just wipe the cache
            if ! diff -q -u1 linux-fast.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
              echo "Kernel configs differ erasing cache"
              rm -rf output-linux output-linux.tar.xz
            fi
          fi

          export CCACHE_DIR=$PWD/output-linux/.ccache
          if [ ! -d output-linux ]; then
            echo "Creating new output directory"
            mkdir output-linux
            cp linux-fast.config output-linux/.config
          fi
          cd output-linux
          ccache -z
          KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux
          ccache -sz
          [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage ..
          # Ccache keeps it's own objects
          # make clean

          # tar up the remaining files including .ccache
          cd ..
          echo "Collecting new cache contents"
          tar -cf - output-linux | xz -T0 -c > output-linux.tar.xz
          ls -hl output-linux.tar.xz

      - name: Archive kernel bzImage
        uses: actions/upload-artifact@v4
        with:
          name: kernel-bzImage
          path: bzImage
          retention-days: 1

  build-cov-kernel:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Checkout kernel repository
        uses: actions/checkout@v4
        with:
          repository: LabNConsulting/iptfs-linux
          path: linux

      - name: Install build depencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils

      - name: Coverage kernel build cache
        id: linux-cov-cache
        uses: actions/cache@v4
        with:
          path: output-cov-linux.tar.xz
          # We need a unique key name in order for new versions to save
          key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }}
          restore-keys: |
            key: ${{ runner.os }}-cov-kernel-

      - name: Build coverage kernel
        run: |
          set -x
          rm -rf output-linux
          if [ -f output-cov-linux.tar.xz ]; then
            echo "Cache found, extracting"
            tar -xpJf output-cov-linux.tar.xz
            # If the kernel config is changed just wipe the cache
            if ! diff -q -u1 linux-cov.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
              echo "Kernel configs differ erasing cache"
              rm -rf output-linux output-cov-linux.tar.xz
            fi
          fi

          export CCACHE_DIR=$PWD/output-linux/.ccache
          if [ ! -d output-linux ]; then
            echo "Creating new output directory"
            mkdir output-linux
            cp linux-cov.config output-linux/.config
          fi
          cd output-linux
          ccache -z
          KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux
          ccache -sz
          [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage ..
          # Ccache keeps it's own objects
          # We want the objects and images too for now, restroy once we know
          # coverage works
          # make clean

          # tar up the remaining files including .ccache
          cd ..
          echo "Collecting new cache contents"
          tar -cf - output-linux | xz -T0 -c > output-cov-linux.tar.xz
          ls -hl output-cov-linux.tar.xz

      - name: Archive coverage kernel bzImage
        uses: actions/upload-artifact@v4
        with:
          name: kernel-cov-bzImage
          path: bzImage
          retention-days: 1

  build-rootfs:
    runs-on: ubuntu-22.04
    timeout-minutes: 600
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Checkout repos
        run: make setup SHALLOW_CLONE=1

      - name: Install build depencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y device-tree-compiler libelf-dev

      - name: Buildroot Build Cache
        id: buildroot-cache
        uses: actions/cache@v4
        with:
          path: output-buildroot.tar.gz
          key: ${{ runner.os }}-br-${{ hashFiles('buildroot-ci.config', 'output-buildroot.tar.gz', 'root-key', 'root-key.pub') }}
          restore-keys: |
            key: ${{ runner.os }}-br-

      - name: Build Rootfs
        timeout-minutes: 540
        run: |
          set -x
          if [ -f output-buildroot.tar.xz ] || [ -f output-buildroot.tar.gz ]; then
            echo "Cache found, extracting"
            if [ -f output-buildroot.tar.xz ]; then
              tar -xpJf output-buildroot.tar.xz
            else
              tar -xpzf output-buildroot.tar.gz
            fi
            # If the buildroot config is changed just wipe the cache
            if ! diff -q -u1 buildroot-ci.config output-buildroot/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
              echo "Buildroot configs differ erasing cache"
              rm -rf output-buildroot output-buildroot.tar.gz
            fi
          fi

          if [ ! -d output-buildroot ]; then
            echo "Creating new output directory"
            mkdir -p output-buildroot
            cp -p buildroot-ci.config output-buildroot/.config
          else
            make -C buildroot -j6 V=0 O=$PWD/output-buildroot iproute2-dirclean
          fi
          # make rootfs
          echo "Starting build: $(date)"
          make -C buildroot -j6 V=1 O=$PWD/output-buildroot LOCALVERSION='' > build-buildroot.txt 2>&1
          echo "Build complete: $(date)"

      - name: Archive buildroot build log
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: buildroot-build-log
          path: build-buildroot.txt
          retention-days: 30
          if-no-files-found: error

      - name: Collect cache contents
        timeout-minutes: 120
        run: |
          # tar up the remaining files including .ccache
          echo "Collecting new cache contents: $(date)"
          tar -czvf output-buildroot.tar.gz output-buildroot root-key root-key.pub || echo "error with tarball creation"
          ls -hl output-buildroot.tar.gz || true
          echo "Done collecting new cache contents: $(date)"

      - name: Archive buildroot rootfs.cpio.gz
        uses: actions/upload-artifact@v4
        with:
          name: rootfs-compressed-cpio
          path: output-buildroot/images/rootfs.cpio.gz
          retention-days: 1
          if-no-files-found: error

      - name: Archive buildroot root-key
        uses: actions/upload-artifact@v4
        with:
          name: root-key
          path: |
            root-key
            root-key.pub
          retention-days: 1
          if-no-files-found: error

  test:
    runs-on: ubuntu-22.04
    needs: ["build-cov-kernel", "build-rootfs"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Checkout kernel repository
        uses: actions/checkout@v4
        with:
          repository: LabNConsulting/iptfs-linux
          path: linux

      - name: Install test depencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y lcov qemu-system-x86 socat
          sudo python3 -m pip install -U munet pytest pytest-asyncio scapy

      #
      # For coverage data, extract the source and full build to get the gcno
      # files.
      #

      - name: Restore coverage kernel build
        id: linux-cov-cache-restore
        uses: actions/cache/restore@v3
        with:
          path: output-cov-linux.tar.xz
          # We need a unique key name in order for new versions to save
          key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }}
          restore-keys: |
            key: ${{ runner.os }}-cov-kernel-

      - name: Extract coverage kernel build
        run: |
          [ -d output-linux ] && echo "Error: found existinb build" && exit 1
          tar -xvpJf output-cov-linux.tar.xz

      #
      # We only need to artifacts for buildroot
      #
      - name: Fetch buildroot rootfs.cpio.gz archive
        uses: actions/download-artifact@v4
        with:
          name: rootfs-compressed-cpio

      - name: Fetch root-key
        uses: actions/download-artifact@v4
        with:
          name: root-key

      # Would be nice to loop this w/ and w/o coverage

      - name: Prep tests
        run: |
          test -f output-linux/arch/x86/boot/bzImage
          mkdir -p output-buildroot/images/
          mv rootfs.cpio.gz output-buildroot/images/
          mkdir -p test-logs

      - name: Print the environment under sudo
        run: sudo -E env

      - name: Config test
        run: |
          set -e
          tmpf=test-logs/results-config.txt
          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/config | tee $tmpf
          grep -v SKIPPED $tmpf

      - name: Errors test
        run: |
          set -e
          tmpf=test-logs/results-errors.txt
          tmpf=/tmp/test-results-$$.txt
          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/errors | tee $tmpf
          grep -v SKIPPED $tmpf

      - name: Simple test
        run: |
          set -e
          tmpf=test-logs/results-simplenet.txt
          tmpf=/tmp/test-results-$$.txt
          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/simplenet | tee $tmpf
          grep -v SKIPPED $tmpf

      # Restore longer running test once working
      - name: UT packet test
        run: |
          set -e
          tmpf=test-logs/results-utpkt.txt
          sudo python3 -m pytest -s --coverage tests/utpkt | tee $tmpf
          grep -v SKIPPED $tmpf

      - name: Extract coverage data
        if: ${{ always() }}
        run: |
          make ci-extract-cov

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.info
          flags: unittests
          root_dir: ./linux
          verbose: true

      - name: Collect test logs
        if: ${{ always() }}
        run: |
          sudo find /tmp/unet-test -name 'gcov-data.tgz' -exec rm {} +
          sudo find /tmp/unet-test -type s -exec rm {} +
          sudo tar -C /tmp/unet-test -cf - . | tar -C test-logs -xf -
          tar -cjf test-logs.tar.bz2 test-logs

      - name: Archive test logs tarball
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: test-logs-tar
          path: test-logs.tar.bz2

  deploy:
    runs-on: ubuntu-22.04
    if: startsWith(github.ref, 'refs/tags/')
    needs: ["build-fast-kernel", "build-rootfs", "test"]
    steps:

      - name: Download kernel bzImage
        uses: actions/download-artifact@v4
        with:
          name: kernel-bzImage

      - name: Download buildroot rootfs.cpio.gz
        uses: actions/download-artifact@v4
        with:
          name: rootfs-compressed-cpio

      - name: Downlaod root-key
        uses: actions/download-artifact@v4
        with:
          name: root-key

      - name: Deploy
        uses: softprops/action-gh-release@v1
        with:
          files: |
            bzImage
            root-key
            root-key.pub
            rootfs.cpio.gz