consistency-functional.yml

name: consistency-functional
on:
  push:
    paths-ignore:
      - 'docs/**'
    branches:
      - main
  pull_request:
  # Run the functional tests every 8 hours.
  # This will help to identify faster if
  # there is a CI failure related to a
  # problem deploying DevStack.
  schedule:
    - cron: '0 */8 * * *'

jobs:
  # TODO: Add a previous job that:
  # * Builds the container image
  # * Store the container image to be reused in all jobs
  # * Import the container image on each job
  build:
    if: github.repository_owner == 'os-migrate'
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.7]
        ansible-version: [2.9, 'latest']
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo locale-gen en_US.UTF-8
          sudo dpkg-reconfigure locales
          sudo python -m pip install --upgrade pip
          sudo pip install tox
          sudo apt install software-properties-common build-essential findutils -y
      - name: Print podman version
        run: |
          podman -v
      - name: Configure podman
        run: |
          mkdir -p ~/.config/containers
          echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf
          echo '[storage]' >> ~/.config/containers/storage.conf
          echo 'driver = "overlay"' >> ~/.config/containers/storage.conf
          echo '[storage.options]' >> ~/.config/containers/storage.conf
          echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf
      - name: Log into GitHub Container Registry
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | \
            podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
      - name: Get the changed files list
        id: file_changes
        if: github.event_name != 'schedule'
        uses: trilom/file-changes-action@v1.2.4
      - name: Build or reuse the toolbox container image
        if: github.event_name != 'schedule'
        run: |
          # The following array defines the expressions that
          # when matching the container image for the toolbox
          # will be created from scratch, otherwise we reuse
          # it from the latest build.
          check_list=('toolbox' 'Makefile' 'tests' 'scripts')

          REUSE=1
          filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') )
          for file in "${filez[@]}"; do
            for index in ${!check_list[*]}; do
              if [[ "${file}" == *"${check_list[$index]}"* ]]; then
                echo "We need to re-create the toolbox container image"
                echo "${check_list[$index]} is in ${file}"
                REUSE=0
              fi
            done
          done
          REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build
      - name: Reuse the toolbox container image for the scheduled job
        if: github.event_name == 'schedule'
        run: |
          # In this case we assume that the scheduled job will
          # pull the container image from the local container
          # registry because that should be the latest correct
          # toolbox image.
          REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build
      - name: Run all the tests non OpenStack dependant
        run: |
          export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core'
          export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements-ansible-${{ matrix.ansible-version }}.txt'
          ./toolbox/run make test-fast
      - name: Validate changelog
        run: |
          set -x
          git log --oneline | head -n50
          make lint-commit-messages


  functional:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.7]
        ansible-version: [2.9, 'latest']
    services:
      rabbitmq:
        image: rabbitmq:latest
        ports:
          - 5672:5672
        options: >-
          --health-cmd "rabbitmqctl node_health_check"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      mysql:
        image: mysql:5.7
        env:
          MYSQL_ROOT_PASSWORD: root
          MYSQL_ALLOW_EMPTY_PASSWORD: true
        ports:
          - 3306:3306
        options: >-
          --health-cmd="mysqladmin ping"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=3
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo locale-gen en_US.UTF-8
          sudo dpkg-reconfigure locales
          sudo apt update
          sudo apt remove --purge mysql-server mysql-client mysql-common -y
          sudo apt autoremove -y
          # MySQL requirements
          sudo apt install -y mysql-client mysql-common
          # Httplib2 requirements
          # We need to install this library un purpose, if not the devstack
          # installer will crash as pip is not able to determine if its
          # installed or not
          sudo apt install python3-httplib2 -y
          sudo apt remove postgresql-client-common
          sudo apt install postgresql-client-common=238
          sudo apt install postgresql-common
          sudo python -m pip install --upgrade pip
          sudo python -m pip install --upgrade virtualenv
          # https://github.com/pypa/setuptools/issues/3772
          sudo python -m pip install --upgrade 'setuptools<66'
      - name: Configure podman
        run: |
          mkdir -p ~/.config/containers
          echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf
          echo '[storage]' >> ~/.config/containers/storage.conf
          echo 'driver = "overlay"' >> ~/.config/containers/storage.conf
          echo '[storage.options]' >> ~/.config/containers/storage.conf
          echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf
      - name: Verify MySQL connection from container
        run: >-
          mysql \
            --host 127.0.0.1 \
            -P 3306 \
            -uroot \
            -proot \
            -e "SHOW DATABASES"
          mysql \
            --host 127.0.0.1 \
            -P 3306 \
            -uroot \
            -proot \
            -e 'USE mysql;
                DELETE FROM `user` WHERE `Host` != "%" AND `User`="root";
                FLUSH PRIVILEGES;'
      # Devstack is not able to configure an external MySQL backend.
      # There is a mandatory DB server restart from the stack.sh script
      # that restart a service called mysql, even if the service is not
      # supposed to be configured locally.
      # In this case we configure the database to point to the external
      # database container and we mock a dummy service called "mysql"
      # to trick stack.sh and not fail.
      - name: Create a dummy MySQL service to trick stack.sh
        run: |
          sudo tee /usr/mock.sh > /dev/null <<'EOF'
          sleep 3600;
          EOF
          sudo chmod u+x /usr/mock.sh
          sudo tee /lib/systemd/system/mysql.service > /dev/null <<'EOF'
          [Unit]
          Description=MySQL mock service
          [Service]
          Type=simple
          ExecStart=/bin/bash /usr/mock.sh
          [Install]
          WantedBy=multi-user.target
          EOF
          sudo systemctl enable mysql.service
          sudo systemctl start mysql.service
          sudo systemctl restart mysql.service
          sudo systemctl status mysql.service
      # there is a script inside devstack that checks to ensure rabbit is
      # enabled.  because we're using rabbit in a container and devstack cannot
      # manage it, we mock a service so it won't fail.
      - name: Create a dummy rabbit service to trick devstack/lib/rpc_backend
        run: |
          sudo tee /usr/mock.sh > /dev/null <<'EOF'
          sleep 3600;
          EOF
          sudo chmod u+x /usr/mock.sh
          sudo tee /lib/systemd/system/rabbit.service > /dev/null <<'EOF'
          [Unit]
          Description=rabbit mock service
          [Service]
          Type=simple
          ExecStart=/bin/bash /usr/mock.sh
          [Install]
          WantedBy=multi-user.target
          EOF
          sudo systemctl enable rabbit.service
          sudo systemctl start rabbit.service
          sudo systemctl restart rabbit.service
          sudo systemctl status rabbit.service
      - name: Log into GitHub Container Registry
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | \
            podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
      - name: Get the changed files list
        id: file_changes
        if: github.event_name != 'schedule'
        uses: trilom/file-changes-action@v1.2.4
      - name: Build or reuse the toolbox container image
        if: github.event_name != 'schedule'
        run: |
          # The following array defines the expressions that
          # when matching the container image for the toolbox
          # will be created from scratch, otherwise we reuse
          # it from the latest build.
          check_list=('toolbox' 'Makefile' 'tests' 'scripts')

          REUSE=1
          filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') )
          for file in "${filez[@]}"; do
            for index in ${!check_list[*]}; do
              if [[ "${file}" == *"${check_list[$index]}"* ]]; then
                echo "We need to re-create the toolbox container image"
                echo "${check_list[$index]} is in ${file}"
                REUSE=0
              fi
            done
          done
          REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build
      - name: Reuse the toolbox container image for the scheduled job
        if: github.event_name == 'schedule'
        run: |
          # In this case we assume that the scheduled job will
          # pull the container image from the local container
          # registry because that should be the latest correct
          # toolbox image.
          REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build
      # We're hitting filesystem ownership issue on some GH Actions workers:
      # https://ubuntuforums.org/showthread.php?t=2406453
      - name: Make sure filesystem ownership is correct
        run: |
          set -x
          sudo chown root:root /
          sudo chown root:root /bin
      - name: Clone devstack
        run: |
          git clone --single-branch --branch master https://opendev.org/openstack/devstack
      - name: Configure devstack
        run: |
          DIR=$(pwd)
          cd devstack
          cat << EOF > local.conf
          [[local|localrc]]
          USE_PYTHON3=True
          SERVICE_PASSWORD=devstack
          ADMIN_PASSWORD=devstack
          SERVICE_TOKEN=devstack
          DATABASE_PASSWORD=root
          DATABASE_TYPE=mysql
          HOST_IP=127.0.0.1
          SERVICE_HOST=127.0.0.1
          DATABASE_HOST=127.0.0.1
          RABBIT_HOST=127.0.0.1
          RABBIT_USERID=guest
          RABBIT_PASSWORD=guest
          VIRT_DRIVER=fake
          API_WORKERS=1
          NUMBER_FAKE_NOVA_COMPUTE=2
          CELLSV2_SETUP="singleconductor"
          OS_PLACEMENT_CONFIG_DIR=/etc/nova
          INSTALL_DATABASE_SERVER_PACKAGES=False
          # Enable only IPv4
          IP_VERSION=4
          # Horizon is enabled by default, but
          # its not required
          disable_service horizon
          # Pre-requisite
          ENABLED_SERVICES=mysql,key
          # Neutron
          # Disable Neutron agents not used with OVN (q-agt,q-l3,q-dhcp,q-meta)
          ENABLED_SERVICES+=,q-svc,q-ovn-metadata-agent
          # Nova
          ENABLED_SERVICES+=,n-sproxy,n-api,n-cond
          # Placement
          ENABLED_SERVICES+=,placement,placement-api,placement-client
          # OVN
          ENABLED_SERVICES+=,ovn-controller,ovn-northd,ovs-vswitchd,ovsdb-server
          # Glance
          ENABLED_SERVICES+=,g-api
          # OVN options
          OVN_L3_CREATE_PUBLIC_NETWORK=false
          [[post-config|/etc/nova/nova.conf]]
          [placement]
          auth_type = password
          auth_url = http://127.0.0.1/identity
          password = devstack
          project_domain_name = Default
          project_name = service
          user_domain_name = Default
          username = nova
          [compute]
          live_migration_wait_for_vif_plug=True
          [libvirt]
          live_migration_uri = qemu+ssh://root@%s/system
          cpu_mode = host-passthrough
          virt_type = qemu
          EOF
          cat local.conf
          sudo ./tools/create-stack-user.sh
          sudo mv $DIR/devstack /opt/stack/devstack
          sudo chown -R stack: /opt/stack/devstack
      - name: Start devstack
        run: |
          # Hacks to make Devstack install fine. See also:
          # https://github.com/openstack/devstack/blob/83821a11ac1d6738b63cb10878b8aaa02e153374/tools/fixup_stuff.sh#L88-L96
          sudo rm -rf /usr/lib/python3/dist-packages/httplib2-*.egg-info
          sudo rm -rf /usr/lib/python3/dist-packages/pyasn1_modules-*.egg-info
          sudo rm -rf /usr/lib/python3/dist-packages/PyYAML-*.egg-info
          sudo rm -rf /usr/lib/python3/dist-packages/simplejson-*.egg-info

          sudo -iu stack bash -c 'cd /opt/stack/devstack; ./stack.sh'
      - name: Get debug data
        if: failure()
        run: |
          echo ">>> Journal logs"
          sudo journalctl --no-pager
      - name: See "systemctl status placement-api.service" for details.
        if: failure()
        run: |
          sudo systemctl status placement-api.service
      - name: See "journalctl -xe" for details.
        if: failure()
        run: |
          sudo journalctl -xe
      - name: Run sanity checks
        run: |
          echo "- Get network and routers list"
          openstack --os-cloud devstack-admin --os-region RegionOne network list
          openstack --os-cloud devstack-admin --os-region RegionOne router list
      - name: Check keystone
        run: |
          echo "- Get endpoints"
          openstack --os-cloud devstack-admin --os-region RegionOne endpoint list
      - name: Connect functional tests to devstack
        run: |
          cp /etc/openstack/clouds.yaml tests/clouds.yaml && \
          ./toolbox/run ./scripts/auth-from-clouds.sh \
              --config tests/clouds.yaml \
              --src devstack \
              --dst devstack-alt-member \
              > tests/auth_tenant.yml && \
          ./toolbox/run ./scripts/auth-from-clouds.sh \
              --config tests/clouds.yaml \
              --src devstack-admin \
              --dst devstack-admin \
              > tests/auth_admin.yml && \
          rm tests/clouds.yaml
      - name: Run functional tests
        run: |
          export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core'
          export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements-ansible-${{ matrix.ansible-version }}.txt'
          ./toolbox/run make test-func