consistency-functional #2788
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: consistency-functional | |
| on: | |
| push: | |
| paths-ignore: | |
| - 'docs/**' | |
| branches: | |
| - main | |
| pull_request: | |
| # Run the functional tests every 8 hours. | |
| # This will help to identify faster if | |
| # there is a CI failure related to a | |
| # problem deploying DevStack. | |
| schedule: | |
| - cron: '0 */8 * * *' | |
| jobs: | |
| # TODO: Add a previous job that: | |
| # * Builds the container image | |
| # * Store the container image to be reused in all jobs | |
| # * Import the container image on each job | |
| build: | |
| if: github.repository_owner == 'os-migrate' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| python-version: [3.7] | |
| ansible-version: ['latest'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo locale-gen en_US.UTF-8 | |
| sudo dpkg-reconfigure locales | |
| sudo python -m pip install --upgrade pip | |
| sudo pip install tox | |
| sudo apt install software-properties-common build-essential findutils -y | |
| - name: Print podman version | |
| run: | | |
| podman -v | |
| - name: Configure podman | |
| run: | | |
| mkdir -p ~/.config/containers | |
| echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf | |
| echo '[storage]' >> ~/.config/containers/storage.conf | |
| echo 'driver = "overlay"' >> ~/.config/containers/storage.conf | |
| echo '[storage.options]' >> ~/.config/containers/storage.conf | |
| echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf | |
| - name: Log into GitHub Container Registry | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | \ | |
| podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin | |
| - name: Get the changed files list | |
| id: file_changes | |
| if: github.event_name != 'schedule' | |
| uses: trilom/[email protected] | |
| - name: Build or reuse the toolbox container image | |
| if: github.event_name != 'schedule' | |
| run: | | |
| # The following array defines the expressions that | |
| # when matching the container image for the toolbox | |
| # will be created from scratch, otherwise we reuse | |
| # it from the latest build. | |
| check_list=('toolbox' 'Makefile' 'tests' 'scripts') | |
| REUSE=1 | |
| filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') ) | |
| for file in "${filez[@]}"; do | |
| for index in ${!check_list[*]}; do | |
| if [[ "${file}" == *"${check_list[$index]}"* ]]; then | |
| echo "We need to re-create the toolbox container image" | |
| echo "${check_list[$index]} is in ${file}" | |
| REUSE=0 | |
| fi | |
| done | |
| done | |
| REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build | |
| - name: Reuse the toolbox container image for the scheduled job | |
| if: github.event_name == 'schedule' | |
| run: | | |
| # In this case we assume that the scheduled job will | |
| # pull the container image from the local container | |
| # registry because that should be the latest correct | |
| # toolbox image. | |
| REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build | |
| - name: Run all the tests non OpenStack dependant | |
| run: | | |
| export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core' | |
| export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements.txt' | |
| ./toolbox/run make test-fast | |
| - name: Validate changelog | |
| run: | | |
| set -x | |
| git log --oneline | head -n50 | |
| make lint-commit-messages | |
| functional: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| python-version: [3.7] | |
| ansible-version: ['latest'] | |
| services: | |
| rabbitmq: | |
| image: rabbitmq:latest | |
| ports: | |
| - 5672:5672 | |
| options: >- | |
| --health-cmd "rabbitmqctl node_health_check" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| mysql: | |
| image: mysql:5.7 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| MYSQL_ALLOW_EMPTY_PASSWORD: true | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=3 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo locale-gen en_US.UTF-8 | |
| sudo dpkg-reconfigure locales | |
| sudo apt update | |
| sudo apt remove --purge mysql-server mysql-client mysql-common -y | |
| sudo apt autoremove -y | |
| # MySQL requirements | |
| sudo apt install -y mysql-client mysql-common | |
| # Httplib2 requirements | |
| # We need to install this library un purpose, if not the devstack | |
| # installer will crash as pip is not able to determine if its | |
| # installed or not | |
| sudo apt install python3-httplib2 -y | |
| sudo apt remove postgresql-client-common | |
| sudo apt install postgresql-client-common=238 | |
| sudo apt install postgresql-common | |
| sudo python -m pip install --upgrade pip | |
| sudo python -m pip install --upgrade virtualenv | |
| # https://github.com/pypa/setuptools/issues/3772 | |
| # sudo python -m pip install --upgrade 'setuptools<66' | |
| sudo python -m pip install --upgrade build | |
| # Installing wheel separately because it may be needed to build some | |
| # of the packages during dependency backtracking and pip >= 22.0 will | |
| # abort backtracking on build failures: | |
| # https://github.com/pypa/pip/issues/10655 | |
| sudo pip install --disable-pip-version-check --user wheel | |
| sudo pip install --disable-pip-version-check --user \ | |
| ovs sphinx setuptools pyelftools pyOpenSSL ncclient logutils | |
| - name: Configure podman | |
| run: | | |
| mkdir -p ~/.config/containers | |
| echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf | |
| echo '[storage]' >> ~/.config/containers/storage.conf | |
| echo 'driver = "overlay"' >> ~/.config/containers/storage.conf | |
| echo '[storage.options]' >> ~/.config/containers/storage.conf | |
| echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf | |
| - name: Verify MySQL connection from container | |
| run: >- | |
| mysql \ | |
| --host 127.0.0.1 \ | |
| -P 3306 \ | |
| -uroot \ | |
| -proot \ | |
| -e "SHOW DATABASES" | |
| mysql \ | |
| --host 127.0.0.1 \ | |
| -P 3306 \ | |
| -uroot \ | |
| -proot \ | |
| -e 'USE mysql; | |
| DELETE FROM `user` WHERE `Host` != "%" AND `User`="root"; | |
| FLUSH PRIVILEGES;' | |
| # Devstack is not able to configure an external MySQL backend. | |
| # There is a mandatory DB server restart from the stack.sh script | |
| # that restart a service called mysql, even if the service is not | |
| # supposed to be configured locally. | |
| # In this case we configure the database to point to the external | |
| # database container and we mock a dummy service called "mysql" | |
| # to trick stack.sh and not fail. | |
| - name: Create a dummy MySQL service to trick stack.sh | |
| run: | | |
| sudo tee /usr/mock.sh > /dev/null <<'EOF' | |
| sleep 3600; | |
| EOF | |
| sudo chmod u+x /usr/mock.sh | |
| sudo tee /lib/systemd/system/mysql.service > /dev/null <<'EOF' | |
| [Unit] | |
| Description=MySQL mock service | |
| [Service] | |
| Type=simple | |
| ExecStart=/bin/bash /usr/mock.sh | |
| [Install] | |
| WantedBy=multi-user.target | |
| EOF | |
| sudo systemctl enable mysql.service | |
| sudo systemctl start mysql.service | |
| sudo systemctl restart mysql.service | |
| sudo systemctl status mysql.service | |
| # there is a script inside devstack that checks to ensure rabbit is | |
| # enabled. because we're using rabbit in a container and devstack cannot | |
| # manage it, we mock a service so it won't fail. | |
| - name: Create a dummy rabbit service to trick devstack/lib/rpc_backend | |
| run: | | |
| sudo tee /usr/mock.sh > /dev/null <<'EOF' | |
| sleep 3600; | |
| EOF | |
| sudo chmod u+x /usr/mock.sh | |
| sudo tee /lib/systemd/system/rabbit.service > /dev/null <<'EOF' | |
| [Unit] | |
| Description=rabbit mock service | |
| [Service] | |
| Type=simple | |
| ExecStart=/bin/bash /usr/mock.sh | |
| [Install] | |
| WantedBy=multi-user.target | |
| EOF | |
| sudo systemctl enable rabbit.service | |
| sudo systemctl start rabbit.service | |
| sudo systemctl restart rabbit.service | |
| sudo systemctl status rabbit.service | |
| - name: Log into GitHub Container Registry | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | \ | |
| podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin | |
| - name: Get the changed files list | |
| id: file_changes | |
| if: github.event_name != 'schedule' | |
| uses: trilom/[email protected] | |
| - name: Build or reuse the toolbox container image | |
| if: github.event_name != 'schedule' | |
| run: | | |
| # The following array defines the expressions that | |
| # when matching the container image for the toolbox | |
| # will be created from scratch, otherwise we reuse | |
| # it from the latest build. | |
| check_list=('toolbox' 'Makefile' 'tests' 'scripts') | |
| REUSE=1 | |
| filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') ) | |
| for file in "${filez[@]}"; do | |
| for index in ${!check_list[*]}; do | |
| if [[ "${file}" == *"${check_list[$index]}"* ]]; then | |
| echo "We need to re-create the toolbox container image" | |
| echo "${check_list[$index]} is in ${file}" | |
| REUSE=0 | |
| fi | |
| done | |
| done | |
| REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build | |
| - name: Reuse the toolbox container image for the scheduled job | |
| if: github.event_name == 'schedule' | |
| run: | | |
| # In this case we assume that the scheduled job will | |
| # pull the container image from the local container | |
| # registry because that should be the latest correct | |
| # toolbox image. | |
| REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build | |
| # We're hitting filesystem ownership issue on some GH Actions workers: | |
| # https://ubuntuforums.org/showthread.php?t=2406453 | |
| - name: Make sure filesystem ownership is correct | |
| run: | | |
| set -x | |
| sudo chown root:root / | |
| sudo chown root:root /bin | |
| - name: Clone devstack | |
| run: | | |
| git clone --single-branch --branch master https://opendev.org/openstack/devstack | |
| - name: Configure devstack | |
| run: | | |
| DIR=$(pwd) | |
| cd devstack | |
| cat << EOF > local.conf | |
| [[local|localrc]] | |
| USE_PYTHON3=True | |
| SERVICE_PASSWORD=devstack | |
| ADMIN_PASSWORD=devstack | |
| SERVICE_TOKEN=devstack | |
| DATABASE_PASSWORD=root | |
| DATABASE_TYPE=mysql | |
| HOST_IP=127.0.0.1 | |
| SERVICE_HOST=127.0.0.1 | |
| DATABASE_HOST=127.0.0.1 | |
| RABBIT_HOST=127.0.0.1 | |
| RABBIT_USERID=guest | |
| RABBIT_PASSWORD=guest | |
| VIRT_DRIVER=fake | |
| API_WORKERS=1 | |
| NUMBER_FAKE_NOVA_COMPUTE=2 | |
| CELLSV2_SETUP="singleconductor" | |
| OS_PLACEMENT_CONFIG_DIR=/etc/nova | |
| INSTALL_DATABASE_SERVER_PACKAGES=False | |
| # Enable only IPv4 | |
| IP_VERSION=4 | |
| # Horizon is enabled by default, but | |
| # its not required | |
| disable_service horizon | |
| # Pre-requisite | |
| ENABLED_SERVICES=mysql,key | |
| # Neutron | |
| # Disable Neutron agents not used with OVN (q-agt,q-l3,q-dhcp,q-meta) | |
| ENABLED_SERVICES+=,q-svc,q-ovn-metadata-agent | |
| # Nova | |
| ENABLED_SERVICES+=,n-sproxy,n-api,n-cond | |
| # Placement | |
| ENABLED_SERVICES+=,placement,placement-api,placement-client | |
| # OVN | |
| ENABLED_SERVICES+=,ovn-controller,ovn-northd,ovs-vswitchd,ovsdb-server | |
| # Glance | |
| ENABLED_SERVICES+=,g-api | |
| # OVN options | |
| OVN_L3_CREATE_PUBLIC_NETWORK=false | |
| [[post-config|/etc/nova/nova.conf]] | |
| [placement] | |
| auth_type = password | |
| auth_url = http://127.0.0.1/identity | |
| password = devstack | |
| project_domain_name = Default | |
| project_name = service | |
| user_domain_name = Default | |
| username = nova | |
| [compute] | |
| live_migration_wait_for_vif_plug=True | |
| [libvirt] | |
| live_migration_uri = qemu+ssh://root@%s/system | |
| cpu_mode = host-passthrough | |
| virt_type = qemu | |
| EOF | |
| cat local.conf | |
| sudo ./tools/create-stack-user.sh | |
| sudo mv $DIR/devstack /opt/stack/devstack | |
| sudo chown -R stack: /opt/stack/devstack | |
| - name: Start devstack | |
| run: | | |
| # Hacks to make Devstack install fine. See also: | |
| # https://github.com/openstack/devstack/blob/83821a11ac1d6738b63cb10878b8aaa02e153374/tools/fixup_stuff.sh#L88-L96 | |
| # sudo rm -rf /usr/lib/python3/dist-packages/httplib2-*.egg-info | |
| sudo rm -rf /usr/lib/python3/dist-packages/pyasn1_modules-*.egg-info | |
| sudo rm -rf /usr/lib/python3/dist-packages/PyYAML-*.egg-info | |
| sudo rm -rf /usr/lib/python3/dist-packages/simplejson-*.egg-info | |
| sudo apt install --reinstall python3-setuptools | |
| sudo -iu stack bash -c 'cd /opt/stack/devstack; ./stack.sh' | |
| - name: Get debug data | |
| if: failure() | |
| run: | | |
| echo ">>> Journal logs" | |
| sudo journalctl --no-pager | |
| - name: See "systemctl status placement-api.service" for details. | |
| if: failure() | |
| run: | | |
| sudo systemctl status placement-api.service | |
| - name: See "journalctl -xe" for details. | |
| if: failure() | |
| run: | | |
| sudo journalctl -xe | |
| - name: Run sanity checks | |
| run: | | |
| echo "- Get network and routers list" | |
| openstack --os-cloud devstack-admin --os-region RegionOne network list | |
| openstack --os-cloud devstack-admin --os-region RegionOne router list | |
| - name: Check keystone | |
| run: | | |
| echo "- Get endpoints" | |
| openstack --os-cloud devstack-admin --os-region RegionOne endpoint list | |
| - name: Connect functional tests to devstack | |
| run: | | |
| cp /etc/openstack/clouds.yaml tests/clouds.yaml && \ | |
| ./toolbox/run ./scripts/auth-from-clouds.sh \ | |
| --config tests/clouds.yaml \ | |
| --src devstack \ | |
| --dst devstack-alt-member \ | |
| > tests/auth_tenant.yml && \ | |
| ./toolbox/run ./scripts/auth-from-clouds.sh \ | |
| --config tests/clouds.yaml \ | |
| --src devstack-admin \ | |
| --dst devstack-admin \ | |
| > tests/auth_admin.yml && \ | |
| rm tests/clouds.yaml | |
| - name: Run functional tests | |
| run: | | |
| export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core' | |
| export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements.txt' | |
| ./toolbox/run make test-func |