Skip to content

consistency-functional #2779

consistency-functional

consistency-functional #2779

name: consistency-functional
on:
push:
paths-ignore:
- 'docs/**'
branches:
- main
pull_request:
# Run the functional tests every 8 hours.
# This will help to identify faster if
# there is a CI failure related to a
# problem deploying DevStack.
schedule:
- cron: '0 */8 * * *'
jobs:
# TODO: Add a previous job that:
# * Builds the container image
# * Store the container image to be reused in all jobs
# * Import the container image on each job
build:
if: github.repository_owner == 'os-migrate'
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version: [3.7]
ansible-version: ['latest']
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo locale-gen en_US.UTF-8
sudo dpkg-reconfigure locales
sudo python -m pip install --upgrade pip
sudo pip install tox
sudo apt install software-properties-common build-essential findutils -y
- name: Print podman version
run: |
podman -v
- name: Configure podman
run: |
mkdir -p ~/.config/containers
echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf
echo '[storage]' >> ~/.config/containers/storage.conf
echo 'driver = "overlay"' >> ~/.config/containers/storage.conf
echo '[storage.options]' >> ~/.config/containers/storage.conf
echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf
- name: Log into GitHub Container Registry
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | \
podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
- name: Get the changed files list
id: file_changes
if: github.event_name != 'schedule'
uses: trilom/[email protected]
- name: Build or reuse the toolbox container image
if: github.event_name != 'schedule'
run: |
# The following array defines the expressions that
# when matching the container image for the toolbox
# will be created from scratch, otherwise we reuse
# it from the latest build.
check_list=('toolbox' 'Makefile' 'tests' 'scripts')
REUSE=1
filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') )
for file in "${filez[@]}"; do
for index in ${!check_list[*]}; do
if [[ "${file}" == *"${check_list[$index]}"* ]]; then
echo "We need to re-create the toolbox container image"
echo "${check_list[$index]} is in ${file}"
REUSE=0
fi
done
done
REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build
- name: Reuse the toolbox container image for the scheduled job
if: github.event_name == 'schedule'
run: |
# In this case we assume that the scheduled job will
# pull the container image from the local container
# registry because that should be the latest correct
# toolbox image.
REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build
- name: Run all the tests non OpenStack dependant
run: |
export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core'
export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements.txt'
./toolbox/run make test-fast
- name: Validate changelog
run: |
set -x
git log --oneline | head -n50
make lint-commit-messages
functional:
needs: build
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version: [3.7]
ansible-version: ['latest']
services:
rabbitmq:
image: rabbitmq:latest
ports:
- 5672:5672
options: >-
--health-cmd "rabbitmqctl node_health_check"
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
MYSQL_ALLOW_EMPTY_PASSWORD: true
ports:
- 3306:3306
options: >-
--health-cmd="mysqladmin ping"
--health-interval=10s
--health-timeout=5s
--health-retries=3
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo locale-gen en_US.UTF-8
sudo dpkg-reconfigure locales
sudo apt update
sudo apt remove --purge mysql-server mysql-client mysql-common -y
sudo apt autoremove -y
# MySQL requirements
sudo apt install -y mysql-client mysql-common
# Httplib2 requirements
# We need to install this library un purpose, if not the devstack
# installer will crash as pip is not able to determine if its
# installed or not
sudo apt install python3-httplib2 -y
sudo apt remove postgresql-client-common
sudo apt install postgresql-client-common=238
sudo apt install postgresql-common
sudo python -m pip install --upgrade pip
sudo python -m pip install --upgrade virtualenv
# https://github.com/pypa/setuptools/issues/3772
# sudo python -m pip install --upgrade 'setuptools<66'
sudo python -m pip install --upgrade build
# Installing wheel separately because it may be needed to build some
# of the packages during dependency backtracking and pip >= 22.0 will
# abort backtracking on build failures:
# https://github.com/pypa/pip/issues/10655
sudo pip install --disable-pip-version-check --user wheel
sudo pip install --disable-pip-version-check --user \
ovs sphinx setuptools pyelftools pyOpenSSL ncclient logutils
- name: Configure podman
run: |
mkdir -p ~/.config/containers
echo 'cgroup_manager = "cgroupfs"' >> ~/.config/containers/libpod.conf
echo '[storage]' >> ~/.config/containers/storage.conf
echo 'driver = "overlay"' >> ~/.config/containers/storage.conf
echo '[storage.options]' >> ~/.config/containers/storage.conf
echo 'mount_program = "/usr/bin/fuse-overlayfs"' >> ~/.config/containers/storage.conf
- name: Verify MySQL connection from container
run: >-
mysql \
--host 127.0.0.1 \
-P 3306 \
-uroot \
-proot \
-e "SHOW DATABASES"
mysql \
--host 127.0.0.1 \
-P 3306 \
-uroot \
-proot \
-e 'USE mysql;
DELETE FROM `user` WHERE `Host` != "%" AND `User`="root";
FLUSH PRIVILEGES;'
# Devstack is not able to configure an external MySQL backend.
# There is a mandatory DB server restart from the stack.sh script
# that restart a service called mysql, even if the service is not
# supposed to be configured locally.
# In this case we configure the database to point to the external
# database container and we mock a dummy service called "mysql"
# to trick stack.sh and not fail.
- name: Create a dummy MySQL service to trick stack.sh
run: |
sudo tee /usr/mock.sh > /dev/null <<'EOF'
sleep 3600;
EOF
sudo chmod u+x /usr/mock.sh
sudo tee /lib/systemd/system/mysql.service > /dev/null <<'EOF'
[Unit]
Description=MySQL mock service
[Service]
Type=simple
ExecStart=/bin/bash /usr/mock.sh
[Install]
WantedBy=multi-user.target
EOF
sudo systemctl enable mysql.service
sudo systemctl start mysql.service
sudo systemctl restart mysql.service
sudo systemctl status mysql.service
# there is a script inside devstack that checks to ensure rabbit is
# enabled. because we're using rabbit in a container and devstack cannot
# manage it, we mock a service so it won't fail.
- name: Create a dummy rabbit service to trick devstack/lib/rpc_backend
run: |
sudo tee /usr/mock.sh > /dev/null <<'EOF'
sleep 3600;
EOF
sudo chmod u+x /usr/mock.sh
sudo tee /lib/systemd/system/rabbit.service > /dev/null <<'EOF'
[Unit]
Description=rabbit mock service
[Service]
Type=simple
ExecStart=/bin/bash /usr/mock.sh
[Install]
WantedBy=multi-user.target
EOF
sudo systemctl enable rabbit.service
sudo systemctl start rabbit.service
sudo systemctl restart rabbit.service
sudo systemctl status rabbit.service
- name: Log into GitHub Container Registry
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | \
podman login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
- name: Get the changed files list
id: file_changes
if: github.event_name != 'schedule'
uses: trilom/[email protected]
- name: Build or reuse the toolbox container image
if: github.event_name != 'schedule'
run: |
# The following array defines the expressions that
# when matching the container image for the toolbox
# will be created from scratch, otherwise we reuse
# it from the latest build.
check_list=('toolbox' 'Makefile' 'tests' 'scripts')
REUSE=1
filez=( $(echo '${{ steps.file_changes.outputs.files }}' | jq -c '.[]') )
for file in "${filez[@]}"; do
for index in ${!check_list[*]}; do
if [[ "${file}" == *"${check_list[$index]}"* ]]; then
echo "We need to re-create the toolbox container image"
echo "${check_list[$index]} is in ${file}"
REUSE=0
fi
done
done
REUSE_TOOLBOX=$REUSE NO_VAGRANT=1 make toolbox-build
- name: Reuse the toolbox container image for the scheduled job
if: github.event_name == 'schedule'
run: |
# In this case we assume that the scheduled job will
# pull the container image from the local container
# registry because that should be the latest correct
# toolbox image.
REUSE_TOOLBOX=1 NO_VAGRANT=1 make toolbox-build
# We're hitting filesystem ownership issue on some GH Actions workers:
# https://ubuntuforums.org/showthread.php?t=2406453
- name: Make sure filesystem ownership is correct
run: |
set -x
sudo chown root:root /
sudo chown root:root /bin
- name: Clone devstack
run: |
git clone --single-branch --branch master https://opendev.org/openstack/devstack
- name: Configure devstack
run: |
DIR=$(pwd)
cd devstack
cat << EOF > local.conf
[[local|localrc]]
USE_PYTHON3=True
SERVICE_PASSWORD=devstack
ADMIN_PASSWORD=devstack
SERVICE_TOKEN=devstack
DATABASE_PASSWORD=root
DATABASE_TYPE=mysql
HOST_IP=127.0.0.1
SERVICE_HOST=127.0.0.1
DATABASE_HOST=127.0.0.1
RABBIT_HOST=127.0.0.1
RABBIT_USERID=guest
RABBIT_PASSWORD=guest
VIRT_DRIVER=fake
API_WORKERS=1
NUMBER_FAKE_NOVA_COMPUTE=2
CELLSV2_SETUP="singleconductor"
OS_PLACEMENT_CONFIG_DIR=/etc/nova
INSTALL_DATABASE_SERVER_PACKAGES=False
# Enable only IPv4
IP_VERSION=4
# Horizon is enabled by default, but
# its not required
disable_service horizon
# Pre-requisite
ENABLED_SERVICES=mysql,key
# Neutron
# Disable Neutron agents not used with OVN (q-agt,q-l3,q-dhcp,q-meta)
ENABLED_SERVICES+=,q-svc,q-ovn-metadata-agent
# Nova
ENABLED_SERVICES+=,n-sproxy,n-api,n-cond
# Placement
ENABLED_SERVICES+=,placement,placement-api,placement-client
# OVN
ENABLED_SERVICES+=,ovn-controller,ovn-northd,ovs-vswitchd,ovsdb-server
# Glance
ENABLED_SERVICES+=,g-api
# OVN options
OVN_L3_CREATE_PUBLIC_NETWORK=false
[[post-config|/etc/nova/nova.conf]]
[placement]
auth_type = password
auth_url = http://127.0.0.1/identity
password = devstack
project_domain_name = Default
project_name = service
user_domain_name = Default
username = nova
[compute]
live_migration_wait_for_vif_plug=True
[libvirt]
live_migration_uri = qemu+ssh://root@%s/system
cpu_mode = host-passthrough
virt_type = qemu
EOF
cat local.conf
sudo ./tools/create-stack-user.sh
sudo mv $DIR/devstack /opt/stack/devstack
sudo chown -R stack: /opt/stack/devstack
- name: Start devstack
run: |
# Hacks to make Devstack install fine. See also:
# https://github.com/openstack/devstack/blob/83821a11ac1d6738b63cb10878b8aaa02e153374/tools/fixup_stuff.sh#L88-L96
# sudo rm -rf /usr/lib/python3/dist-packages/httplib2-*.egg-info
sudo rm -rf /usr/lib/python3/dist-packages/pyasn1_modules-*.egg-info
sudo rm -rf /usr/lib/python3/dist-packages/PyYAML-*.egg-info
sudo rm -rf /usr/lib/python3/dist-packages/simplejson-*.egg-info
sudo apt install --reinstall python3-setuptools
sudo -iu stack bash -c 'cd /opt/stack/devstack; ./stack.sh'
- name: Get debug data
if: failure()
run: |
echo ">>> Journal logs"
sudo journalctl --no-pager
- name: See "systemctl status placement-api.service" for details.
if: failure()
run: |
sudo systemctl status placement-api.service
- name: See "journalctl -xe" for details.
if: failure()
run: |
sudo journalctl -xe
- name: Run sanity checks
run: |
echo "- Get network and routers list"
openstack --os-cloud devstack-admin --os-region RegionOne network list
openstack --os-cloud devstack-admin --os-region RegionOne router list
- name: Check keystone
run: |
echo "- Get endpoints"
openstack --os-cloud devstack-admin --os-region RegionOne endpoint list
- name: Connect functional tests to devstack
run: |
cp /etc/openstack/clouds.yaml tests/clouds.yaml && \
./toolbox/run ./scripts/auth-from-clouds.sh \
--config tests/clouds.yaml \
--src devstack \
--dst devstack-alt-member \
> tests/auth_tenant.yml && \
./toolbox/run ./scripts/auth-from-clouds.sh \
--config tests/clouds.yaml \
--src devstack-admin \
--dst devstack-admin \
> tests/auth_admin.yml && \
rm tests/clouds.yaml
- name: Run functional tests
run: |
export OS_MIGRATE_REQUIREMENTS_UNINSTALL_BEFORE_OVERRIDE='ansible ansible-base ansible-core'
export OS_MIGRATE_REQUIREMENTS_OVERRIDE='toolbox/build/venv-requirements.txt'
./toolbox/run make test-func