Merge branch 'openstack-k8s-operators:main' into multiple_nodesets

.github/workflows/review-ready-label.yml

@@ -3,7 +3,6 @@
 #   - There are no failed CI jobs
 #   - There are no Pending CI jobs excluding Tide
 #   - The PR isn't in draft state
-#   - The commit isn't found in more than one PR
 # Removes the label if any of those checks fail.
 
 name: Toggle review ready label
@@ -59,7 +58,7 @@ jobs:
           pr_search_result_length=$(echo "${pr_search_result}" | jq length)
           if [ "$pr_search_result_length" -ne 1 ]; then
               echo "Commit not found or in multiple PRs"
-              exit 1
+              exit 0
           fi
 
           # Define draft_status and set $ready false if PR is in draft state

.gitignore

@@ -6,3 +6,4 @@ docs/dictionary/tmp
 .vscode
 .venv/*
 .env
+.idea/

Makefile

@@ -53,11 +53,11 @@ new_role: create_new_role role_molecule ## Create a new Ansible role and related
 ##@ Setup steps
 .PHONY: setup_tests
 setup_tests: ## Setup the environment
-	bash scripts/setup_env
+	bash scripts/setup_env 2>&1 | tee $(LOG_DIR)/setup_env.log
 
 .PHONY: setup_molecule
 setup_molecule: setup_tests ## Setup molecule environment
-	bash scripts/setup_molecule
+	bash scripts/setup_molecule 2>&1 | tee $(LOG_DIR)/setup_molecule.log
 
 ##@ General testing
 .PHONY: tests

ci/config/molecule.yaml

@@ -4,6 +4,7 @@
     files:
       - ^roles/dnsmasq/(defaults|files|handlers|library|lookup_plugins|module_utils|tasks|templates|vars).*
       - ^roles/networking_mapper/(defaults|files|handlers|library|lookup_plugins|module_utils|tasks|templates|vars).*
+      - ^roles/config_drive/(defaults|files|handlers|library|lookup_plugins|module_utils|molecule|tasks|templates|vars).*
     timeout: 3600
 - job:
     name: cifmw-molecule-openshift_login

ci/playbooks/pre-ci-bootstrap.yml → ci/playbooks/bootstrap-networking-mapper.yml

@@ -31,6 +31,7 @@
           ~/test-python/bin/ansible-playbook {{ ansible_user_dir }}/networking_mapper.yml
           -i {{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml
           -e @scenarios/centos-9/base.yml
+          -e cifmw_networking_mapper_ifaces_info_path=/etc/ci/env/interfaces-info.yml
           -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           {% if nodepool is defined %}
           -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/nodepool_params.yml"

ci/playbooks/molecule-test.yml

@@ -1,8 +1,16 @@
 ---
 - name: "Run ci/playbooks/molecule-test.yml"
   hosts: "{{ cifmw_zuul_target_host | default('controller') }}"
-  gather_facts: true
+  gather_facts: false
   tasks:
+    - name: Gather required facts
+      ansible.builtin.setup:
+        gather_subset:
+          - "!all"
+          - "!min"
+          - "user_dir"
+          - "env"
+
     - name: Load environment var if instructed to
       when:
         - cifmw_reproducer_molecule_env_file is defined
@@ -12,6 +20,7 @@
     - name: Run molecule
       environment:
         ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/zuul-output/logs/ansible-execution.log"
+        ANSIBLE_CALLBACKS_ENABLED: "ansible.posix.profile_tasks,yaml"
         MOLECULE_REPORT: "/tmp/report.html"
         LC_ALL: en_US.utf8
         LANG: en_US.utf8

ci/playbooks/pre-doc.yml

@@ -43,7 +43,7 @@
 
         - name: Download and extract golang
           ansible.builtin.unarchive:
-            src: "https://golang.org/dl/go{{ go_version | default('1.20.14') }}.linux-amd64.tar.gz"
+            src: "https://golang.org/dl/go{{ go_version | default('1.21.13') }}.linux-amd64.tar.gz"
             dest: "/usr/local"
             remote_src: true
             extra_opts:

create-infra.yml

@@ -0,0 +1,77 @@
+---
+# Purpose of this playbook:
+# - Create a virtual infrastructure consisting in
+#   virtual machines, related storage, and virtual networks.
+#
+# - Create a dnsmasq instance with the appropriate records to provide
+#   fixed IPs, proper DNS resolution
+#
+# - Expose at least one inventory file showing the various resources
+#
+# This playbook is meant to run for architecture driven deployment only.
+
+- name: Parent scenario if needed
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Inherit from parent scenarios if needed
+      ansible.builtin.include_tasks:
+        file: "ci/playbooks/tasks/inherit_parent_scenario.yml"
+
+- name: Manage/generate unique ID
+  ansible.builtin.import_playbook: playbooks/unique-id.yml
+
+- name: Run consistency checks
+  ansible.builtin.import_playbook: "playbooks/adoption/pre.yml"
+
+- name: Prepare infrastructure datasets
+  ansible.builtin.import_playbook: "playbooks/adoption/infra.yml"
+
+- name: Amend infrastructure datasets
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    # Inject "start: false" in the layout to not start any VM yet.
+    # Starting the VM will be done later, either by the tool deploying
+    # OSP, or the one deploy RHOSO.
+    # VM initial configuration, when managed, is done using cloud-init.
+    - name: Ensure no VM is started when we create them during this run
+      vars:
+        _no_start: >-
+          {% set _vms = {}                                          -%}
+          {% for _type in _cifmw_libvirt_manager_layout.vms.keys()  -%}
+          {%   set _ = _vms.update({_type: {'start': false}})       -%}
+          {% endfor                                                 -%}
+          {{ _vms }}
+      ansible.builtin.set_fact:
+        _cifmw_libvirt_manager_layout: >-
+          {{
+            _cifmw_libvirt_manager_layout |
+            combine({'vms': _no_start}, recursive=true)
+          }}
+    - name: Set amount for OCP cluster members to 0
+      vars:
+        _0_ocp:
+          vms:
+            ocp:
+              amount: 0
+            ocp_worker:
+              amount: 0
+      ansible.builtin.set_fact:
+        _cifmw_libvirt_manager_layout: >-
+          {{
+            _cifmw_libvirt_manager_layout | combine(_0_ocp, recursive=true)
+          }}
+
+- name: Prepare networking
+  ansible.builtin.import_playbook: "playbooks/adoption/network.yml"
+
+- name: Deploy infrastructure
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    # Create all the VMs
+    - name: Deploy layout
+      ansible.builtin.import_role:
+        name: "libvirt_manager"
+        tasks_from: "deploy_layout.yml"

deploy-ocp.yml

@@ -0,0 +1,87 @@
+---
+# This playbook deploy OCP, both master and workers.
+# The main goal of this playbook is to be called as
+# a standalone piece of a bigger job, mostly in the
+# adoption context.
+#
+# You would typically run this playbook after the
+# "create-infra.yml" one, passing the exact same parameters.
+#
+# Make sure you passed the needed secrets to deploy OCP!
+
+- name: Parent scenario if needed
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Inherit from parent scenarios if needed
+      ansible.builtin.include_tasks:
+        file: "ci/playbooks/tasks/inherit_parent_scenario.yml"
+
+- name: Manage/generate unique ID
+  ansible.builtin.import_playbook: playbooks/unique-id.yml
+
+- name: Run consistency checks
+  ansible.builtin.import_playbook: "playbooks/adoption/pre.yml"
+
+- name: Prepare infrastructure datasets
+  ansible.builtin.import_playbook: "playbooks/adoption/infra.yml"
+
+- name: Amend infrastructure datasets
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    # Inject "start: false" in the layout to not start any VM yet.
+    # Starting the VM will be done later, either by the tool deploying
+    # OSP, or the one deploy RHOSO.
+    # VM initial configuration, when managed, is done using cloud-init.
+    - name: Ensure no VM is started when we create them during this run
+      vars:
+        _no_start: >-
+          {% set _vms = {}                                          -%}
+          {% for _type in _cifmw_libvirt_manager_layout.vms.keys()  -%}
+          {%   if _type is not match('^ocp.*')                      -%}
+          {%     set _ = _vms.update({_type: {'start': false}})     -%}
+          {%   endif                                                -%}
+          {% endfor                                                 -%}
+          {{ _vms }}
+      ansible.builtin.set_fact:
+        _cifmw_libvirt_manager_layout: >-
+          {{
+            _cifmw_libvirt_manager_layout |
+            combine({'vms': _no_start}, recursive=true)
+          }}
+
+- name: Prepare networking
+  ansible.builtin.import_playbook: "playbooks/adoption/network.yml"
+
+- name: OCP cluster
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Deploy OCP cluster
+      ansible.builtin.import_role:
+        name: "reproducer"
+        tasks_from: "ocp_layout.yml"
+
+    - name: Start OCP cluster based on its overlays
+      ansible.builtin.import_role:
+        name: "libvirt_manager"
+        tasks_from: "deploy_layout.yml"
+
+    # Run from the hypervisor
+    - name: Ensure OCP cluster is stable
+      vars:
+        _auth_path: >-
+          {{
+            (
+            cifmw_devscripts_repo_dir,
+            'ocp',
+            cifmw_devscripts_config.cluster_name,
+            'auth'
+            ) | ansible.builtin.path_join
+          }}
+        cifmw_openshift_adm_op: "stable"
+        cifmw_openshift_kubeconfig: >-
+          {{ (_auth_path, 'kubeconfig') | ansible.builtin.path_join }}
+      ansible.builtin.include_role:
+        name: openshift_adm

docs/dictionary/en-custom.txt

@@ -496,6 +496,7 @@ tempestconf
 testcases
 testenv
 timestamper
+timesync
 tldca
 tls
 tmp

docs/source/usage/01_usage.md

@@ -66,11 +66,13 @@ are shared among multiple roles:
 - `cifmw_nolog`: (Bool) Toggle `no_log` value for selected tasks. Defaults to `true` (hiding those logs by default).
 - `cifmw_parent_scenario`: (String or List(String)) path to existing scenario/parameter file to inherit from.
 - `cifmw_configure_switches`: (Bool) Specifies whether switches should be configured. Computes in `reproducer.yml` playbook. Defaults to `false`.
+- `cifmw_use_ocp_overlay`: (Boolean) Specifies whether OCP nodes deployed via devscripts should use overlay images. Using overlay images speeds up the redeployment when using the reproducer role locally but in CI each job is cleaned up and redeployed. Creating the overlay image takes time so should be disabled when not used. Defaults to `true`.
 - `cifmw_crc_default_network`: (String) name of the untagged network used to address DNS on the crc node. Default is `default`.
 - `cifmw_run_operators_compliance_scans`: (Bool) Specifies whether to run operator compliance scans.  Defaults to `false`.
 - `cifmw_run_compute_compliance_scans`: (Bool) Specifies whether to run compliance scans on the first compute.  Defaults to `false`.
 - `cifmw_run_id`: (String) CI Framework run identifier. This is used in libvirt_manager, to add some uniqueness to some types of virtual machines (anything that's not OCP, CRC nor controller).
   If not set, the Framework will generate a random string for you, and store it on the target host, in `{{ cifmw_basedir }}/artifacts/run-id`
+- `cifmw_deploy_architecture_args`: (String) additional args and parameters to pass to the deploy-architecture script. Default is `''`.
 
 ```{admonition} Words of caution
 :class: danger

docs/toolbelt-catalog.yaml

@@ -0,0 +1,41 @@
+# Catalog entry for Backstage [backstage.io]
+
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+    name: ci-framework
+    title: ci-framework
+    description: |
+        CI Framework - used for CI, QE and Devs to run OSP 18+ jobs in a converged way
+    annotations:
+        github.com/project-slug: openstack-k8s-operators/ci-framework
+        feedback/type: JIRA
+        feedback/host: https://issues.redhat.com
+        jira/project-key: OSPRH
+    links:
+      - title: docs
+        url: https://ci-framework.readthedocs.io/en/latest/
+        icon: docs
+      - title: code
+        url: https://github.com/openstack-k8s-operators/ci-framework
+        icon: github
+      - title: "#osp-podified-ci-support"
+        url: https://app.slack.com/client/E030G10V24F/C03MD4LG22Z
+        icon: chat
+    tags:
+        - testing
+        - test-execution
+        - test-framework
+        - test-management
+        - test-reporting
+        - provisioning
+        - python
+        - openstack
+        - openshift
+        - cloud
+        - continuous-integration
+    namespace: quality-community
+spec:
+    type: tool
+    owner: group:redhat/openstack-k8s-operators-ci
+    lifecycle: production

hooks/playbooks/fetch_compute_facts.yml

@@ -26,6 +26,10 @@
     - name: Load parameters
       ansible.builtin.include_vars:
         dir: "{{ item }}"
+        ignore_unknown_extensions: true
+        extensions:
+          - yaml
+          - yml
       loop:
         - "{{ cifmw_basedir }}/artifacts/parameters"
         - "/etc/ci/env"

playbooks/adoption/infra.yml

@@ -0,0 +1,27 @@
+---
+# Purpose: create infra dataset to be consumed by
+# adoption related playbooks
+
+- name: Create CI virtual infrastructure
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  roles:
+    - role: ci_setup
+    - role: discover_latest_image
+
+  tasks:
+    - name: Generate libvirt layout
+      ansible.builtin.include_role:
+        name: "libvirt_manager"
+        tasks_from: "generate_layout.yml"
+
+    - name: Apply layout patches if it exists
+      when:
+        - _adoption_scenario.libvirt_manager_patch_layout is defined
+      ansible.builtin.set_fact:
+        _cifmw_libvirt_manager_layout: >-
+          {{
+            _cifmw_libvirt_manager_layout |
+            combine(_adoption_scenario.libvirt_manager_patch_layout,
+                    recursive=true)
+          }}

playbooks/adoption/network.yml

@@ -0,0 +1,23 @@
+---
+# Purpose: create infra dataset to be consumed by
+# adoption related playbooks
+
+- name: Build networking_mapper related content
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Apply networking_definition patch if it exists
+      when:
+        - _adoption_scenario.networking_mapper_definition_patch is defined
+      ansible.builtin.set_fact:
+        cifmw_networking_mapper_definition_patch_01_adoption_infra: >-
+          {{ _adoption_scenario.networking_mapper_definition_patch }}
+
+    - name: Prepare networking
+      vars:
+        cifmw_reproducer_validate_network: false
+        _use_crc: false
+        _use_ocp: true
+      ansible.builtin.import_role:
+        name: "reproducer"
+        tasks_from: "prepare_networking.yml"