From 61a95bbf1d38da85e867fe04e42be7e00988e8df Mon Sep 17 00:00:00 2001 From: John Fulton Date: Tue, 13 Aug 2024 09:47:41 -0400 Subject: [PATCH] Configure Ceph RGW on a dedicated Swift network When Ceph RGW is used, an endpoint for Swift storage is hosted not in a pod on k8s but on an EDPM node. Thus, a service hosted on an EDPM node will need to be accessed from a separate network. This patch adds the Swift storage network (swift) with VLAN 25 and range 172.22.0.0/24 in the reproducers networking-definition.yml. It also adds a multus range for this network so that the Tempest pod can access this network for testing. The Swift storage network is added to the OCP nodes for the same reason. The swift network is also added to the ci playbook networking-env-definition for the rhoso-architecture-validate-hci github-check. The Ceph playbook cifmw_cephadm_rgw_network parameter may then be set to the new swift network so that RGW is then configured on a separate network. If this network parameter is not set, then the storage network is used. Signed-off-by: John Fulton --- .../files/networking-env-definition.yml | 54 +++++++++++++++++++ playbooks/ceph.yml | 24 +++++++-- roles/cifmw_cephadm/tasks/check_vip.yml | 2 +- .../reproducers/networking-definition.yml | 17 ++++++ 4 files changed, 92 insertions(+), 5 deletions(-) diff --git a/ci/playbooks/files/networking-env-definition.yml b/ci/playbooks/files/networking-env-definition.yml index 201163757a..efaa11dddf 100644 --- a/ci/playbooks/files/networking-env-definition.yml +++ b/ci/playbooks/files/networking-env-definition.yml @@ -151,6 +151,15 @@ instances: parent_interface: eth1 skip_nm: false vlan_id: 22 + swift: + interface_name: eth1.25 + ip_v4: 172.22.0.100 + mac_addr: '52:54:00:0b:1c:e7' + mtu: 1500 + network_name: swift + parent_interface: eth1 + skip_nm: false + vlan_id: 25 compute-1: hostname: compute-1 name: compute-1 @@ -189,6 +198,15 @@ instances: parent_interface: eth1 skip_nm: false vlan_id: 22 + swift: + interface_name: eth1.25 + ip_v4: 172.22.0.101 + mac_addr: '52:54:00:0b:1c:e8' + mtu: 1500 + network_name: swift + parent_interface: eth1 + skip_nm: false + vlan_id: 25 compute-2: hostname: compute-2 name: compute-2 @@ -227,6 +245,15 @@ instances: parent_interface: eth1 skip_nm: false vlan_id: 22 + swift: + interface_name: eth1.25 + ip_v4: 172.22.0.102 + mac_addr: '52:54:00:0b:1c:e9' + mtu: 1500 + network_name: swift + parent_interface: eth1 + skip_nm: false + vlan_id: 25 controller-0: hostname: controller-0 name: controller-0 @@ -336,6 +363,15 @@ instances: parent_interface: enp6s0 skip_nm: false vlan_id: 22 + swift: + interface_name: enp6s0.25 + ip_v4: 172.22.0.10 + mac_addr: '52:54:00:18:a0:b6' + mtu: 1500 + network_name: swift + parent_interface: enp6s0 + skip_nm: false + vlan_id: 25 ocp-master-1: hostname: ocp-master-1 name: ocp-master-1 @@ -374,6 +410,15 @@ instances: parent_interface: enp6s0 skip_nm: false vlan_id: 22 + swift: + interface_name: enp6s0.25 + ip_v4: 172.22.0.11 + mac_addr: '52:54:00:18:a0:b7' + mtu: 1500 + network_name: swift + parent_interface: enp6s0 + skip_nm: false + vlan_id: 25 ocp-master-2: hostname: ocp-master-2 name: ocp-master-2 @@ -412,6 +457,15 @@ instances: parent_interface: enp6s0 skip_nm: false vlan_id: 22 + swift: + interface_name: enp6s0.25 + ip_v4: 172.22.0.12 + mac_addr: '52:54:00:18:a0:b8' + mtu: 1500 + network_name: swift + parent_interface: enp6s0 + skip_nm: false + vlan_id: 25 networks: ctlplane: dns_v4: diff --git a/playbooks/ceph.yml b/playbooks/ceph.yml index 09ed924786..d2c1f19962 100644 --- a/playbooks/ceph.yml +++ b/playbooks/ceph.yml @@ -338,27 +338,43 @@ # public network always exist because is provided by the ceph_spec role - name: Get Storage network range ansible.builtin.set_fact: - cifmw_cephadm_rgw_network: "{{ lookup('ansible.builtin.ini', 'public_network section=global file=' ~ cifmw_cephadm_bootstrap_conf) }}" + cifmw_cephadm_storage_network: "{{ lookup('ansible.builtin.ini', 'public_network section=global file=' ~ cifmw_cephadm_bootstrap_conf) }}" + + - name: Set RGW network range to storage network only if it was not provided + ansible.builtin.set_fact: + cifmw_cephadm_rgw_network: "{{ cifmw_cephadm_storage_network }}" + when: + - cifmw_cephadm_rgw_network is not defined or + cifmw_cephadm_rgw_network | length == 0 - name: Set IP address of first monitor ansible.builtin.set_fact: - cifmw_cephadm_first_mon_ip: "{{ hostvars[this_host][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first }}" + cifmw_cephadm_first_mon_ip: "{{ hostvars[this_host][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_storage_network) | first }}" vars: this_host: "{{ _target_hosts | first }}" - name: Assert if any EDPM nodes n/w interface is missing in storage network + ansible.builtin.assert: + that: + - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_storage_network) | length > 0 + fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_storage_network }}" + loop: "{{ _target_hosts }}" + + - name: Assert if any EDPM nodes n/w interface is missing in RGW network ansible.builtin.assert: that: - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | length > 0 fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_rgw_network }}" loop: "{{ _target_hosts }}" + when: + - cifmw_cephadm_rgw_network != cifmw_cephadm_storage_network - - name: Get already assigned IP addresses + - name: Get already assigned RGW IP addresses ansible.builtin.set_fact: ips: "{{ ips | default([]) + [ hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first ] }}" loop: "{{ _target_hosts }}" - # cifmw_cephadm_vip is the VIP reserved in the Storage network + # cifmw_cephadm_vip is the VIP reserved in the RGW network - name: Set VIP var as empty string ansible.builtin.set_fact: cifmw_cephadm_vip: "" diff --git a/roles/cifmw_cephadm/tasks/check_vip.yml b/roles/cifmw_cephadm/tasks/check_vip.yml index 0714510e7a..d92bcc0765 100644 --- a/roles/cifmw_cephadm/tasks/check_vip.yml +++ b/roles/cifmw_cephadm/tasks/check_vip.yml @@ -22,7 +22,7 @@ ansible.builtin.set_fact: count: "{{ 2 if count is undefined else count | int + 2 }}" - - name: Get an IP address from the Storage network + - name: Get an IP address from the RGW network ansible.builtin.set_fact: cur_ip: "{{ cifmw_cephadm_rgw_network | ansible.utils.next_nth_usable(count) }}" diff --git a/scenarios/reproducers/networking-definition.yml b/scenarios/reproducers/networking-definition.yml index 4f0ba0ed26..671f83263f 100644 --- a/scenarios/reproducers/networking-definition.yml +++ b/scenarios/reproducers/networking-definition.yml @@ -91,6 +91,19 @@ cifmw_networking_definition: end: 250 vlan: 23 mtu: 1500 + swift: + network: "172.22.0.0/24" + tools: + netconfig: + ranges: + - start: 100 + end: 250 + multus: + ranges: + - start: 30 + end: 70 + vlan: 25 + mtu: 1500 group-templates: ocps: @@ -106,6 +119,8 @@ cifmw_networking_definition: trunk-parent: ctlplane storage: trunk-parent: ctlplane + swift: + trunk-parent: ctlplane ocp_workers: network-template: range: @@ -127,6 +142,8 @@ cifmw_networking_definition: trunk-parent: ctlplane storagemgmt: trunk-parent: ctlplane + swift: + trunk-parent: ctlplane cephs: network-template: range: