Fixes for CLI Tests 1) exposing ports for databases 2) added dockerclients setup for dockers only. 3) updates

pmm_psmdb-pbm_setup/docker-compose-rs.yaml

@@ -33,6 +33,7 @@ services:
       - ./conf/datagen:/etc/datagen:ro
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
       - /tmp/backup_data:/tmp/backup_data
+      - /tmp/mongodb:/tmp
     privileged: true
     environment:
       PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017

pmm_psmdb-pbm_setup/docker-compose-sharded.yaml

@@ -21,6 +21,7 @@ services:
       - ./conf/pbm:/etc/pbm
       - ./conf/mongod-rs1:/etc/mongod
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
+      - /tmp/mongodb:/tmp
     privileged: true
     environment:
       - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017"
@@ -30,6 +31,8 @@ services:
       - "PMM_AGENT_SERVER_INSECURE_TLS=1"
     container_name: rs101
     hostname: rs101
+    ports:
+      - "27027:27017"
     networks:
       - test-network
 

pmm_qa/client_container_ms_setup.sh

@@ -43,6 +43,7 @@ mkdir /tmp || true
 chmod 1777 /tmp || true
 
 ## Deploy DB deployer
+export MS_PORT=3308
 export tar_ball_name=$(ls mysql-*)
 dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ms${ms_version} --overwrite
 export db_version_sandbox=$(ls ~/ms${ms_version})
@@ -55,7 +56,7 @@ if [[ "$number_of_nodes" == 1 ]];then
       node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
       mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"
    else
-      dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force
+      dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --port=$MS_PORT --remote-access=% --bind-address=0.0.0.0 --force
       export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
       node_port=`dbdeployer sandboxes --header | grep  ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
       mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"

pmm_qa/client_container_ps_setup.sh

@@ -49,7 +49,6 @@ chmod 1777 /tmp || true
 export tar_ball_name=$(ls Percona-Server*)
 dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ps${ps_version} --flavor=percona  
 export db_version_sandbox=$(ls ~/ps${ps_version})
-dbdeployer deploy single ${db_version_sandbox} --port=${PS_PORT} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0
 
 export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
 export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999))
@@ -61,7 +60,7 @@ if [[ "$number_of_nodes" == 1 ]];then
       node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
       mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"
    else
-      dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force
+      dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force
       export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}')
       node_port=`dbdeployer sandboxes --header | grep  ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'`
       mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';"

pmm_qa/docker-compose-clients.yaml

@@ -0,0 +1,64 @@
+version: '3.7'
+
+services:
+  pmm-server:
+    container_name: pmm-server-1
+    image: ${DOCKER_VERSION:-perconalab/pmm-server:3-dev-latest}
+    ports:
+      - "2553:8443"
+      - "8085:8080"
+    environment:
+      - PMM_DEBUG=1
+    networks:
+      - docker-client-check
+
+  pmm-client:
+    container_name: pmm-client-1
+    image: ${CLIENT_DOCKER_VERSION:-perconalab/pmm-client:3-dev-latest}
+    environment:
+      - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443
+      - PMM_AGENT_SERVER_USERNAME=admin
+      - PMM_AGENT_SERVER_PASSWORD=admin
+      - PMM_AGENT_SERVER_INSECURE_TLS=1
+      - PMM_AGENT_SETUP=1
+      - PMM_AGENT_SETUP_FORCE=1
+      - PMM_AGENT_CONFIG_FILE=/usr/local/percona/pmm/config/pmm-agent.yaml
+    depends_on:
+      pmm-server:
+        condition: service_healthy
+    networks:
+      - docker-client-check
+
+  ps:
+    container_name: ps-1
+    image: percona:8.0
+    environment:
+      - MYSQL_ROOT_PASSWORD=root
+      - MYSQL_USER=pmm
+      - MYSQL_PASSWORD=pmm-pass
+      - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443
+    networks:
+      - docker-client-check
+
+  mongodb:
+    container_name: psmdb-1
+    image: percona/percona-server-mongodb:7.0
+    environment:
+      - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443
+      - MONGO_INITDB_ROOT_USERNAME=pmm
+      - MONGO_INITDB_ROOT_PASSWORD=pmm-pass
+    networks:
+      - docker-client-check
+
+  postgres:
+    container_name: pdpgsql-1
+    image: percona/percona-distribution-postgresql:16
+    environment:
+      - POSTGRES_USER=pmm
+      - POSTGRES_PASSWORD=pmm-pass
+      - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443
+    networks:
+      - docker-client-check
+
+networks:
+  docker-client-check:

pmm_qa/ms_pmm_setup.yml

@@ -32,6 +32,8 @@
   - name: Prepare Container for MySQL
     shell: >
       docker run -d --name={{ ms_container }}
+      -p 3318:3308
+      -v /tmp:/tmp
       phusion/baseimage:focal-1.2.0
 
   - name: Copy all required Artifacts to the docker ms_container

pmm_qa/pmm-framework.py

@@ -3,6 +3,8 @@
 import os
 import sys
 import ansible_runner
+import requests
+import re
 
 # Database configurations
 database_configs = {
@@ -30,7 +32,7 @@
         "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""}
     },
     "PXC": {
-        "versions": ["7", "8"],
+        "versions": ["5.7", "8.0"],
         "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""}
     },
     "PROXYSQL": {
@@ -50,6 +52,8 @@
         },
         "configurations": {"CLIENT_VERSION": "3-dev-latest"}
     },
+    "DOCKERCLIENTS": {
+    },
 }
 
 
@@ -86,13 +90,20 @@ def get_running_container_name():
         # Check each line for the docker image name
         for line in containers:
             # Extract the image name
-            image_info = line.split('\t')[0]
-            info_parts = image_info.split()[2:]
+            info_parts = line.split('\t')[0]
+            image_info = info_parts.split()[2:]
             # Check if the container is in the list of running containers
             # and establish N/W connection with it.
-            if container_name in info_parts:
-                subprocess.run(['docker', 'network', 'create', 'pmm-qa'])
-                subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name])
+            if container_name in image_info:
+                # Check if pmm-qa n/w exists and already connected to pmm-server n/w.
+                result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True, text=True)
+                if result.returncode != 0:
+                    subprocess.run(['docker', 'network', 'create', 'pmm-qa'])
+                    subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name])
+                else:
+                    networks = result.stdout
+                    if container_name not in networks:
+                        subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name])
                 return container_name
 
     except subprocess.CalledProcessError:
@@ -314,11 +325,11 @@ def setup_external(db_type, db_version=None, db_config=None, args=None):
     run_ansible_playbook(playbook_filename, env_vars, args)
 
 
-def execute_shell_scripts(shell_scripts, env_vars, args):
+def execute_shell_scripts(shell_scripts, scripts_path, env_vars, args):
     # Get script directory
     script_path = os.path.abspath(sys.argv[0])
     script_dir = os.path.dirname(script_path)
-    shell_scripts_path = script_dir + "/../pmm_psmdb-pbm_setup/"
+    shell_scripts_path = script_dir + scripts_path
 
     # Get the original working directory
     original_dir = os.getcwd()
@@ -378,7 +389,9 @@ def mongo_sharding_setup(script_filename, args):
     compose_file_path = scripts_path + compose_filename
 
     # Create pmm-qa n/w used in workaround
-    subprocess.run(['docker', 'network', 'create', 'pmm-qa'])
+    result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True)
+    if not result:
+        subprocess.run(['docker', 'network', 'create', 'pmm-qa'])
 
     no_server = True
     # Add workaround (copy files) till sharding only support is ready.
@@ -407,6 +420,33 @@ def mongo_sharding_setup(script_filename, args):
         print(f"Error occurred: {e}")
 
 
+def get_latest_psmdb_version(psmdb_version):
+    if psmdb_version == "latest":
+        return psmdb_version
+    # Define the data to be sent in the POST request
+    data = {
+        'version': f'percona-server-mongodb-{psmdb_version}'
+    }
+
+    # Make the POST request
+    response = requests.post('https://www.percona.com/products-api.php', data=data)
+
+    # Extract the version number using regular expression
+    version_number = re.findall(r'value="([^"]*)"', response.text)
+
+    if version_number:
+        # Sort the version numbers and extract the latest one
+        latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1]
+
+        # Extract the full version number
+        major_version = latest_version.split('-')[3].strip()  # Trim spaces
+        minor_version = latest_version.split('-')[4].strip()  # Trim spaces
+
+        return f'{major_version}-{minor_version}'
+    else:
+        return None
+
+
 def setup_psmdb(db_type, db_version=None, db_config=None, args=None):
     # Check if PMM server is running
     container_name = get_running_container_name()
@@ -415,7 +455,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None):
         exit(1)
 
     # Gather Version details
-    psmdb_version = os.getenv('PSMDB_VERSION') or db_version or database_configs[db_type]["versions"][-1]
+    psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or database_configs[db_type]["versions"][-1]
 
     # Handle port address for external or internal address
     server_hostname = container_name
@@ -441,16 +481,18 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None):
     }
 
     shell_scripts = []
-    if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss" or "psa":
-        # Shell script names
+    scripts_path = "/../pmm_psmdb-pbm_setup/"
+    setup_type = get_value('SETUP_TYPE', db_type, args, db_config).lower()
+
+    if setup_type in ("pss", "psa"):
         shell_scripts = ['start-rs-only.sh']
-    elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards":
-        shell_scripts = [f'start-sharded-no-server.sh']
+    elif setup_type in ("shards", "sharding"):
+        shell_scripts = ['start-sharded-no-server.sh']
         mongo_sharding_setup(shell_scripts[0], args)
 
     # Execute shell scripts
     if not shell_scripts == []:
-        execute_shell_scripts(shell_scripts, env_vars, args)
+        execute_shell_scripts(shell_scripts, scripts_path, env_vars, args)
 
 
 def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None):
@@ -462,7 +504,7 @@ def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None):
 
     # Gather Version details
     pxc_version = os.getenv('PXC_VERSION') or db_version or database_configs[db_type]["versions"][-1]
-    proxysql_version = os.getenv('PROXYSQL_VERSION') or db_version or database_configs["PROXYSQL"]["versions"][-1]
+    proxysql_version = os.getenv('PROXYSQL_VERSION') or database_configs["PROXYSQL"]["versions"][-1]
 
     # Define environment variables for playbook
     env_vars = {
@@ -486,6 +528,18 @@ def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None):
     run_ansible_playbook(playbook_filename, env_vars, args)
 
 
+def setup_dockerclients(db_type, db_version=None, db_config=None, args=None):
+    # Define environment variables for shell script
+    env_vars = {}
+
+    # Shell script filename
+    shell_scripts = ['setup_docker_client_images.sh']
+    shell_scripts_path = ''
+
+    # Call the function to run the Ansible playbook
+    execute_shell_scripts(shell_scripts, shell_scripts_path, env_vars, args)
+
+
 # Set up databases based on arguments received
 def setup_database(db_type, db_version=None, db_config=None, args=None):
     if args.verbose:
@@ -515,6 +569,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None):
         setup_haproxy(db_type, db_version, db_config, args)
     elif db_type == 'EXTERNAL':
         setup_external(db_type, db_version, db_config, args)
+    elif db_type == 'DOCKERCLIENTS':
+        setup_dockerclients(db_type, db_version, db_config, args)
     else:
         print(f"Database type {db_type} is not recognised, Exiting...")
         exit(1)

pmm_qa/ps_pmm_setup.yml

@@ -32,6 +32,7 @@
   - name: Prepare Container for ps and Proxysql
     shell: >
       docker run -d --name={{ ps_container }}
+      -p 3317:3307
       phusion/baseimage:jammy-1.0.1
 
   - name: Copy all required Artifacts to the docker ps_container

pmm_qa/pxc_proxysql_setup.yml

@@ -5,7 +5,7 @@
   become: true
   become_method: sudo
   vars:
-    pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8', true) ) }}"
+    pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8.0', true) ) }}"
     pxc_tarball: "{{ lookup('vars', 'extra_pxc_tarball', default=lookup('env','PXC_TARBALL') | default('', true) ) }}"
     proxysql_version: "{{ lookup('vars', 'extra_proxysql_version', default=lookup('env','PROXYSQL_VERSION') | default('2', true) ) }}"
     proxysql_package: "{{ lookup('vars', 'extra_proxysql_package', default=lookup('env','PROXYSQL_PACKAGE') | default('', true) ) }}"
@@ -33,6 +33,7 @@
   - name: Prepare Container for PXC and Proxysql
     shell: >
       docker run -d --name={{ pxc_container }}
+      -p 6033:6033
       phusion/baseimage:jammy-1.0.1
 
   - name: Copy all required Artifacts to the docker pxc_container
@@ -97,6 +98,10 @@
       - docker exec {{ pxc_container }} chmod 1777 /tmp
       - docker exec --user pxc {{ pxc_container }} bash -xe ./client_container_pxc_setup.sh --pxc_version {{ pxc_version }} --pxc_tarball {{ pxc_tarball }} --number_of_nodes {{ number_of_nodes }} --pxc_dev_cluster {{ pxc_dev_cluster }} --query_source {{query_source}} > setup_pxc_{{ pxc_version }}.log
 
+  - name: Set Random Number Fact
+    set_fact:
+      random_number: "{{ (10000 | random) | int }}"
+
   - name: Start ProxySQL inside the PXC extra_pxc_container
     shell: "{{ item }}"
     with_items:
@@ -106,7 +111,7 @@
       - docker exec {{ pxc_container }} bash -c 'sed -i s#3306#'"\$(grep 'port' /home/pxc/PXC/node1.cnf | cut -d= -f2)"'# /etc/proxysql-admin.cnf'
       - docker exec {{ pxc_container }} proxysql-admin --config-file=/etc/proxysql-admin.cnf --enable
       - docker exec {{ pxc_container }} sleep 20
-      - docker exec {{ pxc_container }} pmm-admin add proxysql --username=admin --password=admin --service-name=my-new-proxysql_{{ pxc_container }} --host=127.0.0.1 --port=6032
+      - docker exec {{ pxc_container }} pmm-admin add proxysql --username=proxysql_user --password=passw0rd --service-name=my-new-proxysql_{{ pxc_container }}_{{ random_number }} --host=127.0.0.1 --port=6033
 
   - name: Execute Service, Sysbench Script inside the PXC pxc_container
     shell: "{{ item }}"

pmm_qa/requirements.txt

@@ -1 +1,2 @@
 ansible-runner==2.3.2
+requests==2.26.0

pmm_qa/setup_docker_client_images.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -xe
+
+echo
+echo "Configuring Multiple Docker Images with PMM Server and Client"
+echo "Please wait...."
+docker network create docker-client-check || true
+docker-compose -f docker-compose-clients.yaml down -v --remove-orphans
+docker-compose -f docker-compose-clients.yaml build --no-cache
+docker-compose -f docker-compose-clients.yaml up -d
+sleep 20
+echo "Adding DB Clients to PMM Server"
+docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
+docker exec pmm-client-1 pmm-admin add postgresql --query-source=pgstatements --username=pmm --password=pmm-pass --service-name=pdpgsql-1 --host=pdpgsql-1 --port=5432 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
+docker exec pmm-client-1 pmm-admin add mongodb --username=pmm --password=pmm-pass --service-name=mongodb-7.0  --host=psmdb-1 --port=27017 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true
+