Merge branch 'main' into PMM-13317-remove-clickhouse-exporter

.gitignore

@@ -41,3 +41,5 @@ cli-tests/playwright-report/
 
 api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
+
+packer.log

build/ansible/pmm2/post-build-actions.yml

@@ -7,11 +7,12 @@
   vars:
     pmm_client_repos: "pmm2-client testing"
     pmm_client_repos_final: "pmm2-client release"
+    pmm_running_in_docker: false  # Set a default value
 
   tasks:
     # pmm-managed checks that if /srv/pmm-distribution exist, it contains "docker", "ovf", or "ami" (all lowercase)
     - name: Detect distribution        | Create '/srv/pmm-distribution' file for Docker
-      when: ansible_virtualization_type == "docker"
+      when: pmm_running_in_docker
       copy:
         content: "docker"
         dest: /srv/pmm-distribution
@@ -28,6 +29,7 @@
         ( ansible_virtualization_type == "xen"
         or ansible_virtualization_type == "kvm" )
         and ansible_system_vendor != "DigitalOcean"
+        and not pmm_running_in_docker
       copy:
         content: "ami"
         dest: /srv/pmm-distribution
@@ -50,14 +52,14 @@
         state: present
         update_cache: yes
       when:
-        - ansible_virtualization_type != "docker"
+        - not pmm_running_in_docker
         - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux'
         - ansible_distribution_major_version == '9'
 
     - name: Set locale to en_US.utf8   | EL9
       command: localectl set-locale LANG=en_US.utf8
       when:
-        - ansible_virtualization_type != "docker"
+        - not pmm_running_in_docker
         - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux'
         - ansible_distribution_major_version == '9'
 
@@ -102,11 +104,11 @@
       # ignore_errors: True
 
     - name: Supervisord stop           | Stop supervisord service for AMI/OVF
-      when: ansible_virtualization_type != "docker"
+      when: not pmm_running_in_docker
       service: name=supervisord state=stopped enabled=yes
 
     - name: Supervisord stop           | Stop supervisord service for docker
-      when: ansible_virtualization_type == "docker"
+      when: pmm_running_in_docker
       shell: supervisorctl shutdown
 
       # PMM-11336 - The previous steps failed to start PostgreSQL using supervisord,
@@ -118,14 +120,14 @@
       become: yes
       become_user: postgres
       ignore_errors: yes
-      when: ansible_virtualization_type != "docker"
+      when: not pmm_running_in_docker
 
     - name: Start PostgreSQL database without supervisord
       command: /usr/pgsql-14/bin/pg_ctl start -D /srv/postgres14
       become: yes
       become_user: postgres
       ignore_errors: yes
-      when: ansible_virtualization_type != "docker"
+      when: not pmm_running_in_docker
 
     - name: Remove pmm-managed database EL7
       when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7'
@@ -160,18 +162,18 @@
       become: yes
       become_user: postgres
       ignore_errors: yes
-      when: ansible_virtualization_type != "docker"
+      when: not pmm_running_in_docker
 
     - name: Supervisord stop           | Stop supervisord service for AMI/OVF
-      when: ansible_virtualization_type != "docker"
+      when: not pmm_running_in_docker
       service: name=supervisord state=stopped enabled=yes
 
     - name: Supervisord stop EL7       | Stop supervisord service for docker
-      when: ansible_virtualization_type == "docker" and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7'
+      when: pmm_running_in_docker and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7'
       shell: supervisorctl shutdown
 
     - name: Supervisord stop EL9       | Stop supervisord service for docker
-      when: ansible_virtualization_type == "docker" and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9'
+      when: pmm_running_in_docker and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9'
       shell: /usr/local/bin/supervisorctl shutdown
 
     - name: Cleanup yum cache          | Cleanup yum cache

build/ansible/roles/pmm2-images/tasks/main.yml

@@ -29,7 +29,7 @@
 
 # local yum repo for building of pmm server docker image in autobuild jobs
 - name: PMM                        | Add local YUM repository
-  when: ansible_virtualization_type == "docker" 
+  when: pmm_running_in_docker
   yum_repository:
     name: local
     description: Local YUM repository - x86_64
@@ -89,7 +89,7 @@
 - name: Create users for non-docker images          | Create users
   user:
     name: "pmm"
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
 
 - name: Create users and groups in docker container
   block:
@@ -120,7 +120,7 @@
         - { name: grafana, uid: 998, comment: "Grafana Dashboard", shell: "/sbin/nologin", home: "/etc/grafana", group: grafana, }
         - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse, }
         - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent, }
-  when: ansible_virtualization_type == "docker"
+  when: pmm_running_in_docker
 
 - name: Create directories        | Create dirs
   file: path={{ item }} state=directory owner=pmm group=pmm
@@ -138,7 +138,7 @@
     mode: "0775"
 
 - name: Create dirs                | Create dirs
-  when: ansible_virtualization_type == "docker"
+  when: pmm_running_in_docker
   file: path={{ item }} state=directory
   with_items:
     - /var/lib/cloud/scripts/per-once
@@ -178,11 +178,11 @@
     enablerepo: "{{ pmm_client_repo_name }}"
 
 - name: Disable pmm-agent service | Disable pmm-agent
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
   service: name=pmm-agent state=stopped enabled=no
 
 - name: Create tmp dirs           | Create tmp dirs
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
   command: /usr/bin/systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev
 
 - name: Copy grafana.ini file for the first run

build/ansible/roles/pmm2-images/vars/main.yml

@@ -4,3 +4,5 @@ pmm_client_repos: "pmm2-client testing"
 
 # line below is sed'ed by build-server-docker script
 pmm_client_repo_name: "pmm2-client-testing-x86_64"
+
+pmm_running_in_docker: false  # Set a default value

build/ansible/roles/supervisord-init/tasks/main.yml

@@ -69,7 +69,7 @@
     state: directory
 
 - name: Configure supervisor       | Add /etc/tmpfiles.d/supervisor.conf config
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
   copy:
     content: |
       D /var/run/supervisor 0775 root root -
@@ -91,15 +91,15 @@
     value: dummy
 
 - name: Configure supervisor       | Increase number of open files for jobs
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
   ini_file:
     dest: /etc/supervisord.conf
     section: supervisord
     option: minfds
     value: "800000"
 
 - name: Configure supervisor       | Add supervisord.service
-  when: ansible_virtualization_type != "docker"
+  when: not pmm_running_in_docker
   copy:
     src: supervisord.service
     dest: /usr/lib/systemd/system/

build/docker/server/Dockerfile.el9

@@ -33,9 +33,9 @@ COPY gitCommit /tmp/gitCommit
 COPY ansible /opt/ansible
 # NOTE: this needs to be refactored, since some of the playbooks are duplicates
 RUN cp -r /opt/ansible/roles /opt/ansible/pmm2-docker/roles
-RUN ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2-docker/main.yml \
-  && ansible-playbook -vvv -i 'localhost,' -c local /usr/share/pmm-update/ansible/playbook/tasks/update.yml \
-  && ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2/post-build-actions.yml
+RUN ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2-docker/main.yml -e "pmm_running_in_docker=true" \
+  && ansible-playbook -vvv -i 'localhost,' -c local /usr/share/pmm-update/ansible/playbook/tasks/update.yml -e "pmm_running_in_docker=true" \
+  && ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2/post-build-actions.yml -e "pmm_running_in_docker=true"
 
 COPY entrypoint.sh /opt/entrypoint.sh
 HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -f http://127.0.0.1/v1/readyz || exit 1

build/packer/ansible/agent-aws.yml

@@ -22,10 +22,36 @@
     - name: Add a repository for NodeJS
       shell: "curl -fsSL https://rpm.nodesource.com/setup_{{ node_version }} | bash -"
 
+    - name: Install Oracle EPEL Repo
+      yum:
+        name: oracle-epel-release-el9
+        state: latest
+
+    - name: Install yum-utils
+      yum:
+        name: yum-utils
+        state: latest
+
+    - name: Add Docker repository
+      command: yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo
+
+    - name: Install Docker packages
+      yum:
+        name:
+          - docker-ce
+          - docker-ce-cli
+          - containerd.io
+        state: latest
+
+    - name: Enable Docker service
+      systemd:
+        name: docker
+        enabled: yes
+        state: started
+
     - name: Install packages
-      package:
+      yum:
         name:
-          - docker
           - git
           - jq
           - gettext
@@ -41,22 +67,22 @@
           - gcc
           - make
           - perl
-          # - p7zip
+            #- p7zip
           - nodejs
-        state: present
+        state: latest
 
     - name: Install Java17
-      package:
-        name: jdk-17-headless
-        state: present
+      yum:
+        name: java-17-openjdk-headless
+        state: latest
 
     - name: Install pip3 and ansible
-      package:
+      yum:
         name:
           - python3-pip
           - ansible-core
           - glibc-langpack-en
-        state: present
+        state: latest
 
     - name: Install ansible collections
       shell: "ansible-galaxy collection install {{ item }}"
@@ -110,8 +136,7 @@
         mode: "555"
 
     - name: Install Python's virtualenv
-      pip:
-        name: virtualenv
+      command: python3 -m venv /home/ec2-user/venv
 
     - name: Install Python modules
       pip:
@@ -276,16 +301,6 @@
         name: fs.inotify.max_user_watches
         value: 1048576
 
-    - name: Get disk UUID
-      command: blkid -s UUID -o value /dev/nvme0n1p1
-      register: disk_uuid
-
-    - name: Replace fstab entities with new mount options
-      lineinfile:
-        path: /etc/fstab
-        regexp: "^UUID={{ disk_uuid.stdout }}"
-        line: "UUID={{ disk_uuid.stdout }}     /           xfs    defaults,noatime,nobarrier,discard  1   1"
-
     - name: Add repo.ci.percona.com to /etc/hosts
       lineinfile:
         dest: /etc/hosts

build/packer/aws.pkr.hcl

@@ -39,9 +39,9 @@ source "amazon-ebs" "agent" {
     iit-billing-tag = "pmm-worker"
   }
   launch_block_device_mappings {
-    device_name           = "/dev/xvda"
-    volume_size           = 50
-    volume_type           = "gp3"
+    device_name = "/dev/sda1"
+    volume_size = 50
+    volume_type = "gp3"
     delete_on_termination = true
   }
   vpc_filter {
@@ -114,7 +114,7 @@ build {
     user                   = "ec2-user"
     ansible_env_vars       = ["ANSIBLE_NOCOLOR=True"]
     extra_arguments = [
-      "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-v"
+      "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-vvv"
     ]
     playbook_file          = "./ansible/agent-aws.yml"
   }