Make grafana_agent Idempotent

roles/grafana_agent/handlers/main.yaml

@@ -0,0 +1,12 @@
+---
+- name: Restart Grafana Agent
+  become: true
+  ansible.builtin.service:
+    name: grafana-agent
+    state: restarted
+    daemon_reload: true
+  listen: "restart grafana-agent"
+
+- name: Check Grafana Agent is started properly
+  ansible.builtin.include_tasks: ga-started.yml
+  listen: "restart grafana-agent"

roles/grafana_agent/tasks/configure.yaml

@@ -1,22 +1,4 @@
 ---
-- name: Stop grafana-agent if installed
-  block:
-    # this will fail the verify first time of installation if local binary is used
-    - name: Ensure grafana-agent is stopped
-      ansible.builtin.systemd:
-        name: grafana-agent
-        enabled: true
-        state: stopped
-  rescue:
-    # make sure that the service isn't actually installed
-    - name: Get the list of services
-      ansible.builtin.service_facts:
-
-    - name: Verify that grafana-agent is not installed
-      ansible.builtin.assert:
-        that:
-          - _grafana_agent_systemd_unit not in services
-
 # these tasks are ran in both install and configure, as directories could have changed
 - name: Configure directories
   ansible.builtin.import_tasks: install/directories.yaml
@@ -36,6 +18,7 @@
     owner: root
     group: root
     mode: 0644
+  notify: "restart grafana-agent"  
 
 - name: Create the Service Environment file
   ansible.builtin.template:
@@ -44,6 +27,7 @@
     owner: root
     group: "{{ grafana_agent_user_group }}"
     mode: 0640
+  notify: "restart grafana-agent"
 
 - name: Create Grafana Agent config
   ansible.builtin.template:
@@ -53,6 +37,7 @@
     owner: root
     group: "{{ grafana_agent_user_group }}"
     mode: 0640
+  notify: "restart grafana-agent"  
   when: grafana_agent_provisioned_config_file | length == 0
 
 - name: Copy Grafana Agent config
@@ -62,4 +47,5 @@
     owner: root
     group: "{{ grafana_agent_user_group }}"
     mode: 0640
+  notify: "restart grafana-agent"  
   when: grafana_agent_provisioned_config_file | length > 0

roles/grafana_agent/tasks/ga-started.yaml

@@ -0,0 +1,28 @@
+---
+- name:  Health check Grafana Agent
+  ansible.builtin.uri:
+    url: "{{ _grafana_agent_healthcheck_endpoint }}"
+    follow_redirects: none
+    method: GET
+  register: _result
+  failed_when: false
+  until: _result.status == 200
+  retries: 3 
+  delay: 5
+  changed_when: false
+
+- name: Check system logs if Grafana Agent is not started
+  when: _result.status != 200 
+  block:
+  - name: Run journalctl
+    ansible.builtin.shell:
+      cmd: "journalctl -u grafana-agent -b -n20 --no-pager"   
+    register: journal_ret
+    changed_when: false
+  - name: Output Grafana agent logs
+    ansible.builtin.debug:
+      var: journal_ret.stdout_lines   
+  - name: Rise alerts 
+    ansible.builtin.assert:
+      that: false
+      fail_msg: "Service grafana-agent hasn't started." 

roles/grafana_agent/tasks/main.yaml

@@ -40,6 +40,8 @@
     name: grafana-agent
     enabled: true
     state: started
+  failed_when: false
+  changed_when: false
   tags:
     - grafana_agent_install
     - grafana_agent_configure

roles/grafana_agent/vars/main.yaml

@@ -31,3 +31,6 @@
 # systemd info
 _grafana_agent_systemd_dir: /lib/systemd/system/
 _grafana_agent_systemd_unit: grafana-agent.service
+
+# Server http address, used in self health check after start
+_grafana_agent_healthcheck_endpoint: "http://{{ grafana_agent_flags_extra['server.http.address'] if grafana_agent_flags_extra['server.http.address'] is defined else '127.0.0.1:12345' }}/-/ready"