Merge pull request #3 from nl2go/feature/jmx-sasl

Enable JMX authentication

README.md

@@ -28,9 +28,9 @@ Available variables listed and described along with default values in [defaults/
         kafka_zookeeper_connection_hosts:
           - my.zookeeper.host:2181
 
-## Set up monitoring for Kafka
+## Set up custom JVM parameters
 
-For enabling monitoring via JMX you use the `kafka_environment_variables` variable to adjust the respective Kafka settings.
+For adjusting JVM parameters you use the `kafka_environment_variables` variable to adjust the respective Kafka settings.
 
 An overview of the variables used by Kafka can be found in the [Kafka startup script](https://github.com/apache/kafka/blob/trunk/bin/kafka-run-class.sh).
 
@@ -42,8 +42,24 @@ Here is an example playbook:
         - nl2go.kafka
       vars:
         kafka_environment_variables:
-          JMX_PORT: 1099
-          KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+          KAFKA_HEAP_OPTS="-Xmx192M"
+
+## Set up monitoring for Kafka
+
+For enabling monitoring via JMX you use the `kafka_jmx_enabled` variable. Additionally you can enable JMX authentication
+by setting `kafka_jmx_username` and `kafka_jmx_password`. If you leave the authentication variables undefined JMX will
+be set up without authentication.
+
+Here is an example playbook:
+
+    - hosts: all
+      roles:
+        - nl2go.openjdk
+        - nl2go.kafka
+      vars:
+        kafka_jmx_enabled: true
+        kafka_jmx_username: jmx
+        kafka_jmx_password: jmxpass
 
 ## Development
 

defaults/main.yml

@@ -118,16 +118,10 @@ kafka_group_initial_rebalance_delay_ms: 0
 # In order to set these variables for the kafka service use the optional 'kafka_environment_variables' setting.
 # When defined the respective variables will be written to an env file which will be used in the EnvironmentFile setting of the systemd service.
 #
-# Sample file content for configuring JMX:
-# JMX_PORT=1099
-# KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+# Sample file content for configuring KAFKA_HEAP_OPTS:
+# KAFKA_HEAP_OPTS="-Xmx192M"
 #
 # An overview of the variables used by kafka can be found in the kafka startup script here: https://github.com/apache/kafka/blob/trunk/bin/kafka-run-class.sh
-#
-# Sample settings:
-# kafka_environment_variables:
-#   JMX_PORT: 1099
-#   KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
 kafka_environment_variables: {}
 
 # Environment file that will be created by the role and used in the Kafka systemd service definition if kafka_environment_variables is defined
@@ -138,3 +132,13 @@ kafka_server_password: broker
 kafka_client_users:
   - username: guest
     password: guest
+
+# JMX related settings
+kafka_jmx_enabled: false
+kafka_jmx_host: localhost
+kafka_jmx_port: 1099
+kafka_jmx_rmi_port: 1099
+kafka_jmx_role: readonly
+# Optional settings for using authentication
+# kafka_jmx_username: jmx
+# kafka_jmx_password: jmx

handlers/main.yml

@@ -1,5 +1,5 @@
 ---
-- name: Restart kafka
+- name: Restart Kafka
   systemd:
     name: kafka
     state: restarted

molecule/default/converge.yml

@@ -4,3 +4,5 @@
   become: yes
   roles:
     - role: ansible-role-kafka
+      vars:
+        kafka_jmx_enabled: true

molecule/default/molecule.yml

@@ -53,6 +53,10 @@ provisioner:
         kafka_zookeeper_connection_hosts: "{{ groups['zookeeper'] | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.*)', '\\1:2181') | list }}"
         kafka_version: 2.5.0
         zookeeper_members: "{{ groups['zookeeper'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | list }}"
+      kafka:
+        kafka_jmx_host: localhost
+        kafka_jmx_username: jmx
+        kafka_jmx_password: molecule
     host_vars:
       "${MOLECULE_TEST_SCOPE:-default}-kfk-1":
         kafka_broker_id: 1
@@ -72,6 +76,7 @@ provisioner:
   playbooks:
     create: ../resources/playbooks/create.yml
     destroy: ../resources/playbooks/destroy.yml
+    verify: verify/main.yml
 verifier:
   name: ansible
   lint: |

molecule/default/prepare.yml

@@ -29,3 +29,12 @@
           security.protocol=SASL_PLAINTEXT
           sasl.mechanism=PLAIN
         dest: "{{ kafka_test_client_conf }}"
+
+- name: Download JMX CLI
+  hosts: kafka
+  tasks:
+    - name: Download jmxterm
+      get_url:
+        url: https://github.com/jiaqi/jmxterm/releases/download/v1.0.1/jmxterm-1.0.1-uber.jar
+        dest: /opt/jmxterm.jar
+        mode: '0755'

molecule/default/verify/base.yml

@@ -0,0 +1,15 @@
+---
+- name: Gather facts on listening ports
+  listen_ports_facts:
+- name: Create list of listening ports
+  set_fact:
+    tcp_ports: "{{ ansible_facts.tcp_listen | map(attribute='port') | sort | list }}"
+- name: Verify Kafka port is in listening ports
+  assert:
+    that:
+      - 9092 in tcp_ports
+- name: Check that Kafka log file exists
+  stat:
+    path: /var/log/kafka/server.log
+  register: stat_log_result
+  failed_when: not stat_log_result.stat.exists

molecule/default/verify/create_topic.yml

@@ -0,0 +1,37 @@
+---
+- name: Create random topic name
+  set_fact:
+    topic_name: "{{ 9999999999999999999999 | random | to_uuid }}"
+  run_once: yes
+- name: Add a topic to the Kafka server
+  environment:
+    KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg"
+  command: "/usr/local/kafka/bin/kafka-topics.sh \
+    --create --bootstrap-server {{ kafka_host }}:9092 --replication-factor 3 --partitions 1 --topic {{ topic_name }} \
+    --command-config {{ kafka_test_client_conf }}"
+  register: create_topic_result
+  run_once: yes
+- name: Verify topic creation return code
+  assert:
+    that:
+      - create_topic_result.rc == 0
+- name: Verify topic creation output
+  assert:
+    that:
+      - "'Created topic {{ topic_name }}' in create_topic_result.stdout"
+- name: List topics from the Kafka server
+  environment:
+    KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg"
+  command: "/usr/local/kafka/bin/kafka-topics.sh \
+    --list --bootstrap-server {{ kafka_host }}:9092 \
+    --command-config {{ kafka_test_client_conf }}"
+  register: list_topics_result
+  changed_when: no
+- name: Verify topic listing return code
+  assert:
+    that:
+      - list_topics_result.rc == 0
+- name: Verify created topic is in topic list
+  assert:
+    that:
+      - "'{{ topic_name }}' in list_topics_result.stdout"

molecule/default/verify/jmx.yml

@@ -0,0 +1,14 @@
+---
+- name: Test JMX connection without authentication
+  shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099
+  register: jmx_status_noauth
+  changed_when: no
+  failed_when: jmx_status_noauth.rc != 1
+- name: Test JMX connection with authentication
+  shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 -u jmx -p molecule
+  register: jmx_status_auth
+  changed_when: no
+- name: Verify JMX connection
+  assert:
+    that:
+      - "'Welcome to JMX terminal' in jmx_status_auth.stderr"

molecule/default/verify/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Verify Kafka installation
+  hosts: kafka
+  tasks:
+    - include_tasks: base.yml
+    - include_tasks: create_topic.yml
+    - include_tasks: jmx.yml

tasks/config.yml

@@ -7,7 +7,7 @@
     src: kafka.env.j2
     dest: "{{ kafka_conf_dir }}/kafka.env"
     mode: '0644'
-  notify: Restart kafka
+  notify: Restart Kafka
 
 - name: Create kafka server properties file
   template:
@@ -16,4 +16,24 @@
     owner: '{{ kafka_user }}'
     group: '{{ kafka_group }}'
     mode: '0644'
-  notify: Restart kafka
+  notify: Restart Kafka
+
+- name: Create JMX password file
+  template:
+    src: jmxremote.password.j2
+    dest: "{{ kafka_conf_dir }}/jmxremote.password"
+    mode: '0600'
+    owner: kafka
+    group: kafka
+  notify: Restart Kafka
+  when: kafka_jmx_enabled and kafka_jmx_username is defined and kafka_jmx_password is defined
+
+- name: Create JMX access file
+  template:
+    src: jmxremote.access.j2
+    dest: "{{ kafka_conf_dir }}/jmxremote.access"
+    mode: '0600'
+    owner: kafka
+    group: kafka
+  notify: Restart Kafka
+  when: kafka_jmx_enabled and kafka_jmx_username is defined

tasks/install.yml

@@ -47,7 +47,7 @@
     src: kafka.service.j2
     dest: "/etc/systemd/system/kafka.service"
     mode: '0644'
-  notify: Restart kafka
+  notify: Restart Kafka
 
 - name: Enable kafka service
   systemd:

tasks/sasl-auth.yml

@@ -6,7 +6,7 @@
     mode: '0600'
     owner: '{{ kafka_user }}'
     group: '{{ kafka_group }}'
-  notify: Restart kafka
+  notify: Restart Kafka
 
 - name: Add to kafka_environment_variables variable if it already exists
   set_fact:

templates/jmxremote.access.j2

@@ -0,0 +1 @@
+{{ kafka_jmx_username }} {{ kafka_jmx_role }}

templates/jmxremote.password.j2

@@ -0,0 +1 @@
+{{ kafka_jmx_username }} {{ kafka_jmx_password }}

templates/kafka.env.j2

@@ -2,3 +2,21 @@
 {{ environment_variable.key }}="{{ environment_variable.value }}"
 {% endfor %}
 LOG_DIR="{{ kafka_log_dir }}"
+{% if kafka_jmx_enabled %}
+{% if kafka_jmx_username is defined and kafka_jmx_password is defined %}
+KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false \
+-Djava.rmi.server.hostname={{ kafka_jmx_host }} \
+-Dcom.sun.management.jmxremote.port={{ kafka_jmx_port }} \
+-Dcom.sun.management.jmxremote.rmi.port={{ kafka_jmx_rmi_port }} \
+-Dcom.sun.management.jmxremote.host={{ kafka_jmx_host }} \
+-Dcom.sun.management.jmxremote.password.file={{ kafka_conf_dir }}/jmxremote.password \
+-Dcom.sun.management.jmxremote.access.file={{ kafka_conf_dir }}/jmxremote.access"
+{% else %}
+KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false \
+-Dcom.sun.management.jmxremote.authenticate=false \
+-Djava.rmi.server.hostname={{ kafka_jmx_host }} \
+-Dcom.sun.management.jmxremote.port={{ kafka_jmx_port }} \
+-Dcom.sun.management.jmxremote.rmi.port={{ kafka_jmx_rmi_port }} \
+-Dcom.sun.management.jmxremote.host={{ kafka_jmx_host }}"
+{% endif %}
+{% endif %}