From af97f551eca9faebc97c2ac3813d3dd74cb045ed Mon Sep 17 00:00:00 2001 From: Dirk Pahl Date: Wed, 24 Jun 2020 11:17:53 +0200 Subject: [PATCH 1/4] Enable JMX and verify JMX is accessible --- molecule/default/converge.yml | 4 ++++ molecule/default/prepare.yml | 9 +++++++++ molecule/default/verify.yml | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 13f3f00..f22ab44 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -4,3 +4,7 @@ become: yes roles: - role: ansible-role-kafka + vars: + kafka_environment_variables: + JMX_PORT: 1099 + KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 705b7f9..cccf94d 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -29,3 +29,12 @@ security.protocol=SASL_PLAINTEXT sasl.mechanism=PLAIN dest: "{{ kafka_test_client_conf }}" + +- name: Download JMX CLI + hosts: kafka + tasks: + - name: Download jmxterm + get_url: + url: https://github.com/jiaqi/jmxterm/releases/download/v1.0.1/jmxterm-1.0.1-uber.jar + dest: /opt/jmxterm.jar + mode: '0755' diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index a428c98..b68e91d 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -52,3 +52,11 @@ assert: that: - "'{{ topic_name }}' in list_topics_result.stdout" + - name: Test JMX connection + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_host }}:1099 + register: jmx_status + changed_when: no + - name: Verify JMX connection + assert: + that: + - "'Welcome to JMX terminal' in jmx_status.stderr" From 20170c3ba6b13da40c5ab19f9c46e13c657330ed Mon Sep 17 00:00:00 2001 From: Dirk Pahl Date: Wed, 24 Jun 2020 17:36:32 +0200 Subject: [PATCH 2/4] Set up and test JMX authentication --- README.md | 24 ++++++++++++++++++++---- defaults/main.yml | 20 ++++++++++++-------- handlers/main.yml | 2 +- molecule/default/converge.yml | 4 +--- molecule/default/molecule.yml | 3 +++ molecule/default/verify.yml | 20 +++++++++++++++++--- tasks/config.yml | 24 ++++++++++++++++++++++-- tasks/install.yml | 2 +- tasks/sasl-auth.yml | 2 +- templates/jmxremote.access.j2 | 1 + templates/jmxremote.password.j2 | 1 + templates/kafka.env.j2 | 18 ++++++++++++++++++ 12 files changed, 98 insertions(+), 23 deletions(-) create mode 100644 templates/jmxremote.access.j2 create mode 100644 templates/jmxremote.password.j2 diff --git a/README.md b/README.md index 9f8c559..c6f2df8 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ Available variables listed and described along with default values in [defaults/ kafka_zookeeper_connection_hosts: - my.zookeeper.host:2181 -## Set up monitoring for Kafka +## Set up custom JVM parameters -For enabling monitoring via JMX you use the `kafka_environment_variables` variable to adjust the respective Kafka settings. +For adjusting JVM parameters you use the `kafka_environment_variables` variable to adjust the respective Kafka settings. An overview of the variables used by Kafka can be found in the [Kafka startup script](https://github.com/apache/kafka/blob/trunk/bin/kafka-run-class.sh). @@ -42,8 +42,24 @@ Here is an example playbook: - nl2go.kafka vars: kafka_environment_variables: - JMX_PORT: 1099 - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" + KAFKA_HEAP_OPTS="-Xmx192M" + +## Set up monitoring for Kafka + +For enabling monitoring via JMX you use the `kafka_jmx_enabled` variable. Additionally you can enable JMX authentication +by setting `kafka_jmx_username` and `kafka_jmx_password`. If you leave the authentication variables undefined JMX will +be set up without authentication. + +Here is an example playbook: + + - hosts: all + roles: + - nl2go.openjdk + - nl2go.kafka + vars: + kafka_jmx_enabled: true + kafka_jmx_username: jmx + kafka_jmx_password: jmxpass ## Development diff --git a/defaults/main.yml b/defaults/main.yml index c742097..ebfb255 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -118,16 +118,10 @@ kafka_group_initial_rebalance_delay_ms: 0 # In order to set these variables for the kafka service use the optional 'kafka_environment_variables' setting. # When defined the respective variables will be written to an env file which will be used in the EnvironmentFile setting of the systemd service. # -# Sample file content for configuring JMX: -# JMX_PORT=1099 -# KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" +# Sample file content for configuring KAFKA_HEAP_OPTS: +# KAFKA_HEAP_OPTS="-Xmx192M" # # An overview of the variables used by kafka can be found in the kafka startup script here: https://github.com/apache/kafka/blob/trunk/bin/kafka-run-class.sh -# -# Sample settings: -# kafka_environment_variables: -# JMX_PORT: 1099 -# KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" kafka_environment_variables: {} # Environment file that will be created by the role and used in the Kafka systemd service definition if kafka_environment_variables is defined @@ -138,3 +132,13 @@ kafka_server_password: broker kafka_client_users: - username: guest password: guest + +# JMX related settings +kafka_jmx_enabled: false +kafka_jmx_host: "{{ kafka_host }}" +kafka_jmx_port: 1099 +kafka_jmx_rmi_port: 1099 +kafka_jmx_role: readonly +# Optional settings for using authentication +# kafka_jmx_username: jmx +# kafka_jmx_password: jmx diff --git a/handlers/main.yml b/handlers/main.yml index cf280a6..85d194d 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: Restart kafka +- name: Restart Kafka systemd: name: kafka state: restarted diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index f22ab44..9164f0c 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -5,6 +5,4 @@ roles: - role: ansible-role-kafka vars: - kafka_environment_variables: - JMX_PORT: 1099 - KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" + kafka_jmx_enabled: true diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 4207ec9..4d2d3bb 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -53,6 +53,9 @@ provisioner: kafka_zookeeper_connection_hosts: "{{ groups['zookeeper'] | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.*)', '\\1:2181') | list }}" kafka_version: 2.5.0 zookeeper_members: "{{ groups['zookeeper'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | list }}" + kafka: + kafka_jmx_username: jmx + kafka_jmx_password: molecule host_vars: "${MOLECULE_TEST_SCOPE:-default}-kfk-1": kafka_broker_id: 1 diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index b68e91d..a82b92c 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -52,11 +52,25 @@ assert: that: - "'{{ topic_name }}' in list_topics_result.stdout" - - name: Test JMX connection + - name: Test JMX connection without authentication shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_host }}:1099 - register: jmx_status + register: jmx_status_noauth changed_when: no + ignore_errors: yes + - name: Verify JMX status without authentication failed + assert: + that: + - jmx_status_noauth.rc == 1 + - name: Test JMX connection with authentication + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_host }}:1099 -u jmx -p molecule + register: jmx_status_auth + changed_when: no + ignore_errors: yes + - name: Verify JMX status with authentication succeeds + assert: + that: + - jmx_status_auth.rc == 0 - name: Verify JMX connection assert: that: - - "'Welcome to JMX terminal' in jmx_status.stderr" + - "'Welcome to JMX terminal' in jmx_status_auth.stderr" diff --git a/tasks/config.yml b/tasks/config.yml index c991be8..8906cb4 100644 --- a/tasks/config.yml +++ b/tasks/config.yml @@ -7,7 +7,7 @@ src: kafka.env.j2 dest: "{{ kafka_conf_dir }}/kafka.env" mode: '0644' - notify: Restart kafka + notify: Restart Kafka - name: Create kafka server properties file template: @@ -16,4 +16,24 @@ owner: '{{ kafka_user }}' group: '{{ kafka_group }}' mode: '0644' - notify: Restart kafka + notify: Restart Kafka + +- name: Create JMX password file + template: + src: jmxremote.password.j2 + dest: "{{ kafka_conf_dir }}/jmxremote.password" + mode: '0600' + owner: kafka + group: kafka + notify: Restart Kafka + when: kafka_jmx_enabled and kafka_jmx_username is defined and kafka_jmx_password is defined + +- name: Create JMX access file + template: + src: jmxremote.access.j2 + dest: "{{ kafka_conf_dir }}/jmxremote.access" + mode: '0600' + owner: kafka + group: kafka + notify: Restart Kafka + when: kafka_jmx_enabled and kafka_jmx_username is defined diff --git a/tasks/install.yml b/tasks/install.yml index ecb6b27..833caa3 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -47,7 +47,7 @@ src: kafka.service.j2 dest: "/etc/systemd/system/kafka.service" mode: '0644' - notify: Restart kafka + notify: Restart Kafka - name: Enable kafka service systemd: diff --git a/tasks/sasl-auth.yml b/tasks/sasl-auth.yml index 0a8d42f..84ee029 100644 --- a/tasks/sasl-auth.yml +++ b/tasks/sasl-auth.yml @@ -6,7 +6,7 @@ mode: '0600' owner: '{{ kafka_user }}' group: '{{ kafka_group }}' - notify: Restart kafka + notify: Restart Kafka - name: Add to kafka_environment_variables variable if it already exists set_fact: diff --git a/templates/jmxremote.access.j2 b/templates/jmxremote.access.j2 new file mode 100644 index 0000000..61cca46 --- /dev/null +++ b/templates/jmxremote.access.j2 @@ -0,0 +1 @@ +{{ kafka_jmx_username }} {{ kafka_jmx_role }} diff --git a/templates/jmxremote.password.j2 b/templates/jmxremote.password.j2 new file mode 100644 index 0000000..4239fcc --- /dev/null +++ b/templates/jmxremote.password.j2 @@ -0,0 +1 @@ +{{ kafka_jmx_username }} {{ kafka_jmx_password }} diff --git a/templates/kafka.env.j2 b/templates/kafka.env.j2 index c93b129..02d2d90 100644 --- a/templates/kafka.env.j2 +++ b/templates/kafka.env.j2 @@ -2,3 +2,21 @@ {{ environment_variable.key }}="{{ environment_variable.value }}" {% endfor %} LOG_DIR="{{ kafka_log_dir }}" +{% if kafka_jmx_enabled %} +{% if kafka_jmx_username is defined and kafka_jmx_password is defined %} +KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false \ +-Djava.rmi.server.hostname={{ kafka_jmx_host }} \ +-Dcom.sun.management.jmxremote.port={{ kafka_jmx_port }} \ +-Dcom.sun.management.jmxremote.rmi.port={{ kafka_jmx_rmi_port }} \ +-Dcom.sun.management.jmxremote.host={{ kafka_jmx_host }} \ +-Dcom.sun.management.jmxremote.password.file={{ kafka_conf_dir }}/jmxremote.password \ +-Dcom.sun.management.jmxremote.access.file={{ kafka_conf_dir }}/jmxremote.access" +{% else %} +KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false \ +-Dcom.sun.management.jmxremote.authenticate=false \ +-Djava.rmi.server.hostname={{ kafka_jmx_host }} \ +-Dcom.sun.management.jmxremote.port={{ kafka_jmx_port }} \ +-Dcom.sun.management.jmxremote.rmi.port={{ kafka_jmx_rmi_port }} \ +-Dcom.sun.management.jmxremote.host={{ kafka_jmx_host }}" +{% endif %} +{% endif %} From e1025782e3068000e0bdc21b33881a5057898b2e Mon Sep 17 00:00:00 2001 From: mle Date: Fri, 26 Jun 2020 09:53:48 +0200 Subject: [PATCH 3/4] Bind JMX on localhost by default. Simplify JMX verification --- defaults/main.yml | 2 +- molecule/default/molecule.yml | 1 + molecule/default/verify.yml | 15 +++------------ 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index ebfb255..c41a32b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -135,7 +135,7 @@ kafka_client_users: # JMX related settings kafka_jmx_enabled: false -kafka_jmx_host: "{{ kafka_host }}" +kafka_jmx_host: localhost kafka_jmx_port: 1099 kafka_jmx_rmi_port: 1099 kafka_jmx_role: readonly diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 4d2d3bb..af7f418 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -54,6 +54,7 @@ provisioner: kafka_version: 2.5.0 zookeeper_members: "{{ groups['zookeeper'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | list }}" kafka: + kafka_jmx_host: localhost kafka_jmx_username: jmx kafka_jmx_password: molecule host_vars: diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index a82b92c..f84a067 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -53,23 +53,14 @@ that: - "'{{ topic_name }}' in list_topics_result.stdout" - name: Test JMX connection without authentication - shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_host }}:1099 + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 register: jmx_status_noauth changed_when: no - ignore_errors: yes - - name: Verify JMX status without authentication failed - assert: - that: - - jmx_status_noauth.rc == 1 + failed_when: jmx_status_noauth.rc != 1 - name: Test JMX connection with authentication - shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_host }}:1099 -u jmx -p molecule + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 -u jmx -p molecule register: jmx_status_auth changed_when: no - ignore_errors: yes - - name: Verify JMX status with authentication succeeds - assert: - that: - - jmx_status_auth.rc == 0 - name: Verify JMX connection assert: that: From 3ce8297bc797c00cb6c8053ad6475e9d2e562cf2 Mon Sep 17 00:00:00 2001 From: mle Date: Fri, 26 Jun 2020 10:08:33 +0200 Subject: [PATCH 4/4] Organize verification steps --- molecule/default/molecule.yml | 1 + molecule/default/verify.yml | 67 ------------------------ molecule/default/verify/base.yml | 15 ++++++ molecule/default/verify/create_topic.yml | 37 +++++++++++++ molecule/default/verify/jmx.yml | 14 +++++ molecule/default/verify/main.yml | 7 +++ 6 files changed, 74 insertions(+), 67 deletions(-) delete mode 100644 molecule/default/verify.yml create mode 100644 molecule/default/verify/base.yml create mode 100644 molecule/default/verify/create_topic.yml create mode 100644 molecule/default/verify/jmx.yml create mode 100644 molecule/default/verify/main.yml diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index af7f418..20f9dea 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -76,6 +76,7 @@ provisioner: playbooks: create: ../resources/playbooks/create.yml destroy: ../resources/playbooks/destroy.yml + verify: verify/main.yml verifier: name: ansible lint: | diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml deleted file mode 100644 index f84a067..0000000 --- a/molecule/default/verify.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -- name: Verify kafka installation - hosts: kafka - tasks: - - name: Create random topic name - set_fact: - topic_name: "{{ 9999999999999999999999 | random | to_uuid }}" - run_once: yes - - name: Gather facts on listening ports - listen_ports_facts: - - name: Create list of listening ports - set_fact: - tcp_ports: "{{ ansible_facts.tcp_listen | map(attribute='port') | sort | list }}" - - name: Verify kafka port is in listening ports - assert: - that: - - 9092 in tcp_ports - - name: Check that kafka log file exists - stat: - path: /var/log/kafka/server.log - register: stat_log_result - failed_when: not stat_log_result.stat.exists - - name: Add a topic to the kafka server - environment: - KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg" - command: "/usr/local/kafka/bin/kafka-topics.sh \ - --create --bootstrap-server {{ kafka_host }}:9092 --replication-factor 3 --partitions 1 --topic {{ topic_name }} \ - --command-config {{ kafka_test_client_conf }}" - register: create_topic_result - run_once: yes - - name: Verify topic creation return code - assert: - that: - - create_topic_result.rc == 0 - - name: Verify topic creation output - assert: - that: - - "'Created topic {{ topic_name }}' in create_topic_result.stdout" - - name: List topics from the kafka server - environment: - KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg" - command: "/usr/local/kafka/bin/kafka-topics.sh \ - --list --bootstrap-server {{ kafka_host }}:9092 \ - --command-config {{ kafka_test_client_conf }}" - register: list_topics_result - changed_when: no - - name: Verify topic listing return code - assert: - that: - - list_topics_result.rc == 0 - - name: Verify created topic is in topic list - assert: - that: - - "'{{ topic_name }}' in list_topics_result.stdout" - - name: Test JMX connection without authentication - shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 - register: jmx_status_noauth - changed_when: no - failed_when: jmx_status_noauth.rc != 1 - - name: Test JMX connection with authentication - shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 -u jmx -p molecule - register: jmx_status_auth - changed_when: no - - name: Verify JMX connection - assert: - that: - - "'Welcome to JMX terminal' in jmx_status_auth.stderr" diff --git a/molecule/default/verify/base.yml b/molecule/default/verify/base.yml new file mode 100644 index 0000000..4a97517 --- /dev/null +++ b/molecule/default/verify/base.yml @@ -0,0 +1,15 @@ +--- +- name: Gather facts on listening ports + listen_ports_facts: +- name: Create list of listening ports + set_fact: + tcp_ports: "{{ ansible_facts.tcp_listen | map(attribute='port') | sort | list }}" +- name: Verify Kafka port is in listening ports + assert: + that: + - 9092 in tcp_ports +- name: Check that Kafka log file exists + stat: + path: /var/log/kafka/server.log + register: stat_log_result + failed_when: not stat_log_result.stat.exists diff --git a/molecule/default/verify/create_topic.yml b/molecule/default/verify/create_topic.yml new file mode 100644 index 0000000..b795feb --- /dev/null +++ b/molecule/default/verify/create_topic.yml @@ -0,0 +1,37 @@ +--- +- name: Create random topic name + set_fact: + topic_name: "{{ 9999999999999999999999 | random | to_uuid }}" + run_once: yes +- name: Add a topic to the Kafka server + environment: + KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg" + command: "/usr/local/kafka/bin/kafka-topics.sh \ + --create --bootstrap-server {{ kafka_host }}:9092 --replication-factor 3 --partitions 1 --topic {{ topic_name }} \ + --command-config {{ kafka_test_client_conf }}" + register: create_topic_result + run_once: yes +- name: Verify topic creation return code + assert: + that: + - create_topic_result.rc == 0 +- name: Verify topic creation output + assert: + that: + - "'Created topic {{ topic_name }}' in create_topic_result.stdout" +- name: List topics from the Kafka server + environment: + KAFKA_OPTS: "-Djava.security.auth.login.config={{ kafka_conf_dir }}/jaas.cfg" + command: "/usr/local/kafka/bin/kafka-topics.sh \ + --list --bootstrap-server {{ kafka_host }}:9092 \ + --command-config {{ kafka_test_client_conf }}" + register: list_topics_result + changed_when: no +- name: Verify topic listing return code + assert: + that: + - list_topics_result.rc == 0 +- name: Verify created topic is in topic list + assert: + that: + - "'{{ topic_name }}' in list_topics_result.stdout" diff --git a/molecule/default/verify/jmx.yml b/molecule/default/verify/jmx.yml new file mode 100644 index 0000000..9de82ca --- /dev/null +++ b/molecule/default/verify/jmx.yml @@ -0,0 +1,14 @@ +--- +- name: Test JMX connection without authentication + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 + register: jmx_status_noauth + changed_when: no + failed_when: jmx_status_noauth.rc != 1 +- name: Test JMX connection with authentication + shell: echo "exit" | java -jar /opt/jmxterm.jar -l {{ kafka_jmx_host }}:1099 -u jmx -p molecule + register: jmx_status_auth + changed_when: no +- name: Verify JMX connection + assert: + that: + - "'Welcome to JMX terminal' in jmx_status_auth.stderr" diff --git a/molecule/default/verify/main.yml b/molecule/default/verify/main.yml new file mode 100644 index 0000000..c299288 --- /dev/null +++ b/molecule/default/verify/main.yml @@ -0,0 +1,7 @@ +--- +- name: Verify Kafka installation + hosts: kafka + tasks: + - include_tasks: base.yml + - include_tasks: create_topic.yml + - include_tasks: jmx.yml