From 06db2df913df3dcd2f0dc6ff1043e7fad85def48 Mon Sep 17 00:00:00 2001 From: Chandra Gorantla Date: Wed, 25 Jan 2023 17:23:37 -0500 Subject: [PATCH 1/4] NMS-15336: provide an option to disable kafka offset provider --- .../server/ConditionalKafkaOffsetContext.java | 72 +++++++++++++++++++ ...applicationContext-ipc-offset-provider.xml | 36 ++++++++++ ...plicationContext-ipc-sink-kafka-server.xml | 19 +---- .../META-INF/opennms/component-dao.xml | 2 + 4 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java create mode 100644 core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-offset-provider.xml diff --git a/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java new file mode 100644 index 000000000000..807777783b54 --- /dev/null +++ b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * This file is part of OpenNMS(R). + * + * Copyright (C) 2023 The OpenNMS Group, Inc. + * OpenNMS(R) is Copyright (C) 1999-2023 The OpenNMS Group, Inc. + * + * OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc. + * + * OpenNMS(R) is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * OpenNMS(R) is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with OpenNMS(R). If not, see: + * http://www.gnu.org/licenses/ + * + * For more information contact: + * OpenNMS(R) Licensing + * http://www.opennms.org/ + * http://www.opennms.com/ + *******************************************************************************/ + +package org.opennms.core.ipc.sink.kafka.server; + +import org.opennms.core.ipc.sink.api.MessageConsumerManager; +import org.opennms.core.logging.Logging; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.ConfigurationCondition; +import org.springframework.context.annotation.ImportResource; +import org.springframework.core.type.AnnotatedTypeMetadata; + +import static org.opennms.core.ipc.common.kafka.KafkaSinkConstants.KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX; +import static org.opennms.core.ipc.common.kafka.KafkaSinkConstants.KAFKA_CONFIG_SYS_PROP_PREFIX; + +@Configuration +@Conditional(ConditionalKafkaOffsetContext.Condition.class) +@ImportResource("/META-INF/opennms/applicationContext-ipc-offset-provider.xml") +public class ConditionalKafkaOffsetContext { + + private static final Logger LOG = LoggerFactory.getLogger(ConditionalKafkaOffsetContext.class); + + static class Condition implements ConfigurationCondition { + @Override + public ConfigurationPhase getConfigurationPhase() { + return ConfigurationPhase.PARSE_CONFIGURATION; + } + + @Override + public boolean matches(final ConditionContext context, final AnnotatedTypeMetadata metadata) { + // Have to explicityly set disabled to true to disable otherwise defaults to enabled. + boolean disabled = Boolean.getBoolean(KAFKA_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + if (!disabled) { + disabled = Boolean.getBoolean(KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + } + final var enabled = !disabled; + try (Logging.MDCCloseable mdc = Logging.withPrefixCloseable(MessageConsumerManager.LOG_PREFIX)) { + LOG.debug("Enable Kafka Offset: {}", enabled); + } + return enabled; + } + } +} diff --git a/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-offset-provider.xml b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-offset-provider.xml new file mode 100644 index 000000000000..535895f55131 --- /dev/null +++ b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-offset-provider.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + name + Kafka Lag + + + description + Metrics related to Kafka Lag + + + + + + \ No newline at end of file diff --git a/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-sink-kafka-server.xml b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-sink-kafka-server.xml index 48b2c85fbf64..11d0a25a9d5d 100644 --- a/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-sink-kafka-server.xml +++ b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/applicationContext-ipc-sink-kafka-server.xml @@ -21,10 +21,6 @@ - - - - @@ -40,19 +36,6 @@ - - - - - - name - Kafka Lag - - - description - Metrics related to Kafka Lag - - - + diff --git a/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/component-dao.xml b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/component-dao.xml index 4109f6a67fd4..edb22c3637f9 100644 --- a/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/component-dao.xml +++ b/core/ipc/sink/kafka/server/src/main/resources/META-INF/opennms/component-dao.xml @@ -10,4 +10,6 @@ + + From 3953f1c9376f918fb1b216ff135355a72acb6a69 Mon Sep 17 00:00:00 2001 From: Chandra Gorantla Date: Wed, 25 Jan 2023 20:22:38 -0500 Subject: [PATCH 2/4] NMS-15336: Need to enable only when sink strategy is kafka --- .../server/ConditionalKafkaOffsetContext.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java index 807777783b54..59a3e7fc46de 100644 --- a/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java +++ b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java @@ -29,6 +29,7 @@ package org.opennms.core.ipc.sink.kafka.server; import org.opennms.core.ipc.sink.api.MessageConsumerManager; +import org.opennms.core.ipc.sink.common.SinkStrategy; import org.opennms.core.logging.Logging; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +42,7 @@ import static org.opennms.core.ipc.common.kafka.KafkaSinkConstants.KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX; import static org.opennms.core.ipc.common.kafka.KafkaSinkConstants.KAFKA_CONFIG_SYS_PROP_PREFIX; +import static org.opennms.core.ipc.sink.common.SinkStrategy.Strategy.KAFKA; @Configuration @Conditional(ConditionalKafkaOffsetContext.Condition.class) @@ -57,16 +59,20 @@ public ConfigurationPhase getConfigurationPhase() { @Override public boolean matches(final ConditionContext context, final AnnotatedTypeMetadata metadata) { - // Have to explicityly set disabled to true to disable otherwise defaults to enabled. - boolean disabled = Boolean.getBoolean(KAFKA_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); - if (!disabled) { - disabled = Boolean.getBoolean(KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + final boolean kafkaSinkEnabled = KAFKA.equals(SinkStrategy.getSinkStrategy()); + if (kafkaSinkEnabled) { + // Have to explicitly set disabled to true to disable otherwise defaults to enabled. + boolean disabled = Boolean.getBoolean(KAFKA_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + if (!disabled) { + disabled = Boolean.getBoolean(KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + } + final var offsetEnabled = !disabled; + try (Logging.MDCCloseable mdc = Logging.withPrefixCloseable(MessageConsumerManager.LOG_PREFIX)) { + LOG.debug("Enable Kafka Offset: {}", offsetEnabled); + } + return offsetEnabled; } - final var enabled = !disabled; - try (Logging.MDCCloseable mdc = Logging.withPrefixCloseable(MessageConsumerManager.LOG_PREFIX)) { - LOG.debug("Enable Kafka Offset: {}", enabled); - } - return enabled; + return kafkaSinkEnabled; } } } From 40ac1f0429926c5383410da65a91b3024f510290 Mon Sep 17 00:00:00 2001 From: Jesse White Date: Tue, 11 Jul 2023 09:54:08 -0400 Subject: [PATCH 3/4] NMS-15336: Refine implementation and add settings to the docs. --- .../server/ConditionalKafkaOffsetContext.java | 26 +++++++++++-------- .../pages/configuration/tuning-kafka.adoc | 14 +++++++++- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java index 59a3e7fc46de..e6cf6abed5d9 100644 --- a/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java +++ b/core/ipc/sink/kafka/server/src/main/java/org/opennms/core/ipc/sink/kafka/server/ConditionalKafkaOffsetContext.java @@ -60,19 +60,23 @@ public ConfigurationPhase getConfigurationPhase() { @Override public boolean matches(final ConditionContext context, final AnnotatedTypeMetadata metadata) { final boolean kafkaSinkEnabled = KAFKA.equals(SinkStrategy.getSinkStrategy()); - if (kafkaSinkEnabled) { - // Have to explicitly set disabled to true to disable otherwise defaults to enabled. - boolean disabled = Boolean.getBoolean(KAFKA_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); - if (!disabled) { - disabled = Boolean.getBoolean(KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); - } - final var offsetEnabled = !disabled; - try (Logging.MDCCloseable mdc = Logging.withPrefixCloseable(MessageConsumerManager.LOG_PREFIX)) { - LOG.debug("Enable Kafka Offset: {}", offsetEnabled); + if (!kafkaSinkEnabled) { + return false; + } + // Default to enabled, and require an explicit value to disable + boolean disabled = Boolean.getBoolean(KAFKA_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + if (!disabled) { + disabled = Boolean.getBoolean(KAFKA_COMMON_CONFIG_SYS_PROP_PREFIX + "offset.disabled"); + } + final var offsetEnabled = !disabled; + try (Logging.MDCCloseable mdc = Logging.withPrefixCloseable(MessageConsumerManager.LOG_PREFIX)) { + if (offsetEnabled) { + LOG.debug("Kafka offset provider is enabled."); + } else { + LOG.debug("Kafka offset provider is disabled."); } - return offsetEnabled; } - return kafkaSinkEnabled; + return offsetEnabled; } } } diff --git a/docs/modules/reference/pages/configuration/tuning-kafka.adoc b/docs/modules/reference/pages/configuration/tuning-kafka.adoc index bfb31d1b461a..4423ff1504fe 100644 --- a/docs/modules/reference/pages/configuration/tuning-kafka.adoc +++ b/docs/modules/reference/pages/configuration/tuning-kafka.adoc @@ -64,4 +64,16 @@ single-topic=false ---- org.opennms.core.ipc.rpc.kafka.single-topic=false ---- -Create this file if it does not already exist. \ No newline at end of file +Create this file if it does not already exist. + +== Disable the Kafka offset provider + +{page-component-title} Core automatically monitors consumer offsets on Kafka to provide lag related metrics. + +To disable the monitoring of these metrics you can set the following property: + +.Disable offset provider on `$\{OPENNMS_HOME}/etc/opennms.properties.d/kafka.properties` +[source, properties] +---- +org.opennms.core.ipc.kafka.offset.disabled=true +---- From 4a1d9fd1d59e73a1d736f494fe858bb72303fafa Mon Sep 17 00:00:00 2001 From: Jesse White Date: Tue, 11 Jul 2023 13:21:06 -0400 Subject: [PATCH 4/4] Update docs/modules/reference/pages/configuration/tuning-kafka.adoc Co-authored-by: Bonrob2 <59453630+Bonrob2@users.noreply.github.com> --- docs/modules/reference/pages/configuration/tuning-kafka.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/reference/pages/configuration/tuning-kafka.adoc b/docs/modules/reference/pages/configuration/tuning-kafka.adoc index 4423ff1504fe..ea17e0048158 100644 --- a/docs/modules/reference/pages/configuration/tuning-kafka.adoc +++ b/docs/modules/reference/pages/configuration/tuning-kafka.adoc @@ -68,7 +68,7 @@ Create this file if it does not already exist. == Disable the Kafka offset provider -{page-component-title} Core automatically monitors consumer offsets on Kafka to provide lag related metrics. +{page-component-title} Core automatically monitors consumer offsets on Kafka to provide lag-related metrics. To disable the monitoring of these metrics you can set the following property: