diff --git a/build.sbt b/build.sbt index 877d9834..9a36e57e 100755 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import common.mongoCampProject import dev.quadstingray.sbt.json.JsonFile -lazy val root = Project(id = "mc-server-parent", base = file(".")).aggregate(mcCli, mcLibrary, mcServer, mcTest, mcPluginRequestLogging) +lazy val root = Project(id = "mc-server-parent", base = file(".")).aggregate(mcCli, mcLibrary, mcServer, mcTest, mcPluginRequestLogging, mcPluginMicrometer) ThisBuild / scalaVersion := "2.13.12" @@ -20,3 +20,5 @@ lazy val mcCli = mongoCampProject("cli").dependsOn(mcLibrary).enablePlugins(Buil lazy val mcServer = mongoCampProject("server").enablePlugins(BuildInfoPlugin).dependsOn(mcLibrary, mcTest % Test) lazy val mcPluginRequestLogging = mongoCampProject("plugin-requestlogging").dependsOn(mcServer, mcTest % Test).enablePlugins(BuildInfoPlugin) + +lazy val mcPluginMicrometer = mongoCampProject("plugin-micrometer").dependsOn(mcServer, mcTest % Test).enablePlugins(BuildInfoPlugin) diff --git a/docs/plugins/list.md b/docs/plugins/list.md index 7be48e03..75cad1ac 100755 --- a/docs/plugins/list.md +++ b/docs/plugins/list.md @@ -4,9 +4,9 @@ title: List of Plugins # {{ $frontmatter.title }} ## Monitoring -| Name | Description | -|:----------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------:| -| [Micrometer](https://github.com/MongoCamp/mongocamp-micrometer-plugin) | Adds Functionality to use MicroMeter in other Plugins to your MongoCamp Instance and allow logging of Mircometer Registries to your Database | +| Name | Description | +|:---------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------:| +| [Micrometer](./mongocamp/micrometer.md) | Adds Functionality to use MicroMeter in other Plugins to your MongoCamp Instance and allow logging of Mircometer Registries to your Database | ## Logging | Name | Description | diff --git a/docs/plugins/mongocamp/micrometer.md b/docs/plugins/mongocamp/micrometer.md new file mode 100644 index 00000000..b592667d --- /dev/null +++ b/docs/plugins/mongocamp/micrometer.md @@ -0,0 +1,39 @@ +# MongoDB Micrometer Statistics + +This is Plugin adds Micrometer Functionality to a [MongoCamp Server](../../index.md). So other Plugins can use Micrometer or you can get your Micrometer Statistics to Database or call them by HTTP. + + +## Configuration + +### Dependency Setup +Add this to your [Plugin Modules](../../config/properties/plugins-module.md) to download the plugin from Maven Central. + + + + +### MongoCamp Configuration Settings +#### Persistence Plugin + +| Configuration | Description | Default | Type | +|--------------------------------|---------------------------------------------|:-------:|:--------:| +| LOGGING_METRICS_MONGODB_STEP | Duration between persist Metrics to MongoDB | 60m | Duration | +| LOGGING_METRICS_MONGODB_JVM | Should persist JVM Metrics to MongoDB | false | Boolean | +| LOGGING_METRICS_MONGODB_SYSTEM | Should persist SYSTEM Metrics to MongoDB | false | Boolean | +| LOGGING_METRICS_MONGODB_MONGO | Should persist MONGO Metrics to MongoDB | false | Boolean | +| LOGGING_METRICS_MONGODB_EVENT | Should persist EVENT Metrics to MongoDB | false | Boolean | + +#### MongoDB Metrics Plugin + +Plugin to monitor your MongoDB with your Application + +| Configuration | Description | Default | Type | +|--------------------------------|-------------------------------------------------|:-------:|:--------:| +| METRICS_MONGODB_DATABASE | Should monitor all collections in Database | false | Boolean | +| METRICS_MONGODB_COLLECTIONS | Should monitor explicit collections in Database | [] | [String] | +| METRICS_MONGODB_CONNECTIONS | Should monitor CONNECTIONS of your MongoDB | false | Boolean | +| METRICS_MONGODB_NETWORK | Should monitor NETWORK of your MongoDB | false | Boolean | +| METRICS_MONGODB_OPERATION | Should monitor OPERATION of your MongoDB | false | Boolean | +| METRICS_MONGODB_SERVER | Should monitor SERVER of your MongoDB | false | Boolean | +| METRICS_MONGODB_COMMAND | Should monitor COMMAND of your MongoDB | false | Boolean | +| METRICS_MONGODB_CONNECTIONPOOL | Should monitor CONNECTIONPOOL of your MongoDB | false | Boolean | + diff --git a/mongocamp-plugin-micrometer/build.sbt b/mongocamp-plugin-micrometer/build.sbt new file mode 100755 index 00000000..8e7904c0 --- /dev/null +++ b/mongocamp-plugin-micrometer/build.sbt @@ -0,0 +1,9 @@ +name := "mongocamp-plugin-micrometer" + +enablePlugins(BuildInfoPlugin) + +buildInfoPackage := "dev.mongocamp.server.plugin.micrometer" + +buildInfoOptions += BuildInfoOption.BuildTime + +libraryDependencies += "dev.mongocamp" %% "micrometer-mongodb" % "0.6.1" diff --git a/mongocamp-plugin-micrometer/src/main/resources/reference.conf b/mongocamp-plugin-micrometer/src/main/resources/reference.conf new file mode 100644 index 00000000..fbbcbe4a --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/resources/reference.conf @@ -0,0 +1,18 @@ +logging.metrics.mongodb { + jvm = false + system = false + mongo = false + event = false + step = 60m +} + +metrics.mongodb { + database = false + collections = [] + connections = false + network = false + operation = false + command = false + connectionpool = false + server = false +} \ No newline at end of file diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/MetricsConfiguration.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/MetricsConfiguration.scala new file mode 100644 index 00000000..6a0ae3e6 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/MetricsConfiguration.scala @@ -0,0 +1,80 @@ +package dev.mongocamp.server.plugins.monitoring + +import io.micrometer.core.instrument.binder.MeterBinder +import io.micrometer.core.instrument.binder.jvm._ +import io.micrometer.core.instrument.binder.system.{ DiskSpaceMetrics, FileDescriptorMetrics, ProcessorMetrics, UptimeMetrics } +import io.micrometer.core.instrument.{ MeterRegistry, Metrics } + +import java.io.File +import scala.collection.mutable.ArrayBuffer + +object MetricsConfiguration { + + private lazy val jvmMetricsRegistries: ArrayBuffer[MeterRegistry] = ArrayBuffer() + private lazy val systemMetricsRegistries: ArrayBuffer[MeterRegistry] = ArrayBuffer() + private lazy val mongoDbMetricsRegistries: ArrayBuffer[MeterRegistry] = ArrayBuffer() + private lazy val eventMetricsRegistries: ArrayBuffer[MeterRegistry] = ArrayBuffer() + + private lazy val jvmMeterBinder: ArrayBuffer[MeterBinder] = ArrayBuffer( + new ClassLoaderMetrics(), + new JvmMemoryMetrics(), + new JvmGcMetrics(), + new JvmHeapPressureMetrics(), + new JvmThreadMetrics(), + new JvmInfoMetrics(), + new JvmCompilationMetrics() + ) + private lazy val systemMeterBinder: ArrayBuffer[MeterBinder] = ArrayBuffer( + new DiskSpaceMetrics(new File("/")), + new FileDescriptorMetrics(), + new ProcessorMetrics(), + new UptimeMetrics() + ) + + private lazy val mongoDbMeterBinder: ArrayBuffer[MeterBinder] = ArrayBuffer() + private lazy val eventMeterBinder: ArrayBuffer[MeterBinder] = ArrayBuffer() + + def addJvmRegistry(registry: MeterRegistry): Unit = { + jvmMetricsRegistries += registry + Metrics.globalRegistry.add(registry) + } + + def addSystemRegistry(registry: MeterRegistry): Unit = { + systemMetricsRegistries += registry + Metrics.globalRegistry.add(registry) + } + + def addMongoRegistry(registry: MeterRegistry): Unit = { + mongoDbMetricsRegistries += registry + Metrics.globalRegistry.add(registry) + } + + def addEventRegistry(registry: MeterRegistry): Unit = { + eventMetricsRegistries += registry + Metrics.globalRegistry.add(registry) + } + + def getJvmMetricsRegistries = jvmMetricsRegistries.toList + + def getSystemMetricsRegistries = systemMetricsRegistries.toList + + def getMongoDbMetricsRegistries = mongoDbMetricsRegistries.toList + + def getEventMetricsRegistries = eventMetricsRegistries.toList + + def addJvmMeterBinder(meterBinder: MeterBinder): Unit = jvmMeterBinder += meterBinder + + def addSystemMeterBinder(meterBinder: MeterBinder): Unit = systemMeterBinder += meterBinder + + def addMongoDbBinder(meterBinder: MeterBinder): Unit = mongoDbMeterBinder += meterBinder + + def addEventMeterBinder(meterBinder: MeterBinder): Unit = eventMeterBinder += meterBinder + + def bindAll(): Unit = { + getJvmMetricsRegistries.foreach(r => jvmMeterBinder.foreach(_.bindTo(r))) + getSystemMetricsRegistries.foreach(r => systemMeterBinder.foreach(_.bindTo(r))) + getMongoDbMetricsRegistries.foreach(r => mongoDbMeterBinder.foreach(_.bindTo(r))) + getEventMetricsRegistries.foreach(r => eventMeterBinder.foreach(_.bindTo(r))) + } + +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/DefaultMetricsPlugin.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/DefaultMetricsPlugin.scala new file mode 100644 index 00000000..0912580b --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/DefaultMetricsPlugin.scala @@ -0,0 +1,21 @@ +package dev.mongocamp.server.plugins.monitoring.metrics + +import com.typesafe.scalalogging.LazyLogging +import dev.mongocamp.server.event.{Event, EventSystem} +import dev.mongocamp.server.plugin.ServerPlugin +import dev.mongocamp.server.plugins.monitoring.MetricsConfiguration +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import org.apache.pekko.actor.Props + +object DefaultMetricsPlugin extends ServerPlugin with LazyLogging { + override def activate(): Unit = { + MetricsConfiguration.addJvmRegistry(new SimpleMeterRegistry()) + MetricsConfiguration.addSystemRegistry(new SimpleMeterRegistry()) + MetricsConfiguration.addMongoRegistry(new SimpleMeterRegistry()) + MetricsConfiguration.addEventRegistry(new SimpleMeterRegistry()) + + val metricsLoggingActor = EventSystem.eventBusActorSystem.actorOf(Props(classOf[MetricsLoggingActor]), "metricsLoggingActor") + EventSystem.eventStream.subscribe(metricsLoggingActor, classOf[Event]) + } + +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/MetricsLoggingActor.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/MetricsLoggingActor.scala new file mode 100644 index 00000000..9f931ea4 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/metrics/MetricsLoggingActor.scala @@ -0,0 +1,40 @@ +package dev.mongocamp.server.plugins.monitoring.metrics + +import com.typesafe.scalalogging.LazyLogging +import dev.mongocamp.server.event.Event +import dev.mongocamp.server.event.http.HttpRequestCompletedEvent +import dev.mongocamp.server.plugins.monitoring.MetricsConfiguration +import org.apache.pekko.actor.Actor + +import java.time.Duration + +class MetricsLoggingActor extends Actor with LazyLogging { + def receive: Receive = { + + case "info" => + logger.info(this.getClass.getSimpleName) + + case e: HttpRequestCompletedEvent => + val eventNameArray = e.getClass.getName.split('.') + val indexOfEvent = eventNameArray.indexOf("event") + val pathArray = eventNameArray.slice(indexOfEvent, eventNameArray.length) + val regex = "([a-z])([A-Z]+)" + val replacement = "$1.$2" + val metricsName = s"${pathArray.mkString(".")}".replaceAll("Event", "").replaceAll(regex, replacement).toLowerCase() + MetricsConfiguration.getEventMetricsRegistries.foreach(_.timer(metricsName).record(Duration.ofMillis(e.duration.getMillis))) + MetricsConfiguration.getEventMetricsRegistries.foreach( + _.timer(s"$metricsName.${e.controller.toLowerCase()}.${e.controllerMethod.toLowerCase()}").record(Duration.ofMillis(e.duration.getMillis)) + ) + + case e: Event => + val eventNameArray = e.getClass.getName.split('.') + val indexOfEvent = eventNameArray.indexOf("event") + val pathArray = eventNameArray.slice(indexOfEvent, eventNameArray.length) + val regex = "([a-z])([A-Z]+)" + val replacement = "$1.$2" + val metricsName = s"${pathArray.mkString(".")}".replaceAll("Event", "").replaceAll(regex, replacement).toLowerCase() + MetricsConfiguration.getEventMetricsRegistries.foreach(_.summary(metricsName).record(1)) + + } + +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Measurement.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Measurement.scala new file mode 100644 index 00000000..4ec93cf8 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Measurement.scala @@ -0,0 +1,3 @@ +package dev.mongocamp.server.plugins.monitoring.model + +case class Measurement(statisticType: String, value: Double) diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Metric.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Metric.scala new file mode 100644 index 00000000..845e47d0 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/model/Metric.scala @@ -0,0 +1,42 @@ +package dev.mongocamp.server.plugins.monitoring.model + +import com.typesafe.scalalogging.LazyLogging +import io.micrometer.core.instrument.Meter + +import scala.jdk.CollectionConverters._ + +case class Metric(name: String, metricsType: String, description: String, baseUnit: String, measurements: List[Measurement]) + +object Metric extends LazyLogging { + def apply(meter: Meter): Metric = { + Metric( + validStringValue(meter.getId.getName), + validStringValue(meter.getId.getType.name()), + validStringValue(meter.getId.getDescription), + validStringValue(meter.getId.getBaseUnit), + meter + .measure() + .asScala + .toList + .map(v => Measurement(validStringValue(v.getStatistic.name()), validDoubleValue(v.getValue))) + ) + } + + private def validStringValue(value: String): String = { + if (value == null) { + "" + } + else { + value + } + } + + private def validDoubleValue(value: Double): Double = { + if (value == null) { + 0.0 + } + else { + value + } + } +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MetricsMongoDBPersistenceLoggingPlugin.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MetricsMongoDBPersistenceLoggingPlugin.scala new file mode 100644 index 00000000..34b04057 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MetricsMongoDBPersistenceLoggingPlugin.scala @@ -0,0 +1,44 @@ +package dev.mongocamp.server.plugins.monitoring.mongodb + +import com.typesafe.scalalogging.LazyLogging +import dev.mongocamp.micrometer.mongodb.registry.MongoStepMeterRegistry +import dev.mongocamp.server.database.MongoDatabase +import dev.mongocamp.server.model.MongoCampConfiguration +import dev.mongocamp.server.plugin.ServerPlugin +import dev.mongocamp.server.plugins.monitoring.MetricsConfiguration +import dev.mongocamp.server.service.ConfigurationService + +import scala.concurrent.duration.Duration + +object MetricsMongoDBPersistenceLoggingPlugin extends ServerPlugin with LazyLogging { + private val ConfKeyMicrometerStep = "LOGGING_METRICS_MONGODB_STEP" + private val ConfKeyLoggingJvmToMongoDb = "LOGGING_METRICS_MONGODB_JVM" + private val ConfKeyLoggingSystemToMongoDb = "LOGGING_METRICS_MONGODB_SYSTEM" + private val ConfKeyLoggingMongoToMongoDb = "LOGGING_METRICS_MONGODB_MONGO" + private val ConfKeyLoggingEventToMongoDb = "LOGGING_METRICS_MONGODB_EVENT" + + override def activate(): Unit = { + + ConfigurationService.registerConfig(ConfKeyMicrometerStep, MongoCampConfiguration.confTypeDuration) + ConfigurationService.registerConfig(ConfKeyLoggingJvmToMongoDb, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyLoggingSystemToMongoDb, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyLoggingMongoToMongoDb, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyLoggingEventToMongoDb, MongoCampConfiguration.confTypeBoolean) + + val stepDuration = ConfigurationService.getConfigValue[Duration](ConfKeyMicrometerStep) + val configMap = Map("step" -> s"${stepDuration.toMillis}ms") + if (ConfigurationService.getConfigValue[Boolean](ConfKeyLoggingJvmToMongoDb)) { + MetricsConfiguration.addJvmRegistry(MongoStepMeterRegistry(MongoDatabase.databaseProvider.dao("monitoring_jvm"), configMap)) + } + if (ConfigurationService.getConfigValue[Boolean](ConfKeyLoggingSystemToMongoDb)) { + MetricsConfiguration.addSystemRegistry(MongoStepMeterRegistry(MongoDatabase.databaseProvider.dao("monitoring_system"), configMap)) + } + if (ConfigurationService.getConfigValue[Boolean](ConfKeyLoggingMongoToMongoDb)) { + MetricsConfiguration.addMongoRegistry(MongoStepMeterRegistry(MongoDatabase.databaseProvider.dao("monitoring_mongo_db"), configMap)) + } + if (ConfigurationService.getConfigValue[Boolean](ConfKeyLoggingEventToMongoDb)) { + MetricsConfiguration.addEventRegistry(MongoStepMeterRegistry(MongoDatabase.databaseProvider.dao("monitoring_event"), configMap)) + } + } + +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MongoDbMetricsPlugin.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MongoDbMetricsPlugin.scala new file mode 100644 index 00000000..2cd2339c --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/mongodb/MongoDbMetricsPlugin.scala @@ -0,0 +1,91 @@ +package dev.mongocamp.server.plugins.monitoring.mongodb + +import com.typesafe.scalalogging.LazyLogging +import dev.mongocamp.micrometer.mongodb.binder._ +import dev.mongocamp.server.Server +import dev.mongocamp.server.database.MongoDatabase +import dev.mongocamp.server.model.MongoCampConfiguration +import dev.mongocamp.server.plugin.ServerPlugin +import dev.mongocamp.server.plugins.monitoring.MetricsConfiguration +import dev.mongocamp.server.service.ConfigurationService +import io.micrometer.core.instrument.binder.mongodb.{ MongoMetricsCommandListener, MongoMetricsConnectionPoolListener } + +import scala.collection.mutable.ArrayBuffer + +object MongoDbMetricsPlugin extends ServerPlugin with LazyLogging { + + private val ConfKeyMetricsDatabase = "METRICS_MONGODB_DATABASE" + private val ConfKeyMetricsCollectionList = "METRICS_MONGODB_COLLECTIONS" + private val ConfKeyMetricsConnections = "METRICS_MONGODB_CONNECTIONS" + private val ConfKeyMetricsNetwork = "METRICS_MONGODB_NETWORK" + private val ConfKeyMetricsOperation = "METRICS_MONGODB_OPERATION" + private val ConfKeyMetricsServer = "METRICS_MONGODB_SERVER" + private val ConfKeyMetricsCommand = "METRICS_MONGODB_COMMAND" + private val ConfKeyMetricsConnectionpool = "METRICS_MONGODB_CONNECTIONPOOL" + + override def activate(): Unit = { + ConfigurationService.registerConfig(ConfKeyMetricsDatabase, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsConnections, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsNetwork, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsOperation, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsServer, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsCollectionList, MongoCampConfiguration.confTypeStringList) + ConfigurationService.registerConfig(ConfKeyMetricsCommand, MongoCampConfiguration.confTypeBoolean) + ConfigurationService.registerConfig(ConfKeyMetricsConnectionpool, MongoCampConfiguration.confTypeBoolean) + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsDatabase)) { + MetricsConfiguration.addMongoDbBinder(DatabaseMetrics(MongoDatabase.databaseProvider.database())) + } + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsConnections)) { + MetricsConfiguration.addMongoDbBinder(ConnectionsMetrics(MongoDatabase.databaseProvider.database())) + } + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsNetwork)) { + MetricsConfiguration.addMongoDbBinder(NetworkMetrics(MongoDatabase.databaseProvider.database())) + } + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsOperation)) { + MetricsConfiguration.addMongoDbBinder(OperationMetrics(MongoDatabase.databaseProvider.database())) + } + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsServer)) { + MetricsConfiguration.addMongoDbBinder(ServerMetrics(MongoDatabase.databaseProvider.database())) + } + + val registerFunctionsList: ArrayBuffer[() => Unit] = ArrayBuffer() + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsCommand)) { + registerFunctionsList += { () => + { + MetricsConfiguration.getMongoDbMetricsRegistries.foreach(r => { + val commandListener = new MongoMetricsCommandListener(r) + MongoDatabase.registerCommandListener(commandListener) + }) + } + } + } + + if (ConfigurationService.getConfigValue[Boolean](ConfKeyMetricsConnectionpool)) { + registerFunctionsList += { () => + { + MetricsConfiguration.getMongoDbMetricsRegistries.foreach(r => { + val commandListener = new MongoMetricsConnectionPoolListener(r) + MongoDatabase.registerConnectionPoolListener(commandListener) + }) + } + } + } + + ConfigurationService + .getConfigValue[List[String]](ConfKeyMetricsCollectionList) + .foreach(collection => MetricsConfiguration.addMongoDbBinder(CollectionMetrics(MongoDatabase.databaseProvider.database(), collection))) + + Server.registerAfterStartCallBack(() => { + MetricsConfiguration.bindAll() + registerFunctionsList.foreach(f => f()) + MongoDatabase.createNewDatabaseProvider() + }) + } + +} diff --git a/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/routes/MetricsRoutes.scala b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/routes/MetricsRoutes.scala new file mode 100644 index 00000000..49a2d9b1 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/main/scala/dev/mongocamp/server/plugins/monitoring/routes/MetricsRoutes.scala @@ -0,0 +1,103 @@ +package dev.mongocamp.server.plugins.monitoring.routes + +import dev.mongocamp.server.exception.ErrorDescription +import dev.mongocamp.server.plugin.RoutesPlugin +import dev.mongocamp.server.plugins.monitoring.MetricsConfiguration +import dev.mongocamp.server.plugins.monitoring.model.Metric +import dev.mongocamp.server.route.BaseRoute +import io.circe.generic.auto._ +import sttp.capabilities.WebSockets +import sttp.capabilities.pekko.PekkoStreams +import sttp.model.{Method, StatusCode} +import sttp.tapir._ +import sttp.tapir.json.circe.jsonBody +import sttp.tapir.server.ServerEndpoint + +import scala.concurrent.Future +import scala.jdk.CollectionConverters._ + +object MetricsRoutes extends BaseRoute with RoutesPlugin { + private val applicationApiBaseEndpoint = adminEndpoint.tag("Application") + + val jvmMetricsRoutes = applicationApiBaseEndpoint + .in("system" / "monitoring" / "jvm") + .out(jsonBody[List[Metric]]) + .summary("JVM Metrics") + .description("Returns the JVM Metrics of the running MongoCamp Application") + .method(Method.GET) + .name("jvmMetrics") + .serverLogic(_ => _ => jvmMetrics()) + + def jvmMetrics(): Future[Either[(StatusCode, ErrorDescription, ErrorDescription), List[Metric]]] = { + Future.successful(Right({ + val meters = MetricsConfiguration.getJvmMetricsRegistries.head.getMeters.asScala.toList + meters.map(Metric(_)) + })) + } + + val systemMetricsRoutes = applicationApiBaseEndpoint + .in("system" / "monitoring" / "system") + .out(jsonBody[List[Metric]]) + .summary("System Metrics") + .description("Returns the Metrics of the MongoCamp System") + .method(Method.GET) + .name("systemMetrics") + .serverLogic(_ => _ => systemMetrics()) + + def systemMetrics(): Future[Either[(StatusCode, ErrorDescription, ErrorDescription), List[Metric]]] = { + Future.successful(Right({ + val meters = MetricsConfiguration.getSystemMetricsRegistries.head.getMeters.asScala.toList + meters.map(Metric(_)) + })) + } + + val mongoDbMetricsRoutes = applicationApiBaseEndpoint + .in("system" / "monitoring" / "mongodb") + .out(jsonBody[List[Metric]]) + .summary("MongoDb Metrics") + .description("Returns the MongoDB Metrics of the running MongoCamp Application") + .method(Method.GET) + .name("mongoDbMetrics") + .serverLogic(_ => _ => mongoDbMetrics()) + + def mongoDbMetrics(): Future[Either[(StatusCode, ErrorDescription, ErrorDescription), List[Metric]]] = { + Future.successful(Right({ + val meters = MetricsConfiguration.getMongoDbMetricsRegistries.head.getMeters.asScala.toList + meters.map(Metric(_)) + })) + } + + val eventMetricsRoutes = applicationApiBaseEndpoint + .in("system" / "monitoring" / "events") + .out(jsonBody[List[Metric]]) + .summary("Event Metrics") + .description("Returns the Metrics of events of the running MongoCamp Application.") + .method(Method.GET) + .name("eventMetrics") + .serverLogic(_ => _ => eventMetrics()) + + def eventMetrics(): Future[Either[(StatusCode, ErrorDescription, ErrorDescription), List[Metric]]] = { + Future.successful(Right({ + val meters = MetricsConfiguration.getEventMetricsRegistries.head.getMeters.asScala.toList + meters.map(Metric(_)) + })) + } + + override def endpoints = { + var endpoints: List[ServerEndpoint[PekkoStreams with WebSockets, Future]] = List() + if (MetricsConfiguration.getJvmMetricsRegistries.nonEmpty) { + endpoints ++= List(jvmMetricsRoutes) + } + if (MetricsConfiguration.getSystemMetricsRegistries.nonEmpty) { + endpoints ++= List(systemMetricsRoutes) + } + if (MetricsConfiguration.getMongoDbMetricsRegistries.nonEmpty) { + endpoints ++= List(mongoDbMetricsRoutes) + } + if (MetricsConfiguration.getEventMetricsRegistries.nonEmpty) { + endpoints ++= List(eventMetricsRoutes) + } + endpoints + } + +} diff --git a/mongocamp-plugin-micrometer/src/test/resources/application.conf b/mongocamp-plugin-micrometer/src/test/resources/application.conf new file mode 100644 index 00000000..37b5e6e6 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/resources/application.conf @@ -0,0 +1,20 @@ +logging.metrics.mongodb { + jvm = true + system = true + mongo = true + event = true + step = 500ms +} + +metrics.mongodb { + database = true + collections = [] + connections = true + network = true + operation = true + command = true + connectionpool = true + server = true +} + +docs.swagger=true \ No newline at end of file diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/api/MetricsApi.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/api/MetricsApi.scala new file mode 100644 index 00000000..048cbf54 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/api/MetricsApi.scala @@ -0,0 +1,100 @@ +/** + * mongocamp-server + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.4.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ +package dev.mongocamp.server.client.api + +import dev.mongocamp.server.test.TestServer +import dev.mongocamp.server.test.client.core.JsonSupport._ +import dev.mongocamp.server.test.client.model.Metric +import sttp.client3._ +import sttp.model.Method + +object MetricsApi { + +def apply(baseUrl: String = TestServer.serverBaseUrl) = new MetricsApi(baseUrl) +} + +class MetricsApi(baseUrl: String) { + + /** Returns the JVM Metrics of the running MongoCamp Application + * + * Expected answers: code 200 : Seq[Metric] () code 0 : ErrorDescription () Headers : x-error-code - Error Code x-error-message - Message of the + * MongoCampException x-error-additional-info - Additional information for the MongoCampException + * + * Available security schemes: httpAuth1 (http) httpAuth (http) apiKeyAuth (apiKey) + */ + def jvmMetrics(username: String, password: String, bearerToken: String, apiKey: String)() = + basicRequest + .method(Method.GET, uri"$baseUrl/system/monitoring/jvm") + .contentType("application/json") + .auth + .basic(username, password) + .auth + .bearer(bearerToken) + .header("X-AUTH-APIKEY", apiKey) + .response(asJson[Seq[Metric]]) + + /** Returns the Metrics of events of the running MongoCamp Application. + * + * Expected answers: code 200 : Seq[Metric] () code 0 : ErrorDescription () Headers : x-error-code - Error Code x-error-message - Message of the + * MongoCampException x-error-additional-info - Additional information for the MongoCampException + * + * Available security schemes: httpAuth1 (http) httpAuth (http) apiKeyAuth (apiKey) + */ + def eventMetrics(username: String, password: String, bearerToken: String, apiKey: String)() = + basicRequest + .method(Method.GET, uri"$baseUrl/system/monitoring/events") + .contentType("application/json") + .auth + .basic(username, password) + .auth + .bearer(bearerToken) + .header("X-AUTH-APIKEY", apiKey) + .response(asJson[Seq[Metric]]) + + + /** Returns the MongoDB Metrics of the running MongoCamp Application + * + * Expected answers: code 200 : Seq[Metric] () code 0 : ErrorDescription () Headers : x-error-code - Error Code x-error-message - Message of the + * MongoCampException x-error-additional-info - Additional information for the MongoCampException + * + * Available security schemes: httpAuth1 (http) httpAuth (http) apiKeyAuth (apiKey) + */ + def mongoDbMetrics(username: String, password: String, bearerToken: String, apiKey: String)() = + basicRequest + .method(Method.GET, uri"$baseUrl/system/monitoring/mongodb") + .contentType("application/json") + .auth + .basic(username, password) + .auth + .bearer(bearerToken) + .header("X-AUTH-APIKEY", apiKey) + .response(asJson[Seq[Metric]]) + + /** Returns the Metrics of the MongoCamp System + * + * Expected answers: code 200 : Seq[Metric] () code 0 : ErrorDescription () Headers : x-error-code - Error Code x-error-message - Message of the + * MongoCampException x-error-additional-info - Additional information for the MongoCampException + * + * Available security schemes: httpAuth1 (http) httpAuth (http) apiKeyAuth (apiKey) + */ + def systemMetrics(username: String, password: String, bearerToken: String, apiKey: String)() = + basicRequest + .method(Method.GET, uri"$baseUrl/system/monitoring/system") + .contentType("application/json") + .auth + .basic(username, password) + .auth + .bearer(bearerToken) + .header("X-AUTH-APIKEY", apiKey) + .response(asJson[Seq[Metric]]) + +} diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Measurement.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Measurement.scala new file mode 100644 index 00000000..47138dbd --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Measurement.scala @@ -0,0 +1,18 @@ +/** + * mongocamp-server + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.4.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ +package dev.mongocamp.server.client.model + +case class Measurement( + statisticType: String, + value: Double +) + diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Metric.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Metric.scala new file mode 100644 index 00000000..722c413a --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/client/model/Metric.scala @@ -0,0 +1,21 @@ +/** + * mongocamp-server + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 1.4.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ +package dev.mongocamp.server.client.model + +case class Metric( + name: String, + metricsType: String, + description: String, + baseUnit: String, + measurements: Option[Seq[Measurement]] = None +) + diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/InformationSuite.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/InformationSuite.scala new file mode 100644 index 00000000..68d648f8 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/InformationSuite.scala @@ -0,0 +1,19 @@ +package dev.mongocamp.server.plugins.monitoring + +import dev.mongocamp.server.BuildInfo +import dev.mongocamp.server.test.MongoCampBaseServerSuite +import dev.mongocamp.server.test.client.api.InformationApi +import org.joda.time.DateTime + +class InformationSuite extends MicrometerBaseServerSuite { + + lazy val informationApi: InformationApi = InformationApi() + + test("check version by api request") { + val version = executeRequestToResponse(informationApi.version()) + assertEquals(version.name, BuildInfo.name) + assertEquals(version.version, BuildInfo.version) + assertEquals(version.builtAt, new DateTime(BuildInfo.builtAtMillis)) + } + +} diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MetricsSuite.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MetricsSuite.scala new file mode 100644 index 00000000..5b87ffa2 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MetricsSuite.scala @@ -0,0 +1,32 @@ +package dev.mongocamp.server.plugins.monitoring + +import dev.mongocamp.server.client.api.MetricsApi +import dev.mongocamp.server.test.MongoCampBaseServerSuite +import dev.mongocamp.server.test.client.api.ApplicationApi + +class MetricsSuite extends MicrometerBaseServerSuite { + + lazy val applicationApi: ApplicationApi = ApplicationApi() + lazy val metricsApi: MetricsApi = MetricsApi() + + test("Show all JVM Metrics") { + val jvmMetrics = executeRequestToResponse(metricsApi.jvmMetrics("", "", adminBearerToken, "")()) + assertEquals(jvmMetrics.size > 40, true) + } + + test("Show all System Metrics") { + val systemMetrics = executeRequestToResponse(metricsApi.systemMetrics("", "", adminBearerToken, "")()) + assertEquals(systemMetrics.size, 10) + } + + test("Show all MongoDb Metrics") { + val mongoDbMetrics = executeRequestToResponse(metricsApi.mongoDbMetrics("", "", adminBearerToken, "")()) + assertEquals(mongoDbMetrics.size > 25, true) + } + + test("Show all Event Metrics") { + val eventMetrics = executeRequestToResponse(metricsApi.eventMetrics("", "", adminBearerToken, "")()) + assert(eventMetrics.size > 5, "More than 10 event metrics") + } + +} diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerBaseServerSuite.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerBaseServerSuite.scala new file mode 100644 index 00000000..141ec908 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerBaseServerSuite.scala @@ -0,0 +1,7 @@ +package dev.mongocamp.server.plugins.monitoring + +import dev.mongocamp.server.test.MongoCampBaseServerSuite + +abstract class MicrometerBaseServerSuite extends MongoCampBaseServerSuite{ + MicrometerServer.micrometerBasedServerStarted() +} diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerServer.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerServer.scala new file mode 100644 index 00000000..98779b2b --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/MicrometerServer.scala @@ -0,0 +1,13 @@ +package dev.mongocamp.server.plugins.monitoring + +import dev.mongocamp.server.test.TestServer.ignoreRunningInstanceAndReset + +object MicrometerServer { + private var micrometerStarted = false + def micrometerBasedServerStarted(): Unit = { + if (!micrometerStarted) { + ignoreRunningInstanceAndReset() + micrometerStarted = true + } + } +} diff --git a/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/PersistenceSuite.scala b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/PersistenceSuite.scala new file mode 100644 index 00000000..12094375 --- /dev/null +++ b/mongocamp-plugin-micrometer/src/test/scala/dev/mongocamp/server/plugins/monitoring/PersistenceSuite.scala @@ -0,0 +1,64 @@ +package dev.mongocamp.server.plugins.monitoring + +import dev.mongocamp.driver.mongodb._ +import dev.mongocamp.server.client.api.MetricsApi +import dev.mongocamp.server.database.MongoDatabase +import dev.mongocamp.server.test.MongoCampBaseServerSuite +import dev.mongocamp.server.test.client.api.InformationApi +import org.joda.time.DateTime +class PersistenceSuite extends MicrometerBaseServerSuite { + + lazy val metricsApi: MetricsApi = MetricsApi() + lazy val informationApi: InformationApi = InformationApi() + + override def beforeEach(context: BeforeEach): Unit = { + val jvmMetrics = executeRequestToResponse(metricsApi.jvmMetrics("", "", adminBearerToken, "")()) + val systemMetrics = executeRequestToResponse(metricsApi.systemMetrics("", "", adminBearerToken, "")()) + val mongoDbMetrics = executeRequestToResponse(metricsApi.mongoDbMetrics("", "", adminBearerToken, "")()) + val eventMetrics = executeRequestToResponse(metricsApi.eventMetrics("", "", adminBearerToken, "")()) + val version = executeRequestToResponse(informationApi.version()) + val startTime = new DateTime() + while (startTime.plusSeconds(15).isBeforeNow) { + // wait for collect metrics + } + } + + test("Check JVM Metrics at Database") { + val dbResponse = MongoDatabase.databaseProvider.dao("monitoring_jvm").find(sort = Map("date" -> -1)).results() + assertEquals(dbResponse.nonEmpty, true) + val document = dbResponse.head + assertEquals(document.getDoubleValue("jvm_threads_live.value") > 50.0, true) + assertEquals(document.getStringValue("jvm_buffer_memory_used.metricType"), "gauge") + } + + test("Check System Metrics at Database") { + val dbResponse = MongoDatabase.databaseProvider.dao("monitoring_system").find(sort = Map("date" -> -1)).results() + assertEquals(dbResponse.nonEmpty, true) + val document = dbResponse.head + assertEquals(document.getDoubleValue("process_files_open.value") > 0.0, true) + assertEquals(document.getDoubleValue("process_files_open.value") > 5.0, true) + assertEquals(document.getStringValue("process_cpu_usage.metricType"), "gauge") + assertEquals(document.getStringValue("disk_total.metricType"), "gauge") + } + + test("Check MongoDb Metrics at Database") { + val dbResponse = MongoDatabase.databaseProvider.dao("monitoring_mongo_db").find(sort = Map("date" -> -1)).results() + assertEquals(dbResponse.nonEmpty, true) + val document = dbResponse.head + assertEquals(document.getDoubleValue("mongodb_server_status_operations_query.value") > 50.0, true) + assertEquals(document.getStringValue("mongodb_server_status_operations_query.metricType"), "gauge") + assertEquals(document.getStringValue("mongodb_collection_test_mc_users_avgObjSize.metricType"), "gauge") + assertEquals(document.getStringValue("mongodb_driver_pool_size.metricType"), "gauge") + assertEquals(document.getDoubleValue("mongodb_driver_pool_size.value") >= 1.0, true) + assertEquals(document.getStringValue("mongodb_driver_commands.metricType"), "timer") + } + + test("Check Event Metrics at Database") { + val dbResponse = MongoDatabase.databaseProvider.dao("monitoring_event").find(sort = Map("date" -> -1)).results() + assertEquals(dbResponse.nonEmpty, true) + val document = dbResponse.head + assertEquals(document.getStringValue("event_http_http_request_start.metricType"), "distribution_summary") + assertEquals(document.getStringValue("event_http_http_request_completed.metricType"), "timer") + } + +} diff --git a/mongocamp-test-server/src/main/scala/dev/mongocamp/server/test/TestServer.scala b/mongocamp-test-server/src/main/scala/dev/mongocamp/server/test/TestServer.scala index fcaddc97..361018cd 100755 --- a/mongocamp-test-server/src/main/scala/dev/mongocamp/server/test/TestServer.scala +++ b/mongocamp-test-server/src/main/scala/dev/mongocamp/server/test/TestServer.scala @@ -64,6 +64,8 @@ object TestServer extends LazyLogging { _serverRunning } + def ignoreRunningInstanceAndReset(): Unit = _serverRunning = false + def serverBaseUrl: String = { "http://%s:%s".format(server.interface, server.port) }