Implement exporting matches via prometheus metrics

api/src/main/java/dev/vrba/dubs/domain/Match.java

@@ -0,0 +1,58 @@
+package dev.vrba.dubs.domain;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.validation.constraints.Size;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.math.BigInteger;
+
+@Getter
+@Entity(name = "matches")
+@NoArgsConstructor
+@AllArgsConstructor
+public class Match {
+    @Id
+    @Column(name = "row_id")
+    private Integer id;
+
+    @Size(max = 64)
+    @Column(name = "pattern_name", length = 64)
+    private String patternName;
+
+    @Column(name = "pattern_points")
+    private Long patternPoints;
+
+    @Column(name = "pattern_is_rare")
+    private Boolean patternIsRare;
+
+    @Size(max = 32)
+    @Column(name = "user_id", length = 32)
+    private String userId;
+
+    @Size(max = 128)
+    @Column(name = "user_name", length = 128)
+    private String userName;
+
+    @Size(max = 32)
+    @Column(name = "channel_id", length = 32)
+    private String channelId;
+
+    @Size(max = 128)
+    @Column(name = "channel_name", length = 128)
+    private String channelName;
+
+    @Size(max = 32)
+    @Column(name = "guild_id", length = 32)
+    private String guildId;
+
+    @Size(max = 128)
+    @Column(name = "guild_name", length = 128)
+    private String guildName;
+
+    @Column(name = "count")
+    private BigInteger count;
+}

api/src/main/java/dev/vrba/dubs/metrics/MatchedPatternsMetric.java

@@ -0,0 +1,41 @@
+package dev.vrba.dubs.metrics;
+
+import dev.vrba.dubs.domain.Channel;
+import dev.vrba.dubs.domain.Guild;
+import dev.vrba.dubs.domain.Pattern;
+import dev.vrba.dubs.domain.User;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.Tags;
+import io.micronaut.core.annotation.NonNull;
+import jakarta.inject.Singleton;
+import lombok.RequiredArgsConstructor;
+
+@Singleton
+@RequiredArgsConstructor
+public class MatchedPatternsMetric {
+
+    @NonNull
+    private final MeterRegistry registry;
+
+    public void record(
+            @NonNull User user,
+            @NonNull Channel channel,
+            @NonNull Guild guild,
+            @NonNull Pattern pattern
+    ) {
+        registry.counter("discord.messages.patterns",
+                Tags.of(
+                        Tag.of("guild.id", guild.getId()),
+                        Tag.of("guild.name", guild.getName()),
+                        Tag.of("channel.id", channel.getId()),
+                        Tag.of("channel.name", channel.getName()),
+                        Tag.of("guild.id", guild.getId()),
+                        Tag.of("guild.name", guild.getName()),
+                        Tag.of("user.id", user.getId()),
+                        Tag.of("user.name", user.getName()),
+                        Tag.of("pattern", pattern.getName())
+                )
+        ).increment();
+    }
+}

api/src/main/java/dev/vrba/dubs/metrics/MatchesMetricConfiguration.java

@@ -0,0 +1,59 @@
+package dev.vrba.dubs.metrics;
+
+import dev.vrba.dubs.domain.Match;
+import dev.vrba.dubs.repository.MatchRepository;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.MultiGauge;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.Tags;
+import io.micronaut.core.annotation.NonNull;
+import io.micronaut.scheduling.annotation.Scheduled;
+import jakarta.inject.Singleton;
+
+import java.util.stream.Collectors;
+
+@Singleton
+public class MatchesMetricConfiguration {
+
+    @NonNull
+    private final MultiGauge gauge;
+
+    @NonNull
+    private final MatchRepository repository;
+
+    public MatchesMetricConfiguration(
+            final @NonNull MeterRegistry registry,
+            final @NonNull MatchRepository repository
+    ) {
+        this.gauge = MultiGauge.builder("pattern.matches").register(registry);
+        this.repository = repository;
+    }
+
+    @Scheduled(fixedRate = "PT1M")
+    public void refresh() {
+        gauge.register(
+                repository.findAll()
+                        .stream()
+                        .map(this::mapPatternToGaugeRow)
+                        .collect(Collectors.toList()),
+                true
+        );
+    }
+
+    private MultiGauge.Row<Number> mapPatternToGaugeRow(final @NonNull Match match) {
+        return MultiGauge.Row.of(
+                Tags.of(
+                        Tag.of("user.id", match.getUserId()),
+                        Tag.of("user.name", match.getUserName()),
+                        Tag.of("channel.id", match.getChannelId()),
+                        Tag.of("channel.name", match.getChannelName()),
+                        Tag.of("guild.id", match.getGuildId()),
+                        Tag.of("guild.name", match.getGuildName()),
+                        Tag.of("pattern.name", match.getPatternName()),
+                        Tag.of("pattern.rare", match.getPatternIsRare().toString()),
+                        Tag.of("pattern.points", match.getPatternPoints().toString())
+                ),
+                match.getCount()
+        );
+    }
+}

api/src/main/java/dev/vrba/dubs/metrics/ProcessedMessagesMetric.java

@@ -1,5 +1,8 @@
 package dev.vrba.dubs.metrics;
 
+import dev.vrba.dubs.domain.Channel;
+import dev.vrba.dubs.domain.Guild;
+import dev.vrba.dubs.domain.User;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
@@ -15,15 +18,18 @@ public class ProcessedMessagesMetric {
     private final MeterRegistry registry;
 
     public void record(
-            @NonNull String user,
-            @NonNull String channel,
-            @NonNull String guild
+            @NonNull User user,
+            @NonNull Channel channel,
+            @NonNull Guild guild
     ) {
-        registry.counter("discord.messages.processes",
+        registry.counter("discord.messages",
                 Tags.of(
-                        Tag.of("guild", guild),
-                        Tag.of("channel", channel),
-                        Tag.of("user", user)
+                        Tag.of("user.id", user.getId()),
+                        Tag.of("user.name", user.getName()),
+                        Tag.of("guild.id", guild.getId()),
+                        Tag.of("guild.name", guild.getName()),
+                        Tag.of("channel.id", channel.getId()),
+                        Tag.of("channel.name", channel.getName())
                 )
         ).increment();
     }

api/src/main/java/dev/vrba/dubs/repository/MatchRepository.java

@@ -0,0 +1,9 @@
+package dev.vrba.dubs.repository;
+
+import dev.vrba.dubs.domain.Match;
+import io.micronaut.data.annotation.Repository;
+import io.micronaut.data.repository.CrudRepository;
+
+@Repository
+public interface MatchRepository extends CrudRepository<Match, Integer> {
+}

api/src/main/java/dev/vrba/dubs/service/MessageProcessingService.java

@@ -4,6 +4,7 @@
 import dev.vrba.dubs.dto.ChannelDto;
 import dev.vrba.dubs.dto.GuildDto;
 import dev.vrba.dubs.dto.UserDto;
+import dev.vrba.dubs.metrics.MatchedPatternsMetric;
 import dev.vrba.dubs.metrics.ProcessedMessagesMetric;
 import dev.vrba.dubs.repository.MatchedPatternRepository;
 import io.micronaut.core.annotation.NonNull;
@@ -35,43 +36,38 @@ public class MessageProcessingService {
     private final PatternMatchingService patternMatchingService;
 
     @NonNull
-    private final ProcessedMessagesMetric metric;
+    private final ProcessedMessagesMetric processedMessageMetric;
 
     @NonNull
-    @Transactional
-    public List<Pattern> processMessage(
-            final @NonNull String message,
-            final @NonNull UserDto user,
-            final @NonNull ChannelDto channel,
-            final @NonNull GuildDto guild
-    ) {
-        metric.record(user.id(), channel.id(), guild.id());
+    private final MatchedPatternsMetric matchedPatternsMetric;
 
+    @NonNull
+    @Transactional
+    public List<Pattern> processMessage(final @NonNull String message, final @NonNull UserDto user, final @NonNull ChannelDto channel, final @NonNull GuildDto guild) {
         final Guild guildEntity = guildService.upsertGuild(guild.id(), guild.name(), guild.icon());
         final User userEntity = userService.upsertUser(user.id(), user.name(), user.avatar());
         final Channel channelEntity = channelService.upsertChannel(channel.id(), channel.name(), guildEntity);
         final List<Pattern> patterns = patternMatchingService.matchMessagePatterns(message);
 
+        processedMessageMetric.record(userEntity, channelEntity, guildEntity);
+
         if (!patterns.isEmpty()) {
-            final List<MatchedPattern> mappedPatterns = patterns.stream()
-                    .map(pattern -> new MatchedPattern(
-                            null,
-                            channelEntity.getId(),
-                            userEntity.getId(),
-                            pattern.getName(),
-                            pattern.getPoints(),
-                            pattern.isRare()
-                    ))
-                    .toList();
-
-            final BigInteger totalPoints = patterns.stream()
-                    .map(Pattern::getPoints)
-                    .map(BigInteger::valueOf)
-                    .reduce(BigInteger.ZERO, BigInteger::add);
+            final List<MatchedPattern> mappedPatterns = patterns.stream().map(pattern -> new MatchedPattern(null, channelEntity.getId(), userEntity.getId(), pattern.getName(), pattern.getPoints(), pattern.isRare())).toList();
+
+            final BigInteger totalPoints = patterns.stream().map(Pattern::getPoints).map(BigInteger::valueOf).reduce(BigInteger.ZERO, BigInteger::add);
 
             userService.incrementUserPoints(userEntity, totalPoints);
             matchedPatternRepository.saveAll(mappedPatterns);
 
+            patterns.forEach(pattern -> {
+                matchedPatternsMetric.record(
+                        userEntity,
+                        channelEntity,
+                        guildEntity,
+                        pattern
+                );
+            });
+
             return patterns;
         }
 

api/src/main/resources/application.yml

@@ -8,11 +8,20 @@ micronaut:
 
   metrics:
     enabled: true
+    binders:
+      jdbc.enabled: false
+      jvm.enabled: false
+      logback.enabled: false
+      processor.enabled: false
+      files.enabled: false
+      uptime.enabled: false
+      web.enabled: false
+      executor.enabled: false
     export:
       prometheus:
         enabled: true
-        descriptions: true
-        step: PT1M
+        descriptions: false
+        step: PT60S
 
   security:
     basic-auth:

api/src/main/resources/migrations/V6__add_row_id_to_matches.sql

@@ -0,0 +1,25 @@
+drop view if exists matches;
+create or replace view matches as
+(
+select row_number() over (order by users.id) as row_id,
+       pattern_name,
+       pattern_points,
+       pattern_is_rare,
+       users.id                             as user_id,
+       users.name                           as user_name,
+       channels.id                          as channel_id,
+       channels.name                        as channel_name,
+       guilds.id                            as guild_id,
+       guilds.name                          as guild_name,
+       count(*)                             as count
+from matched_patterns
+         left join users on matched_patterns.user_id = users.id
+         left join channels on matched_patterns.channel_id = channels.id
+         left join guilds on channels.guild_id = guilds.id
+group by users.id,
+         channels.id,
+         guilds.id,
+         pattern_name,
+         pattern_points,
+         pattern_is_rare
+    );

docker/prometheus.yml

@@ -1,20 +1,10 @@
-global:
-  scrape_interval: 15s
-  evaluation_interval: 30s
-  body_size_limit: 15MB
-  sample_limit: 1500
-  target_limit: 30
-  label_limit: 30
-  label_name_length_limit: 200
-  label_value_length_limit: 200
-
 scrape_configs:
   - job_name: prometheus
     honor_labels: true
     static_configs:
       - targets:
         - "host.docker.internal:8080"
-    scrape_interval: 50s
+    scrape_interval: 30s
     scrape_timeout: 5s
     body_size_limit: 10MB
     sample_limit: 1000