diff --git a/api/src/main/java/dev/vrba/dubs/domain/Match.java b/api/src/main/java/dev/vrba/dubs/domain/Match.java new file mode 100644 index 0000000..6fd94ea --- /dev/null +++ b/api/src/main/java/dev/vrba/dubs/domain/Match.java @@ -0,0 +1,58 @@ +package dev.vrba.dubs.domain; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.validation.constraints.Size; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.math.BigInteger; + +@Getter +@Entity(name = "matches") +@NoArgsConstructor +@AllArgsConstructor +public class Match { + @Id + @Column(name = "row_id") + private Integer id; + + @Size(max = 64) + @Column(name = "pattern_name", length = 64) + private String patternName; + + @Column(name = "pattern_points") + private Long patternPoints; + + @Column(name = "pattern_is_rare") + private Boolean patternIsRare; + + @Size(max = 32) + @Column(name = "user_id", length = 32) + private String userId; + + @Size(max = 128) + @Column(name = "user_name", length = 128) + private String userName; + + @Size(max = 32) + @Column(name = "channel_id", length = 32) + private String channelId; + + @Size(max = 128) + @Column(name = "channel_name", length = 128) + private String channelName; + + @Size(max = 32) + @Column(name = "guild_id", length = 32) + private String guildId; + + @Size(max = 128) + @Column(name = "guild_name", length = 128) + private String guildName; + + @Column(name = "count") + private BigInteger count; +} diff --git a/api/src/main/java/dev/vrba/dubs/metrics/MatchedPatternsMetric.java b/api/src/main/java/dev/vrba/dubs/metrics/MatchedPatternsMetric.java new file mode 100644 index 0000000..3c46c29 --- /dev/null +++ b/api/src/main/java/dev/vrba/dubs/metrics/MatchedPatternsMetric.java @@ -0,0 +1,41 @@ +package dev.vrba.dubs.metrics; + +import dev.vrba.dubs.domain.Channel; +import dev.vrba.dubs.domain.Guild; +import dev.vrba.dubs.domain.Pattern; +import dev.vrba.dubs.domain.User; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micronaut.core.annotation.NonNull; +import jakarta.inject.Singleton; +import lombok.RequiredArgsConstructor; + +@Singleton +@RequiredArgsConstructor +public class MatchedPatternsMetric { + + @NonNull + private final MeterRegistry registry; + + public void record( + @NonNull User user, + @NonNull Channel channel, + @NonNull Guild guild, + @NonNull Pattern pattern + ) { + registry.counter("discord.messages.patterns", + Tags.of( + Tag.of("guild.id", guild.getId()), + Tag.of("guild.name", guild.getName()), + Tag.of("channel.id", channel.getId()), + Tag.of("channel.name", channel.getName()), + Tag.of("guild.id", guild.getId()), + Tag.of("guild.name", guild.getName()), + Tag.of("user.id", user.getId()), + Tag.of("user.name", user.getName()), + Tag.of("pattern", pattern.getName()) + ) + ).increment(); + } +} diff --git a/api/src/main/java/dev/vrba/dubs/metrics/MatchesMetricConfiguration.java b/api/src/main/java/dev/vrba/dubs/metrics/MatchesMetricConfiguration.java new file mode 100644 index 0000000..b62500f --- /dev/null +++ b/api/src/main/java/dev/vrba/dubs/metrics/MatchesMetricConfiguration.java @@ -0,0 +1,59 @@ +package dev.vrba.dubs.metrics; + +import dev.vrba.dubs.domain.Match; +import dev.vrba.dubs.repository.MatchRepository; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.MultiGauge; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micronaut.core.annotation.NonNull; +import io.micronaut.scheduling.annotation.Scheduled; +import jakarta.inject.Singleton; + +import java.util.stream.Collectors; + +@Singleton +public class MatchesMetricConfiguration { + + @NonNull + private final MultiGauge gauge; + + @NonNull + private final MatchRepository repository; + + public MatchesMetricConfiguration( + final @NonNull MeterRegistry registry, + final @NonNull MatchRepository repository + ) { + this.gauge = MultiGauge.builder("pattern.matches").register(registry); + this.repository = repository; + } + + @Scheduled(fixedRate = "PT1M") + public void refresh() { + gauge.register( + repository.findAll() + .stream() + .map(this::mapPatternToGaugeRow) + .collect(Collectors.toList()), + true + ); + } + + private MultiGauge.Row mapPatternToGaugeRow(final @NonNull Match match) { + return MultiGauge.Row.of( + Tags.of( + Tag.of("user.id", match.getUserId()), + Tag.of("user.name", match.getUserName()), + Tag.of("channel.id", match.getChannelId()), + Tag.of("channel.name", match.getChannelName()), + Tag.of("guild.id", match.getGuildId()), + Tag.of("guild.name", match.getGuildName()), + Tag.of("pattern.name", match.getPatternName()), + Tag.of("pattern.rare", match.getPatternIsRare().toString()), + Tag.of("pattern.points", match.getPatternPoints().toString()) + ), + match.getCount() + ); + } +} diff --git a/api/src/main/java/dev/vrba/dubs/metrics/ProcessedMessagesMetric.java b/api/src/main/java/dev/vrba/dubs/metrics/ProcessedMessagesMetric.java index 83a0392..7f8ac9d 100644 --- a/api/src/main/java/dev/vrba/dubs/metrics/ProcessedMessagesMetric.java +++ b/api/src/main/java/dev/vrba/dubs/metrics/ProcessedMessagesMetric.java @@ -1,5 +1,8 @@ package dev.vrba.dubs.metrics; +import dev.vrba.dubs.domain.Channel; +import dev.vrba.dubs.domain.Guild; +import dev.vrba.dubs.domain.User; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; @@ -15,15 +18,18 @@ public class ProcessedMessagesMetric { private final MeterRegistry registry; public void record( - @NonNull String user, - @NonNull String channel, - @NonNull String guild + @NonNull User user, + @NonNull Channel channel, + @NonNull Guild guild ) { - registry.counter("discord.messages.processes", + registry.counter("discord.messages", Tags.of( - Tag.of("guild", guild), - Tag.of("channel", channel), - Tag.of("user", user) + Tag.of("user.id", user.getId()), + Tag.of("user.name", user.getName()), + Tag.of("guild.id", guild.getId()), + Tag.of("guild.name", guild.getName()), + Tag.of("channel.id", channel.getId()), + Tag.of("channel.name", channel.getName()) ) ).increment(); } diff --git a/api/src/main/java/dev/vrba/dubs/repository/MatchRepository.java b/api/src/main/java/dev/vrba/dubs/repository/MatchRepository.java new file mode 100644 index 0000000..6db4ed8 --- /dev/null +++ b/api/src/main/java/dev/vrba/dubs/repository/MatchRepository.java @@ -0,0 +1,9 @@ +package dev.vrba.dubs.repository; + +import dev.vrba.dubs.domain.Match; +import io.micronaut.data.annotation.Repository; +import io.micronaut.data.repository.CrudRepository; + +@Repository +public interface MatchRepository extends CrudRepository { +} diff --git a/api/src/main/java/dev/vrba/dubs/service/MessageProcessingService.java b/api/src/main/java/dev/vrba/dubs/service/MessageProcessingService.java index a6a76fb..89508d4 100644 --- a/api/src/main/java/dev/vrba/dubs/service/MessageProcessingService.java +++ b/api/src/main/java/dev/vrba/dubs/service/MessageProcessingService.java @@ -4,6 +4,7 @@ import dev.vrba.dubs.dto.ChannelDto; import dev.vrba.dubs.dto.GuildDto; import dev.vrba.dubs.dto.UserDto; +import dev.vrba.dubs.metrics.MatchedPatternsMetric; import dev.vrba.dubs.metrics.ProcessedMessagesMetric; import dev.vrba.dubs.repository.MatchedPatternRepository; import io.micronaut.core.annotation.NonNull; @@ -35,43 +36,38 @@ public class MessageProcessingService { private final PatternMatchingService patternMatchingService; @NonNull - private final ProcessedMessagesMetric metric; + private final ProcessedMessagesMetric processedMessageMetric; @NonNull - @Transactional - public List processMessage( - final @NonNull String message, - final @NonNull UserDto user, - final @NonNull ChannelDto channel, - final @NonNull GuildDto guild - ) { - metric.record(user.id(), channel.id(), guild.id()); + private final MatchedPatternsMetric matchedPatternsMetric; + @NonNull + @Transactional + public List processMessage(final @NonNull String message, final @NonNull UserDto user, final @NonNull ChannelDto channel, final @NonNull GuildDto guild) { final Guild guildEntity = guildService.upsertGuild(guild.id(), guild.name(), guild.icon()); final User userEntity = userService.upsertUser(user.id(), user.name(), user.avatar()); final Channel channelEntity = channelService.upsertChannel(channel.id(), channel.name(), guildEntity); final List patterns = patternMatchingService.matchMessagePatterns(message); + processedMessageMetric.record(userEntity, channelEntity, guildEntity); + if (!patterns.isEmpty()) { - final List mappedPatterns = patterns.stream() - .map(pattern -> new MatchedPattern( - null, - channelEntity.getId(), - userEntity.getId(), - pattern.getName(), - pattern.getPoints(), - pattern.isRare() - )) - .toList(); - - final BigInteger totalPoints = patterns.stream() - .map(Pattern::getPoints) - .map(BigInteger::valueOf) - .reduce(BigInteger.ZERO, BigInteger::add); + final List mappedPatterns = patterns.stream().map(pattern -> new MatchedPattern(null, channelEntity.getId(), userEntity.getId(), pattern.getName(), pattern.getPoints(), pattern.isRare())).toList(); + + final BigInteger totalPoints = patterns.stream().map(Pattern::getPoints).map(BigInteger::valueOf).reduce(BigInteger.ZERO, BigInteger::add); userService.incrementUserPoints(userEntity, totalPoints); matchedPatternRepository.saveAll(mappedPatterns); + patterns.forEach(pattern -> { + matchedPatternsMetric.record( + userEntity, + channelEntity, + guildEntity, + pattern + ); + }); + return patterns; } diff --git a/api/src/main/resources/application.yml b/api/src/main/resources/application.yml index c1f345a..dee8bcf 100644 --- a/api/src/main/resources/application.yml +++ b/api/src/main/resources/application.yml @@ -8,11 +8,20 @@ micronaut: metrics: enabled: true + binders: + jdbc.enabled: false + jvm.enabled: false + logback.enabled: false + processor.enabled: false + files.enabled: false + uptime.enabled: false + web.enabled: false + executor.enabled: false export: prometheus: enabled: true - descriptions: true - step: PT1M + descriptions: false + step: PT60S security: basic-auth: diff --git a/api/src/main/resources/migrations/V6__add_row_id_to_matches.sql b/api/src/main/resources/migrations/V6__add_row_id_to_matches.sql new file mode 100644 index 0000000..a1c9bb7 --- /dev/null +++ b/api/src/main/resources/migrations/V6__add_row_id_to_matches.sql @@ -0,0 +1,25 @@ +drop view if exists matches; +create or replace view matches as +( +select row_number() over (order by users.id) as row_id, + pattern_name, + pattern_points, + pattern_is_rare, + users.id as user_id, + users.name as user_name, + channels.id as channel_id, + channels.name as channel_name, + guilds.id as guild_id, + guilds.name as guild_name, + count(*) as count +from matched_patterns + left join users on matched_patterns.user_id = users.id + left join channels on matched_patterns.channel_id = channels.id + left join guilds on channels.guild_id = guilds.id +group by users.id, + channels.id, + guilds.id, + pattern_name, + pattern_points, + pattern_is_rare + ); diff --git a/docker/prometheus.yml b/docker/prometheus.yml index 5c12810..e5f2594 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -1,20 +1,10 @@ -global: - scrape_interval: 15s - evaluation_interval: 30s - body_size_limit: 15MB - sample_limit: 1500 - target_limit: 30 - label_limit: 30 - label_name_length_limit: 200 - label_value_length_limit: 200 - scrape_configs: - job_name: prometheus honor_labels: true static_configs: - targets: - "host.docker.internal:8080" - scrape_interval: 50s + scrape_interval: 30s scrape_timeout: 5s body_size_limit: 10MB sample_limit: 1000