chore/database-migration (#154)

prepare migration from MongoDB to MariaDB

build.gradle.kts

@@ -20,9 +20,11 @@ plugins {
     alias(libs.plugins.dependencyManagement)
     alias(libs.plugins.kotlin.jvm)
     alias(libs.plugins.kotlin.spring)
+    alias(libs.plugins.kotlin.jpa)
 
     alias(libs.plugins.kover)
     alias(libs.plugins.detekt)
+    alias(libs.plugins.flyway)
 
     id("fr.nicopico.conventions.kotlin-strict")
     id("fr.nicopico.conventions.quality")
@@ -68,6 +70,26 @@ npi {
     }
 }
 
+// Allows usage of Flyway commands through the Gradle plugin
+flyway {
+    url = "jdbc:mariadb://localhost:3306/n2rss-database"
+    user = "n2rss"
+    password = "secret"
+
+    schemas = arrayOf("n2rss-database")
+    locations = arrayOf(
+        "classpath:db/migration",
+        "classpath:fr/nicopico/n2rss/newsletter/data/migration",
+    )
+
+    cleanDisabled = false
+}
+
+// Enable support for Java migration for Flyway Gradle plugin
+tasks.named<Task>("flywayMigrate") {
+    dependsOn(tasks.named("classes"))
+}
+
 repositories {
     mavenCentral()
 }
@@ -87,6 +109,10 @@ dependencies {
     implementation(libs.jsonPath)
     implementation(libs.annotations)
 
+    runtimeOnly(libs.mariadb.driver)
+    implementation(libs.flyway.core)
+    implementation(libs.flyway.mysql)
+
     testImplementation(libs.springBoot.starter.test) {
         exclude(group = "org.mockito")
     }
@@ -97,4 +123,5 @@ dependencies {
     testImplementation(libs.greenmail)
     testImplementation(libs.greenmail.junit5)
     testImplementation(libs.mockwebserver)
+    testRuntimeOnly(libs.h2.database)
 }

compose.yaml

@@ -1,12 +1,26 @@
 version: '3.8'
 services:
   db:
-    image: mongo:latest
+    image: mongo:4.2
     ports:
       - '27017:27017'
     volumes:
       - db:/data/db
+  db2:
+    image: mariadb:10.4
+    environment:
+      - 'MARIADB_DATABASE=n2rss-database'
+      - 'MARIADB_PASSWORD=secret'
+      - 'MARIADB_ROOT_PASSWORD=verysecret'
+      - 'MARIADB_USER=n2rss'
+    ports:
+      - '3306:3306'
+    volumes:
+      - db2:/data/db2
 volumes:
   db:
     driver:
       local
+  db2:
+    driver:
+      local

deploy/application.properties

@@ -16,7 +16,11 @@
 # DEALINGS IN THE SOFTWARE.
 #
 spring.profiles.active=default
-
+n2rss.persistenceMode=${N2RSS_PERSISTENCE_MODE}
+spring.datasource.url=${N2RSS_DATABASE_URL}
+spring.datasource.username=${N2RSS_DATABASE_USERNAME}
+spring.datasource.password=${N2RSS_DATABASE_PASSWORD}
+# TODO Remove MongoDB after the migration
 spring.data.mongodb.host=${N2RSS_MONGODB_HOST}
 spring.data.mongodb.port=${N2RSS_MONGODB_PORT:27017}
 spring.data.mongodb.database=${N2RSS_MONGODB_DATABASE}

gradle/libs.versions.toml

@@ -1,32 +1,34 @@
 [versions]
-springBoot = "3.3.5"
+annotations = "26.0.1"
 dependencyManagement = "1.1.6"
-kotlin = "2.0.10"
-kover = "0.8.2"
 detekt = "1.23.7"
+flyway = "10.21.0"
+greenmail = "2.1.0"
+h2 = "2.2.224"
 jakartaMail = "2.0.1"
-kotlinxDatetime = "0.6.1"
-jsoup = "1.18.1"
-rome = "2.1.0"
 jsonPath = "2.9.0"
-annotations = "26.0.1"
+jsoup = "1.18.1"
 junitJupiterEngine = "5.10.3"
 kotestAssertions = "5.9.1"
 kotestKotlinxDatetime = "1.1.0"
+kotlin = "2.0.10"
+kotlinxDatetime = "0.6.1"
+kover = "0.8.2"
 mockk = "1.13.13"
-greenmail = "2.1.0"
 mockwebserver = "4.12.0"
+rome = "2.1.0"
+springBoot = "3.3.5"
 
 [libraries]
 # Kotlin
-kotlin-stdLib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" }
 kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" }
 kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinxDatetime" }
 
 # Spring Boot Starters
 springBoot-starter-web = { module = "org.springframework.boot:spring-boot-starter-web", version.ref = "springBoot" }
 springBoot-starter-actuator = { module = "org.springframework.boot:spring-boot-starter-actuator", version.ref = "springBoot" }
-springBoot-starter-dataMongo = { module = "org.springframework.boot:spring-boot-starter-data-mongodb", version.ref = "springBoot" }
+springBoot-starter-data-mongo = { module = "org.springframework.boot:spring-boot-starter-data-mongodb", version.ref = "springBoot" }
+springBoot-starter-data-jpa = { module = "org.springframework.boot:spring-boot-starter-data-jpa", version.ref = "springBoot" }
 springBoot-starter-thymeleaf = { module = "org.springframework.boot:spring-boot-starter-thymeleaf", version.ref = "springBoot" }
 springBoot-starter-validation = { module = "org.springframework.boot:spring-boot-starter-validation", version.ref = "springBoot" }
 springBoot-starter-aop = { module = "org.springframework.boot:spring-boot-starter-aop", version.ref = "springBoot" }
@@ -36,27 +38,32 @@ springBoot-configurationProcessor = { module = "org.springframework.boot:spring-
 springBoot-starter-test = { module = "org.springframework.boot:spring-boot-starter-test", version.ref = "springBoot" }
 
 # Other Libraries
+annotations = { module = "org.jetbrains:annotations", version.ref = "annotations" }
+flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flyway" }
+flyway-mysql = { module = "org.flywaydb:flyway-mysql", version.ref = "flyway" }
+h2-database = { module = "com.h2database:h2", version.ref = "h2" }
 jacksonModuleKotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin" }
 jakartaMail = { module = "com.sun.mail:jakarta.mail", version.ref = "jakartaMail" }
+jsonPath = { module = "com.jayway.jsonpath:json-path", version.ref = "jsonPath" }
 jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" }
+mariadb-driver = { module = "org.mariadb.jdbc:mariadb-java-client" }
 rome = { module = "com.rometools:rome", version.ref = "rome" }
-jsonPath = { module = "com.jayway.jsonpath:json-path", version.ref = "jsonPath" }
-annotations = { module = "org.jetbrains:annotations", version.ref = "annotations" }
 
 # Testing Libraries
+greenmail = { module = "com.icegreen:greenmail", version.ref = "greenmail" }
+greenmail-junit5 = { module = "com.icegreen:greenmail-junit5", version.ref = "greenmail" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junitJupiterEngine" }
 kotest-assertions-core = { module = "io.kotest:kotest-assertions-core-jvm", version.ref = "kotestAssertions" }
 kotest-assertions-kotlinxDatetime = { module = "io.kotest.extensions:kotest-assertions-kotlinx-datetime", version.ref = "kotestKotlinxDatetime" }
 mockk = { module = "io.mockk:mockk", version.ref = "mockk" }
-greenmail = { module = "com.icegreen:greenmail", version.ref = "greenmail" }
-greenmail-junit5 = { module = "com.icegreen:greenmail-junit5", version.ref = "greenmail" }
 mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "mockwebserver" }
 
 [bundles]
 springBoot-starters = [
     "springBoot-starter-web",
     "springBoot-starter-actuator",
-    "springBoot-starter-dataMongo",
+    "springBoot-starter-data-mongo",
+    "springBoot-starter-data-jpa",
     "springBoot-starter-thymeleaf",
     "springBoot-starter-validation",
     "springBoot-starter-aop",
@@ -67,9 +74,11 @@ springBoot-dev = [
 ]
 
 [plugins]
-springBoot = { id = "org.springframework.boot", version.ref = "springBoot" }
 dependencyManagement = { id = "io.spring.dependency-management", version.ref = "dependencyManagement" }
+detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" }
+flyway = { id = "org.flywaydb.flyway", version.ref = "flyway" }
+kotlin-jpa = { id = "org.jetbrains.kotlin.plugin.jpa", version.ref = "kotlin" }
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" }
 kotlin-spring = { id = "org.jetbrains.kotlin.plugin.spring", version.ref = "kotlin" }
 kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" }
-detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" }
+springBoot = { id = "org.springframework.boot", version.ref = "springBoot" }

settings.gradle.kts

@@ -19,3 +19,13 @@
 rootProject.name = "n2rss"
 
 includeBuild("build-conventions")
+
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        // Use by Flyway Gradle Plugin
+        classpath("org.flywaydb:flyway-mysql:10.21.0")
+    }
+}

src/main/kotlin/fr/nicopico/n2rss/config/N2RssConfiguration.kt

@@ -63,6 +63,7 @@ constructor(
     val email: EmailProperties,
     val analytics: AnalyticsProperties,
     val github: GithubProperties,
+    val persistenceMode: PersistenceMode = PersistenceMode.LEGACY,
 ) {
     data class MaintenanceProperties(
         val secretKey: String,

src/main/kotlin/fr/nicopico/n2rss/config/MongoConfiguration.kt → src/main/kotlin/fr/nicopico/n2rss/config/PersistenceConfiguration.kt

@@ -24,7 +24,7 @@ import org.springframework.data.convert.PropertyValueConverterFactory
 import org.springframework.data.mongodb.core.convert.MongoCustomConversions
 
 @Configuration
-class MongoConfiguration {
+class PersistenceConfiguration {
 
     @Bean
     fun customConversions(beanFactory: BeanFactory): MongoCustomConversions {
@@ -35,4 +35,9 @@ class MongoConfiguration {
             it.registerPropertyValueConverterFactory(propertyValueConverterFactory)
         }
     }
+
+    @Bean
+    fun persistenceMode(config: N2RssProperties): PersistenceMode {
+        return config.persistenceMode
+    }
 }

src/main/kotlin/fr/nicopico/n2rss/monitoring/data/GithubIssueEmailClientErrorRepository.kt → src/main/kotlin/fr/nicopico/n2rss/config/PersistenceMode.kt

@@ -16,11 +16,10 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-package fr.nicopico.n2rss.monitoring.data
+package fr.nicopico.n2rss.config
 
-import fr.nicopico.n2rss.monitoring.github.IssueId
-import org.springframework.data.mongodb.repository.MongoRepository
-
-interface GithubIssueEmailClientErrorRepository : MongoRepository<GithubIssueData.EmailClientError, IssueId> {
-    fun getEmailClientErrorByErrorMessage(error: String): GithubIssueData.EmailClientError?
+enum class PersistenceMode {
+    LEGACY,
+    MIGRATION,
+    DEFAULT,
 }

src/main/kotlin/fr/nicopico/n2rss/controller/maintenance/MaintenanceController.kt

@@ -20,13 +20,15 @@ package fr.nicopico.n2rss.controller.maintenance
 import fr.nicopico.n2rss.analytics.models.AnalyticsEvent
 import fr.nicopico.n2rss.analytics.service.AnalyticsService
 import fr.nicopico.n2rss.config.N2RssProperties
+import fr.nicopico.n2rss.newsletter.service.MigrationService
 import jakarta.servlet.http.HttpServletResponse
 import org.jetbrains.annotations.VisibleForTesting
 import org.springframework.boot.SpringApplication
 import org.springframework.context.ApplicationContext
 import org.springframework.context.ConfigurableApplicationContext
 import org.springframework.http.MediaType
 import org.springframework.stereotype.Controller
+import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestHeader
 import org.springframework.web.bind.annotation.RequestParam
@@ -36,6 +38,7 @@ import kotlin.concurrent.thread
 class MaintenanceController(
     private val applicationContext: ApplicationContext,
     private val analyticsService: AnalyticsService,
+    private val migrationService: MigrationService,
     private val properties: N2RssProperties.MaintenanceProperties,
 ) {
     @PostMapping("/notifyRelease")
@@ -71,6 +74,13 @@ class MaintenanceController(
         }
     }
 
+    @GetMapping("/migrate-database")
+    fun migrateDatabase(response: HttpServletResponse) {
+        migrationService.migrateToNewDatabase()
+        response.status = HttpServletResponse.SC_OK
+        response.writer.write("Database has been migrated")
+    }
+
     companion object {
         @VisibleForTesting
         const val RESTART_DELAY_MS = 1000L

src/main/kotlin/fr/nicopico/n2rss/monitoring/GithubMonitoringService.kt

@@ -20,7 +20,7 @@ package fr.nicopico.n2rss.monitoring
 
 import fr.nicopico.n2rss.mail.models.Email
 import fr.nicopico.n2rss.monitoring.data.GithubIssueData
-import fr.nicopico.n2rss.monitoring.data.GithubIssueRepository
+import fr.nicopico.n2rss.monitoring.data.GithubIssueService
 import fr.nicopico.n2rss.monitoring.github.GithubClient
 import fr.nicopico.n2rss.monitoring.github.GithubException
 import kotlinx.datetime.Clock
@@ -36,7 +36,7 @@ import java.net.URL
 
 @Service
 class GithubMonitoringService(
-    private val repository: GithubIssueRepository,
+    private val repository: GithubIssueService,
     private val client: GithubClient,
     private val clock: Clock,
 ) : MonitoringService {

src/main/kotlin/fr/nicopico/n2rss/monitoring/data/GithubIssueData.kt

@@ -18,32 +18,55 @@
 package fr.nicopico.n2rss.monitoring.data
 
 import fr.nicopico.n2rss.monitoring.github.IssueId
-import org.springframework.data.annotation.Id
-import org.springframework.data.mongodb.core.index.Indexed
-import org.springframework.data.mongodb.core.mapping.Document
+import jakarta.persistence.Column
+import jakarta.persistence.DiscriminatorColumn
+import jakarta.persistence.DiscriminatorType
+import jakarta.persistence.DiscriminatorValue
+import jakarta.persistence.Entity
+import jakarta.persistence.Id
+import jakarta.persistence.Inheritance
+import jakarta.persistence.InheritanceType
 import java.net.URL
 
-@Document(collection = "github_issues")
-sealed class GithubIssueData(
+@Entity(name = "GITHUB_ISSUES")
+@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
+@DiscriminatorColumn(
+    name = "issue_type",
+    discriminatorType = DiscriminatorType.STRING,
+)
+open class GithubIssueData(
     @Id
+    @Column(name = "issue_id")
     val issueId: IssueId
 ) {
+
+    @Entity
+    @DiscriminatorValue("email-client-error")
     class EmailClientError(
         issueId: IssueId,
+
+        @Column(nullable = false)
         val errorMessage: String
     ) : GithubIssueData(issueId)
 
+    @Entity
+    @DiscriminatorValue("email-processing-error")
     class EmailProcessingError(
         issueId: IssueId,
-        @Indexed
+
+        @Column(nullable = false)
         val emailTitle: String,
-        @Indexed
+
+        @Column(nullable = false)
         val errorMessage: String,
     ) : GithubIssueData(issueId)
 
+    @Entity
+    @DiscriminatorValue("newsletter-request")
     class NewsletterRequest(
         issueId: IssueId,
-        @Indexed(unique = true)
+
+        @Column(nullable = false)
         val newsletterUrl: URL,
     ) : GithubIssueData(issueId)
 }