Skip to content

Commit

Permalink
feat: export/import
Browse files Browse the repository at this point in the history
  • Loading branch information
ptitFicus committed Sep 17, 2024
1 parent ea51878 commit 324cb16
Show file tree
Hide file tree
Showing 49 changed files with 2,843 additions and 820 deletions.
1 change: 1 addition & 0 deletions app/fr/maif/izanami/application.scala
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ class IzanamiComponentsInstances(
lazy val eventController = wire[EventController]
lazy val webhookController = wire[WebhookController]
lazy val frontendController = wire[FrontendController]
lazy val exportController = wire[ExportController]
lazy val searchController = wire[SearchController]

override lazy val assets: Assets = wire[Assets]
Expand Down
450 changes: 450 additions & 0 deletions app/fr/maif/izanami/datastores/ImportExportDatastore.scala

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions app/fr/maif/izanami/env/env.scala
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class Datastores(env: Env) {
val configuration: ConfigurationDatastore = new ConfigurationDatastore(env)
val webhook: WebhooksDatastore = new WebhooksDatastore(env)
val stats: StatsDatastore = new StatsDatastore(env)
val exportDatastore: ImportExportDatastore = new ImportExportDatastore(env)
val search : SearchDatastore = new SearchDatastore(env)

def onStart(): Future[Unit] = {
Expand Down
32 changes: 27 additions & 5 deletions app/fr/maif/izanami/errors/Errors.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package fr.maif.izanami.errors

import fr.maif.izanami.models.ExportedType
import play.api.http.Status.{BAD_REQUEST, FORBIDDEN, INTERNAL_SERVER_ERROR, NOT_FOUND, UNAUTHORIZED}
import play.api.libs.json.{Json, Writes}
import play.api.libs.json.{JsObject, Json, Writes}
import play.api.mvc.{Result, Results}

import java.util.Objects
Expand Down Expand Up @@ -39,7 +40,7 @@ case class MissingFeatureFields()
extends IzanamiError(message = "Some fields are missing for feature object", status = BAD_REQUEST)
case class FeatureNotFound(id: String)
extends IzanamiError(message = s"Feature ${id} does not exists", status = NOT_FOUND)
case class KeyNotFound(name: String) extends IzanamiError(message = s"Key ${name} does not exists", status = NOT_FOUND)
case class KeyNotFound(name: String) extends IzanamiError(message = s"Key ${name} does not exists", status = NOT_FOUND)
case class ProjectContextOrFeatureDoesNotExist(project: String, context: String, feature: String)
extends IzanamiError(
message = s"Project ${project}, context ${context} or feature ${feature} does not exist",
Expand Down Expand Up @@ -124,9 +125,30 @@ case class WebhookCreationFailed(
case class WebhookDoesNotExists(id: String)
extends IzanamiError(message = s"No webhook with id $id", status = NOT_FOUND)

case class WebhookCallError(callStatus: Int, body: Option[String], hookName: String) extends IzanamiError(message = s"Webhook $hookName call failed with status $callStatus and response body ${body.getOrElse("No body")}", status = callStatus)
case class EventNotFound(tenant: String, event: Long) extends IzanamiError(message = s"Event $event not found", status = 500)
case class WebhookRetryCountExceeded() extends IzanamiError(message = s"Exceeded webhook retry count", status = 500)
case class WebhookCallError(callStatus: Int, body: Option[String], hookName: String)
extends IzanamiError(
message = s"Webhook $hookName call failed with status $callStatus and response body ${body.getOrElse("No body")}",
status = callStatus
)
case class EventNotFound(tenant: String, event: Long)
extends IzanamiError(message = s"Event $event not found", status = 500)
case class WebhookRetryCountExceeded() extends IzanamiError(message = s"Exceeded webhook retry count", status = 500)
case class TableDoesNotExist(tenant: String, table: String)
extends IzanamiError(message = s"Table $table does not exist for tenant $tenant", status = 500)
case class ConflictingName(tenant: String, entityTpe: String, row: JsObject)
extends IzanamiError(
message =
s"An entity of type $entityTpe already exists in tenant $tenant with the same unique values but with different id. Row is ${row
.toString()}",
status = 400
)
case class GenericBadRequest(override val message: String) extends IzanamiError(message = message, status = 400)
case class PartialImportFailure(failedElements: Map[ExportedType, Seq[JsObject]]) extends IzanamiError(message= s"Some element couldn't be imported", status = 400)
case class ImportError(table: String, json: String, errorMessage: String)
extends IzanamiError(
message = s"Error key while inserting into table $table with error $errorMessage values $json : ",
status = 400
)
object IzanamiError {
implicit val errorWrite: Writes[IzanamiError] = { err =>
Json.obj(
Expand Down
119 changes: 119 additions & 0 deletions app/fr/maif/izanami/models/ExportType.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package fr.maif.izanami.models

sealed trait ExportedType {
def order: Int
def table: String
def displayName: String
}
case object ScriptType extends ExportedType {
override def order: Int = 0
override def table: String = "wasm_script_configurations"
override def displayName: String = "WASM Script"
}
case object ProjectType extends ExportedType {
override def order: Int = 1
override def table: String = "projects"
override def displayName: String = "Project"
}
case object KeyType extends ExportedType {
override def order: Int = 2
override def table: String = "apikeys"
override def displayName: String = "Key"
}
case object KeyProjectType extends ExportedType {
override def order: Int = 3
override def table: String = "apikeys_projects"
override def displayName: String = "Key right on project"
}
case object GlobalContextType extends ExportedType {
override def order: Int = 4
override def table: String = "global_feature_contexts"
override def displayName: String = "Global context"
}
case object LocalContextType extends ExportedType {
override def order: Int = 5
override def table: String = "feature_contexts"
override def displayName: String = "Local context"
}
case object WebhookType extends ExportedType {
override def order: Int = 6
override def table: String = "webhooks"
override def displayName: String = "Webhook"
}
case object TagType extends ExportedType {
override def order: Int = 7
override def table: String = "tags"
override def displayName: String = "Tag"
}
case object FeatureType extends ExportedType {
override def order: Int = 8
override def table: String = "features"
override def displayName: String = "Feature"
}
case object FeatureTagType extends ExportedType {
override def order: Int = 9
override def table: String = "features_tags"
override def displayName: String = "Feature tags link"
}
case object OverloadType extends ExportedType {
override def order: Int = 10
override def table: String = "feature_contexts_strategies"
override def displayName: String = "Feature overload"
}
case object ProjectRightType extends ExportedType {
override def order: Int = 11
override def table: String = "users_projects_rights"
override def displayName: String = "User right on project"
}
case object KeyRightType extends ExportedType {
override def order: Int = 12

override def table: String = "users_keys_rights"
override def displayName: String = "User right on key"
}
case object WebhookRightType extends ExportedType {
override def order: Int = 13

override def table: String = "users_webhooks_rights"
override def displayName: String = "User right on webhook"
}
case object WebhookFeatureType extends ExportedType {
override def order: Int = 14

override def table: String = "webhooks_features"
override def displayName: String = "Webhook feature link"
}
case object WebhookProjectType extends ExportedType {
override def order: Int = 15
override def table: String = "webhooks_projects"
override def displayName: String = "Webhook project link"
}

object ExportedType {
val exportedTypeToExportedNameAssociation: Seq[(String, ExportedType)] = Seq(
("project", ProjectType),
("feature", FeatureType),
("tag", TagType),
("feature_tag", FeatureTagType),
("overload", OverloadType),
("local_context", LocalContextType),
("global_context", GlobalContextType),
("key", KeyType),
("apikey_project", KeyProjectType),
("webhook", WebhookType),
("webhook_feature", WebhookFeatureType),
("webhook_project", WebhookProjectType),
("user_webhook_right", WebhookRightType),
("project_right", ProjectRightType),
("key_right", KeyRightType),
("script", ScriptType)
)

def parseExportedType(typestr: String): Option[ExportedType] = {
exportedTypeToExportedNameAssociation.find(t => t._1 == typestr).map(t => t._2)
}

def exportedTypeToString(exportedType: ExportedType): Option[String] = {
exportedTypeToExportedNameAssociation.find(t => t._2 == exportedType).map(t => t._1)
}
}
38 changes: 38 additions & 0 deletions app/fr/maif/izanami/utils/helpers.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package fr.maif.izanami.utils

import scala.concurrent.{ExecutionContext, Future}

object Helpers {
def sequence[A, B](seq: Seq[Either[A, B]]): Either[A, Seq[B]] = seq.foldRight(Right(Nil): Either[A, List[B]]) {
(e, acc) => for (xs <- acc; x <- e) yield x :: xs
}

def sequence[A, B, C](seq: Seq[Either[A, B]], init: => C, acc: (C, A) => C): Either[C, Seq[B]] =
seq.foldRight(Right(Nil): Either[C, List[B]]) {
case (Left(err), Left(errAcc)) => Left(acc(errAcc, err))
case (Left(err), Right(_)) => Left(acc(init, err))
case (Right(v), Right(vs)) => Right(v :: vs)
case (Right(_), Left(errAcc)) => Left(errAcc)
}

def chainFutures[E, R](
futures: Seq[Future[Either[E, R]]]
)(implicit ec: ExecutionContext): Future[Either[Seq[E], Seq[R]]] = {
def act(fs: Seq[Future[Either[E, R]]], res: Either[Seq[E], Seq[R]]): Future[Either[Seq[E], Seq[R]]] = {
fs.headOption
.map(f => {
f.map(e =>
(res, e) match {
case (Left(errs), Left(err)) => Left(errs.appended(err))
case (_, Left(err)) => Left(Seq(err))
case (Left(errs), _) => Left(errs)
case (Right(rs), Right(r)) => Right(rs.appended(r))
}
).flatMap(e => act(fs.tail, e))
})
.getOrElse(Future.successful(res))
}

act(futures, Right(Seq()))
}
}
111 changes: 111 additions & 0 deletions app/fr/maif/izanami/web/ExportController.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package fr.maif.izanami.web

import akka.util.ByteString
import fr.maif.izanami.env.Env
import fr.maif.izanami.models.Feature.lightweightFeatureWrite
import fr.maif.izanami.models.{LightWeightFeature, RightLevels}
import play.api.http.HttpEntity
import play.api.libs.json._
import play.api.mvc._

import scala.concurrent.{ExecutionContext, Future}

class ExportController(
val env: Env,
val controllerComponents: ControllerComponents,
val authAction: TenantAuthActionFactory
) extends BaseController {
implicit val ec: ExecutionContext = env.executionContext

def exportTenantData(tenant: String): Action[JsValue] = authAction(tenant, RightLevels.Admin).async(parse.json) {
implicit request =>
{
ExportController.tenantExportRequestReads
.reads(request.body)
.asEither
.map(req => {
env.datastores.exportDatastore.exportTenantData(tenant, req)
})
.fold(
_ => Future.successful(BadRequest(Json.obj("message" -> "Bad body format"))),
futureResult =>
futureResult.map(jsons => {
Result(
header =
ResponseHeader(200, Map("Content-Disposition" -> "attachment", "filename" -> "export.ndjson")),
body = HttpEntity.Streamed(
akka.stream.scaladsl.Source.single(ByteString(jsons.mkString("\n"), "UTF-8")),
None,
Some("application/x-ndjson")
)
)
})
)
}
}
}

object ExportController {
val exportRequestReads: Reads[ExportRequest] = json => {
(json \ "tenants")
.asOpt[Map[String, TenantExportRequest]](Reads.map(tenantExportRequestReads))
.map(projects => JsSuccess(ExportRequest(projects)))
.getOrElse(JsError("Bad body format"))
}

val tenantExportRequestReads: Reads[TenantExportRequest] = json => {
(for (
projects: ExportList <- (json \ "allProjects")
.asOpt[Boolean]
.flatMap(isAllProjects =>
if (isAllProjects) Some(ExportAllItems)
else (json \ "projects").asOpt[Set[String]].map(set => ExportItemList(set))
);
keys: ExportList <- (json \ "allKeys")
.asOpt[Boolean]
.flatMap(isAllProjects =>
if (isAllProjects) Some(ExportAllItems)
else (json \ "keys").asOpt[Set[String]].map(set => ExportItemList(set))
);
webhooks: ExportList <- (json \ "allWebhooks")
.asOpt[Boolean]
.flatMap(isAllProjects =>
if (isAllProjects) Some(ExportAllItems)
else (json \ "webhooks").asOpt[Set[String]].map(set => ExportItemList(set))
);
userRights <- (json \ "userRights").asOpt[Boolean]
) yield JsSuccess(TenantExportRequest(projects, keys, webhooks, userRights))).getOrElse(JsError("Bad body format"))
}

val exportResultWrites: Writes[ExportResult] = exportResult => {
Json.obj(
"tenants" -> Json.toJson(exportResult.tenants)(Writes.map(tenantExportResultWrites))
)
}

val tenantExportResultWrites: Writes[TenantExportResult] = exportResult => {
Json.obj("projects" -> Json.toJson(exportResult.projects)(Writes.map(projectExportResultWrites)))
}

val projectExportResultWrites: Writes[ProjectExportResult] = exportResult => {
Json.obj("features" -> Json.toJson(exportResult.features)(Writes.list(lightweightFeatureWrite)))
}

case class ExportRequest(tenants: Map[String, TenantExportRequest]) {}
case class TenantExportRequest(projects: ExportList, keys: ExportList, webhooks: ExportList, userRights: Boolean)

sealed trait ExportList
case object ExportAllItems extends ExportList
case class ExportItemList(items: Set[String]) extends ExportList

case class ExportResult(
tenants: Map[String, TenantExportResult]
)

case class TenantExportResult(
projects: Map[String, ProjectExportResult]
)

case class ProjectExportResult(features: List[LightWeightFeature])

}
Loading

0 comments on commit 324cb16

Please sign in to comment.