From 5874129e3b00f308dcacee1e9abae3250020ab65 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 21 Oct 2020 02:26:03 -0400 Subject: [PATCH 01/51] add play-slick, play-slick-evolutions, and postgresql dependencies to project kgLib --- build.sbt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 41e6296f..18791897 100644 --- a/build.sbt +++ b/build.sbt @@ -50,7 +50,10 @@ lazy val kgLib = "io.github.tetherless-world" %% "twxplore-base" % twxploreVersion, "io.github.tetherless-world" %% "twxplore-test" % twxploreVersion % Test, "me.tongfei" % "progressbar" % "0.8.1", - "org.neo4j.driver" % "neo4j-java-driver" % "4.0.1" + "org.neo4j.driver" % "neo4j-java-driver" % "4.0.1", + "com.typesafe.play" %% "play-slick" % "5.0.0", + "com.typesafe.play" %% "play-slick-evolutions" % "5.0.0", + "org.postgresql" % "postgresql" % "42.2.18" ), maintainer := maintainerValue, name := "mcs-kg-lib" From 50173f72dfe8b522a18f28eee4c4ed2dfff55765 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 23 Oct 2020 02:54:01 -0400 Subject: [PATCH 02/51] Add config for slick and attempt at tables for KgNode, KgEdge, and KgSource --- app/kg/conf/application.conf | 5 + .../lib/kg/stores/KgStoresModule.scala | 4 + .../postgres/AbstractPostgresKgStore.scala | 91 +++++++++++++++++++ .../postgres/PostgresKgQueryStore.scala | 48 ++++++++++ 4 files changed, 148 insertions(+) create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala diff --git a/app/kg/conf/application.conf b/app/kg/conf/application.conf index cd44e9ba..54adb1c3 100644 --- a/app/kg/conf/application.conf +++ b/app/kg/conf/application.conf @@ -5,6 +5,11 @@ kgStore = "neo4j" neo4j.password = "nC1aB4mji623s2Zs" neo4j.uri = "bolt://mcs-neo4j:7687" neo4j.user = "neo4j" +slick.dbs.default.profile = "slick.jdbc.PostgresProfile$" +slick.dbs.default.db.driver = "org.postgresql.Driver" +slick.dbs.default.db.url = "postgres://mcs-postgres:5432/kg" +slick.dbs.default.db.user = "mcs" +slick.dbs.default.db.password = "7EAdu7jJvZNxxrNZ" play.filters.enabled += "play.filters.cors.CORSFilter" play.filters.disabled += "play.filters.csrf.CSRFFilter" play.filters.disabled += "play.filters.hosts.AllowedHostsFilter" diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala index 58255cf9..bfef0477 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala @@ -4,6 +4,7 @@ import com.google.inject.AbstractModule import io.github.tetherlessworld.mcsapps.lib.kg.stores.empty.EmptyKgStore import io.github.tetherlessworld.mcsapps.lib.kg.stores.mem.MemKgStore import io.github.tetherlessworld.mcsapps.lib.kg.stores.neo4j.{Neo4jKgCommandStore, Neo4jKgQueryStore} +import io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres.PostgresKgQueryStore import io.github.tetherlessworld.mcsapps.lib.kg.stores.test.TestKgStore import org.slf4j.LoggerFactory import play.api.{Configuration, Environment} @@ -27,6 +28,9 @@ final class KgStoresModule(environment: Environment, configuration: Configuratio bind(classOf[KgCommandStore]).to(classOf[TestKgStore]) bind(classOf[KgQueryStore]).to(classOf[TestKgStore]) } + case "postgres" => { + bind(classOf[KgQueryStore]).to(classOf[PostgresKgQueryStore]) + } case _ => { bind(classOf[KgCommandStore]).to(classOf[Neo4jKgCommandStore]) bind(classOf[KgQueryStore]).to(classOf[Neo4jKgQueryStore]) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala new file mode 100644 index 00000000..b1db57cc --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -0,0 +1,91 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode +import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} +import slick.jdbc.PostgresProfile + + +abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { + import profile.api._ + + lazy val KgNodes = TableQuery[KgNodeTable] + lazy val KgEdges = TableQuery[KgEdgeTable] + lazy val KgSources = TableQuery[KgSourceTable] + + private class KgNodeTable(tag: Tag) extends Table[KgNode](tag, "KgNodes") { + def id = column[String]("ID", O.PrimaryKey) + + // def inDegree = column[Option[Int]]("IN_DEGREE") + // def labels = column[String]("LABELS") + // def outDegree = column[Option[Int]]("OUT_DEGREE") + // def pageRank = column[Option[Double]]("PAGERANK") + def pos = column[Option[Char]]("POS") + + // def sourceIds = column[String]("SOURCE_IDS") + def wordNetSenseNumber = column[Option[Int]]("WORD_NET_SENSE_NUMBER") + + type KgNodeTableColumnTypes = (String, Option[Char], Option[Int]) + + def * = (id, pos, wordNetSenseNumber) <> (create, extract.lift) + + private def create: KgNodeTableColumnTypes => KgNode = { + case (id, pos, wordNetSenseNumber) => + KgNode(id, None, List(), None, None, pos, List(), wordNetSenseNumber) + } + + private def extract: PartialFunction[KgNode, KgNodeTableColumnTypes] = { + case KgNode(id, inDegree, labels, outDegree, pageRank, pos, sourceIds, wordNetSenseNumber) => + (id, pos, wordNetSenseNumber) + } + } + + class KgEdgeTable(tag: Tag) extends Table[KgEdge](tag, "KgEdges") { + def id = column[String]("ID", O.PrimaryKey) +// labels: List[String], + def objectNodeId = column[String]("OBJECT_NODE_ID") + def predicate = column[String]("PREDICATE") +// sentences: List[String], +// sourceIds: List[String], + def subjectNodeId = column[String]("SUBJECT_NODE_ID") + + type KgEdgeTableColumnTypes = (String, String, String, String) + + def objectNode = foreignKey("OBJECT_NODE_FK", objectNodeId, KgNodes)(_.id) + def subjectNode = foreignKey("SUBJECT_NODE_FK", subjectNodeId, KgNodes)(_.id) + + def * = (id, objectNodeId, predicate, subjectNodeId) <> (create, extract.lift) + + private def create: KgEdgeTableColumnTypes => KgEdge = { + case (id, objectNodeId, predicate, subjectNodeId) => + KgEdge(id, List(), objectNodeId, predicate, List(), List(), subjectNodeId) + } + + private def extract: PartialFunction[KgEdge, KgEdgeTableColumnTypes] = { + case KgEdge(id, labels, object_, predicate, sentences, sourceIds, subject) => + (id, object_, predicate, subject) + } + } + + class KgSourceTable(tag: Tag) extends Table[KgSource](tag, "KgSources") { + def id = column[String]("ID", O.PrimaryKey) + def label = column[String]("LABEL") + + type KgSourceTableColumnTypes = (String, String) + + def * = (id, label) <> ((KgSource.apply _).tupled, KgSource.unapply) + } + + class KgNodeKgSource(tag: Tag) extends Table[(Int, String, String)](tag, "KgNodeKgSource") { + def id = column[Int]("ID", O.PrimaryKey, O.AutoInc) + + def kgNodeId = column[String]("KG_Node_ID") + def kgSourceId = column[String]("KG_SOURCE_ID") + + def * = (id, kgNodeId, kgSourceId) + + def kgNode = foreignKey("KG_Node_FK", kgNodeId, KgNodes)(_.id) + def kgSource = foreignKey("KG_SOURCE_FK", kgSourceId, KgSources)(_.id) + } +} diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala new file mode 100644 index 00000000..cb439159 --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -0,0 +1,48 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import com.google.inject.Inject +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.{KgNode, KgNodeContext, KgNodeLabel, KgNodeLabelContext} +import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath +import io.github.tetherlessworld.mcsapps.lib.kg.models.search.{KgSearchFacets, KgSearchQuery, KgSearchResult, KgSearchSort} +import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource +import io.github.tetherlessworld.mcsapps.lib.kg.stores.KgQueryStore +import javax.inject.Singleton +import play.api.db.slick.DatabaseConfigProvider + +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} +import scala.concurrent.Await +import scala.concurrent.duration._ + +@Singleton +final class PostgresKgQueryStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgQueryStore { + import profile.api._ + + override def getNode(id: String): Option[KgNode] = { + Await.result(db.run((for { node <- KgNodes if node.id === id } yield node).result.headOption), 10 seconds) + } + + override def getNodeContext(id: String): Option[KgNodeContext] = None + + override def getNodeLabel(label: String): Option[KgNodeLabel] = None + + override def getNodeLabelContext(label: String): Option[KgNodeLabelContext] = None + + override def getPath(id: String): Option[KgPath] = None + + override def getRandomNode: KgNode = KgNode("", None, List(), None, None, None, List(), None) + + override def getSourcesById: Map[String, KgSource] = Map() + + override def getTotalEdgesCount: Int = 0 + + override def getTotalNodesCount: Int = 0 + + override def isEmpty: Boolean = true + + override def search(limit: Int, offset: Int, query: KgSearchQuery, sorts: Option[List[KgSearchSort]]): List[KgSearchResult] = List() + + override def searchCount(query: KgSearchQuery): Int = 0 + + override def searchFacets(query: KgSearchQuery): KgSearchFacets = KgSearchFacets(List(), List()) +} From 0e5fb21c306f9363117081611e0bc4e449df8854 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Mon, 26 Oct 2020 01:24:15 -0400 Subject: [PATCH 03/51] alphabetical order --- app/kg/conf/application.conf | 10 +++++----- build.sbt | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/kg/conf/application.conf b/app/kg/conf/application.conf index 54adb1c3..3cba903e 100644 --- a/app/kg/conf/application.conf +++ b/app/kg/conf/application.conf @@ -5,12 +5,12 @@ kgStore = "neo4j" neo4j.password = "nC1aB4mji623s2Zs" neo4j.uri = "bolt://mcs-neo4j:7687" neo4j.user = "neo4j" -slick.dbs.default.profile = "slick.jdbc.PostgresProfile$" -slick.dbs.default.db.driver = "org.postgresql.Driver" -slick.dbs.default.db.url = "postgres://mcs-postgres:5432/kg" -slick.dbs.default.db.user = "mcs" -slick.dbs.default.db.password = "7EAdu7jJvZNxxrNZ" play.filters.enabled += "play.filters.cors.CORSFilter" play.filters.disabled += "play.filters.csrf.CSRFFilter" play.filters.disabled += "play.filters.hosts.AllowedHostsFilter" play.modules.enabled += "stores.KgStoresModule" +slick.dbs.default.db.driver = "org.postgresql.Driver" +slick.dbs.default.db.password = "7EAdu7jJvZNxxrNZ" +slick.dbs.default.db.user = "mcs" +slick.dbs.default.db.url = "postgres://mcs-postgres:5432/kg" +slick.dbs.default.profile = "slick.jdbc.PostgresProfile$" diff --git a/build.sbt b/build.sbt index 18791897..47087fe6 100644 --- a/build.sbt +++ b/build.sbt @@ -47,12 +47,12 @@ lazy val kgLib = libraryDependencies ++= Seq( // Implement search in the MemStore (and thus the TestStore) "com.outr" %% "lucene4s" % "1.9.1", + "com.typesafe.play" %% "play-slick" % "5.0.0", + "com.typesafe.play" %% "play-slick-evolutions" % "5.0.0", "io.github.tetherless-world" %% "twxplore-base" % twxploreVersion, "io.github.tetherless-world" %% "twxplore-test" % twxploreVersion % Test, "me.tongfei" % "progressbar" % "0.8.1", "org.neo4j.driver" % "neo4j-java-driver" % "4.0.1", - "com.typesafe.play" %% "play-slick" % "5.0.0", - "com.typesafe.play" %% "play-slick-evolutions" % "5.0.0", "org.postgresql" % "postgresql" % "42.2.18" ), maintainer := maintainerValue, From 04159389ea8aca94105eea3e399528caf2f94bd3 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Mon, 26 Oct 2020 01:50:59 -0400 Subject: [PATCH 04/51] sql conventions for table and column names --- .../postgres/AbstractPostgresKgStore.scala | 75 ++++++------------- 1 file changed, 24 insertions(+), 51 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index b1db57cc..42c25e72 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,8 +1,5 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge -import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode -import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import slick.jdbc.PostgresProfile @@ -14,78 +11,54 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC lazy val KgEdges = TableQuery[KgEdgeTable] lazy val KgSources = TableQuery[KgSourceTable] - private class KgNodeTable(tag: Tag) extends Table[KgNode](tag, "KgNodes") { - def id = column[String]("ID", O.PrimaryKey) + private class KgNodeTable(tag: Tag) extends Table[(String, Option[Char], Option[Int])](tag, "kg_node") { + + def id = column[String]("id", O.PrimaryKey) // def inDegree = column[Option[Int]]("IN_DEGREE") // def labels = column[String]("LABELS") // def outDegree = column[Option[Int]]("OUT_DEGREE") // def pageRank = column[Option[Double]]("PAGERANK") - def pos = column[Option[Char]]("POS") + def pos = column[Option[Char]]("pos") // def sourceIds = column[String]("SOURCE_IDS") - def wordNetSenseNumber = column[Option[Int]]("WORD_NET_SENSE_NUMBER") - - type KgNodeTableColumnTypes = (String, Option[Char], Option[Int]) - - def * = (id, pos, wordNetSenseNumber) <> (create, extract.lift) - - private def create: KgNodeTableColumnTypes => KgNode = { - case (id, pos, wordNetSenseNumber) => - KgNode(id, None, List(), None, None, pos, List(), wordNetSenseNumber) - } + def wordNetSenseNumber = column[Option[Int]]("word_net_sense_number") - private def extract: PartialFunction[KgNode, KgNodeTableColumnTypes] = { - case KgNode(id, inDegree, labels, outDegree, pageRank, pos, sourceIds, wordNetSenseNumber) => - (id, pos, wordNetSenseNumber) - } + def * = (id, pos, wordNetSenseNumber) } - class KgEdgeTable(tag: Tag) extends Table[KgEdge](tag, "KgEdges") { - def id = column[String]("ID", O.PrimaryKey) + private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String)](tag, "kg_edge") { + def id = column[String]("id", O.PrimaryKey) // labels: List[String], - def objectNodeId = column[String]("OBJECT_NODE_ID") - def predicate = column[String]("PREDICATE") + def objectNodeId = column[String]("object_node_id") + def predicate = column[String]("predicate") // sentences: List[String], // sourceIds: List[String], - def subjectNodeId = column[String]("SUBJECT_NODE_ID") + def subjectNodeId = column[String]("subject_node_id") - type KgEdgeTableColumnTypes = (String, String, String, String) + def * = (id, objectNodeId, predicate, subjectNodeId) - def objectNode = foreignKey("OBJECT_NODE_FK", objectNodeId, KgNodes)(_.id) - def subjectNode = foreignKey("SUBJECT_NODE_FK", subjectNodeId, KgNodes)(_.id) + def objectNode = foreignKey("object_node_fk", objectNodeId, KgNodes)(_.id) + def subjectNode = foreignKey("subject_node_fk", subjectNodeId, KgNodes)(_.id) - def * = (id, objectNodeId, predicate, subjectNodeId) <> (create, extract.lift) - - private def create: KgEdgeTableColumnTypes => KgEdge = { - case (id, objectNodeId, predicate, subjectNodeId) => - KgEdge(id, List(), objectNodeId, predicate, List(), List(), subjectNodeId) - } - - private def extract: PartialFunction[KgEdge, KgEdgeTableColumnTypes] = { - case KgEdge(id, labels, object_, predicate, sentences, sourceIds, subject) => - (id, object_, predicate, subject) - } } - class KgSourceTable(tag: Tag) extends Table[KgSource](tag, "KgSources") { - def id = column[String]("ID", O.PrimaryKey) - def label = column[String]("LABEL") - - type KgSourceTableColumnTypes = (String, String) + private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { + def id = column[String]("id", O.PrimaryKey) + def label = column[String]("label") - def * = (id, label) <> ((KgSource.apply _).tupled, KgSource.unapply) + def * = (id, label) } - class KgNodeKgSource(tag: Tag) extends Table[(Int, String, String)](tag, "KgNodeKgSource") { - def id = column[Int]("ID", O.PrimaryKey, O.AutoInc) + private class KgNodeKgSource(tag: Tag) extends Table[(Int, String, String)](tag, "kg_node_kg_source") { + def id = column[Int]("id", O.PrimaryKey, O.AutoInc) - def kgNodeId = column[String]("KG_Node_ID") - def kgSourceId = column[String]("KG_SOURCE_ID") + def kgNodeId = column[String]("kg_node_id") + def kgSourceId = column[String]("kg_source_id") def * = (id, kgNodeId, kgSourceId) - def kgNode = foreignKey("KG_Node_FK", kgNodeId, KgNodes)(_.id) - def kgSource = foreignKey("KG_SOURCE_FK", kgSourceId, KgSources)(_.id) + def kgNode = foreignKey("kg_node_fk", kgNodeId, KgNodes)(_.id) + def kgSource = foreignKey("kg_source_fk", kgSourceId, KgSources)(_.id) } } From 0d12ee70978280c2a69a72b7a072d23fe8f79461 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Mon, 26 Oct 2020 02:58:33 -0400 Subject: [PATCH 05/51] add tables labels and sources and revise existing --- .../postgres/AbstractPostgresKgStore.scala | 91 +++++++++++++------ 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 42c25e72..0153698d 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -6,59 +6,90 @@ import slick.jdbc.PostgresProfile abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { import profile.api._ + + lazy val kgEdges = TableQuery[KgEdgeTable] + lazy val kgEdgeLabels = TableQuery[KgEdgeLabelTable] + lazy val kgEdgeKgSources = TableQuery[KgEdgeKgSourceTable] + lazy val kgNodes = TableQuery[KgNodeTable] + lazy val kgNodeLabels = TableQuery[KgNodeLabelTable] + lazy val kgNodeKgSources = TableQuery[KgNodeKgSourceTable] + lazy val kgSources = TableQuery[KgSourceTable] + + private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String, String)](tag, "kg_edge") { + def id = column[String]("id", O.PrimaryKey) + def objectNodeId = column[String]("object_node_id") + def predicate = column[String]("predicate") + def sentences = column[String]("sentences") + def subjectNodeId = column[String]("subject_node_id") - lazy val KgNodes = TableQuery[KgNodeTable] - lazy val KgEdges = TableQuery[KgEdgeTable] - lazy val KgSources = TableQuery[KgSourceTable] + def * = (id, objectNodeId, predicate, sentences, subjectNodeId) - private class KgNodeTable(tag: Tag) extends Table[(String, Option[Char], Option[Int])](tag, "kg_node") { + def objectNode = foreignKey("object_node_fk", objectNodeId, KgNodes)(_.id) + def subjectNode = foreignKey("subject_node_fk", subjectNodeId, KgNodes)(_.id) - def id = column[String]("id", O.PrimaryKey) + def unique_constraint = index("idx_kg_edge_unique", (objectNodeId, subjectNodeId, predicate), unique = true) + } - // def inDegree = column[Option[Int]]("IN_DEGREE") - // def labels = column[String]("LABELS") - // def outDegree = column[Option[Int]]("OUT_DEGREE") - // def pageRank = column[Option[Double]]("PAGERANK") - def pos = column[Option[Char]]("pos") + private class KgEdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_label") { + def kgEdgeId = column[String]("kg_edge_id") + def label = column[String]("label") - // def sourceIds = column[String]("SOURCE_IDS") - def wordNetSenseNumber = column[Option[Int]]("word_net_sense_number") + def * = (kgEdgeId, label) + + def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, KgEdges)(_.id) - def * = (id, pos, wordNetSenseNumber) + def pk = primaryKey("pk_kg_edge_label", (kgEdgeId, label)) } - private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String)](tag, "kg_edge") { - def id = column[String]("id", O.PrimaryKey) -// labels: List[String], - def objectNodeId = column[String]("object_node_id") - def predicate = column[String]("predicate") -// sentences: List[String], -// sourceIds: List[String], - def subjectNodeId = column[String]("subject_node_id") + private class KgEdgeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_kg_source") { + def kgEdgeId = column[String]("kg_edge_id") + def kgSourceId = column[String]("kg_source_id") - def * = (id, objectNodeId, predicate, subjectNodeId) + def * = (kgEdgeId, kgSourceId) - def objectNode = foreignKey("object_node_fk", objectNodeId, KgNodes)(_.id) - def subjectNode = foreignKey("subject_node_fk", subjectNodeId, KgNodes)(_.id) + def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, KgEdges)(_.id) + def pk = primaryKey("pk_kg_edge_kg_source", (kgEdgeId, kgSourceId)) } - private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { + private class KgNodeTable(tag: Tag) extends Table[(String, Option[Int], Option[Int], Option[Double], Option[Char], Option[Int])](tag, "kg_node") { def id = column[String]("id", O.PrimaryKey) - def label = column[String]("label") + def inDegree = column[Option[Int]]("in_degree") + def outDegree = column[Option[Int]]("out_degree") + def pageRank = column[Option[Double]]("page_rank") + def pos = column[Option[Char]]("pos") + def wordNetSenseNumber = column[Option[Int]]("word_net_sense_number") - def * = (id, label) + def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) } - private class KgNodeKgSource(tag: Tag) extends Table[(Int, String, String)](tag, "kg_node_kg_source") { - def id = column[Int]("id", O.PrimaryKey, O.AutoInc) + private class KgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_label") { + def kgNodeId = column[String]("kg_node_id") + def label = column[String]("label") + + def * = (kgNodeId, label) + def kgNode = foreignKey("kg_node_fk", kgNodeId, KgNodes)(_.id) + + def pk = primaryKey("pk_kg_node_label", (kgNodeId, label)) + } + + private class KgNodeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_kg_node_source") { def kgNodeId = column[String]("kg_node_id") def kgSourceId = column[String]("kg_source_id") - def * = (id, kgNodeId, kgSourceId) + def * = (kgNodeId, kgSourceId) def kgNode = foreignKey("kg_node_fk", kgNodeId, KgNodes)(_.id) def kgSource = foreignKey("kg_source_fk", kgSourceId, KgSources)(_.id) + + def pk = primaryKey("pk_kg_node_kg_source", (kgNodeId, kgSourceId)) + } + + private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { + def id = column[String]("id", O.PrimaryKey) + def label = column[String]("label") + + def * = (id, label) } } From 675b46a00eb4f10a6dfcf4e1fea559ba4c10b0e3 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 27 Oct 2020 00:56:01 -0400 Subject: [PATCH 06/51] use char(1) and short instead of int --- .../postgres/AbstractPostgresKgStore.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 0153698d..399864bb 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -6,7 +6,7 @@ import slick.jdbc.PostgresProfile abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { import profile.api._ - + lazy val kgEdges = TableQuery[KgEdgeTable] lazy val kgEdgeLabels = TableQuery[KgEdgeLabelTable] lazy val kgEdgeKgSources = TableQuery[KgEdgeKgSourceTable] @@ -24,8 +24,8 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (id, objectNodeId, predicate, sentences, subjectNodeId) - def objectNode = foreignKey("object_node_fk", objectNodeId, KgNodes)(_.id) - def subjectNode = foreignKey("subject_node_fk", subjectNodeId, KgNodes)(_.id) + def objectNode = foreignKey("object_node_fk", objectNodeId, kgNodes)(_.id) + def subjectNode = foreignKey("subject_node_fk", subjectNodeId, kgNodes)(_.id) def unique_constraint = index("idx_kg_edge_unique", (objectNodeId, subjectNodeId, predicate), unique = true) } @@ -36,7 +36,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (kgEdgeId, label) - def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, KgEdges)(_.id) + def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) def pk = primaryKey("pk_kg_edge_label", (kgEdgeId, label)) } @@ -47,18 +47,18 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (kgEdgeId, kgSourceId) - def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, KgEdges)(_.id) + def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) def pk = primaryKey("pk_kg_edge_kg_source", (kgEdgeId, kgSourceId)) } - private class KgNodeTable(tag: Tag) extends Table[(String, Option[Int], Option[Int], Option[Double], Option[Char], Option[Int])](tag, "kg_node") { + private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Double], Option[Char], Option[Short])](tag, "kg_node") { def id = column[String]("id", O.PrimaryKey) - def inDegree = column[Option[Int]]("in_degree") - def outDegree = column[Option[Int]]("out_degree") + def inDegree = column[Option[Short]]("in_degree") + def outDegree = column[Option[Short]]("out_degree") def pageRank = column[Option[Double]]("page_rank") - def pos = column[Option[Char]]("pos") - def wordNetSenseNumber = column[Option[Int]]("word_net_sense_number") + def pos = column[Option[Char]]("pos", O.Length(1)) + def wordNetSenseNumber = column[Option[Short]]("word_net_sense_number") def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) } @@ -69,7 +69,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (kgNodeId, label) - def kgNode = foreignKey("kg_node_fk", kgNodeId, KgNodes)(_.id) + def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) def pk = primaryKey("pk_kg_node_label", (kgNodeId, label)) } @@ -80,8 +80,8 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (kgNodeId, kgSourceId) - def kgNode = foreignKey("kg_node_fk", kgNodeId, KgNodes)(_.id) - def kgSource = foreignKey("kg_source_fk", kgSourceId, KgSources)(_.id) + def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) + def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) def pk = primaryKey("pk_kg_node_kg_source", (kgNodeId, kgSourceId)) } From 18a8cf621fe09bdd17cde49cc4a486159aa1ff90 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 27 Oct 2020 02:13:08 -0400 Subject: [PATCH 07/51] add kgNodeLabel tables --- .../postgres/AbstractPostgresKgStore.scala | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 399864bb..3f09e47b 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -12,6 +12,9 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC lazy val kgEdgeKgSources = TableQuery[KgEdgeKgSourceTable] lazy val kgNodes = TableQuery[KgNodeTable] lazy val kgNodeLabels = TableQuery[KgNodeLabelTable] + lazy val kgNodeLabelEdges = TableQuery[KgNodeLabelEdgeTable] + lazy val kgNodeLabelEdgeKgSources = TableQuery[KgNodeLabelEdgeKgSourceTable] + lazy val kgNodeLabelKgSource = TableQuery[KgNodeLabelKgSourceTable] lazy val kgNodeKgSources = TableQuery[KgNodeKgSourceTable] lazy val kgSources = TableQuery[KgSourceTable] @@ -63,7 +66,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) } - private class KgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_label") { + private class KgNodeKgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_kg_node_label") { def kgNodeId = column[String]("kg_node_id") def label = column[String]("label") @@ -86,6 +89,50 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("pk_kg_node_kg_source", (kgNodeId, kgSourceId)) } + private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Double])](tag, "kg_node_label") { + def label = column[String]("label", O.PrimaryKey) + def pageRank = column[Option[Double]]("page_rank") + + def * = (label, pageRank) + } + + private class KgNodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "kg_node_label_edge") { + def id = column[Int]("id", O.PrimaryKey, O.AutoInc) + def objectKgNodeLabelLabel = column[String]("object_kg_node_label_label") + def subjectKgNodeLabelLabel = column[String]("subject_kg_node_label_label") + + def * = (id, objectKgNodeLabelLabel, subjectKgNodeLabelLabel) + + def objectKgNodeLabel = foreignKey("object_kg_node_label_fk", objectKgNodeLabelLabel, kgNodeLabels)(_.label) + def subjectKgNodeLabel = foreignKey("subject_kg_node_label_fk", subjectKgNodeLabelLabel, kgNodeLabels)(_.label) + + def unique_constraint = index("idx_kg_node_label_edge_unique", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) + } + + private class KgNodeLabelEdgeKgSourceTable(tag: Tag) extends Table[(Int, String)](tag, "kg_node_label_edge_kg_source") { + def kgNodeLabelEdgeId = column[Int]("kg_node_label_edge_id") + def kgSourceId = column[String]("kg_source_id") + + def * = (kgNodeLabelEdgeId, kgSourceId) + + def kgNodeLabelEdge = foreignKey("kg_node_label_edge_fk", kgNodeLabelEdgeId, kgNodeLabelEdges)(_.id) + def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) + + def pk = primaryKey("pk_kg_node_label_edge_kg_source", (kgNodeLabelEdgeId, kgSourceId)) + } + + private class KgNodeLabelKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_label_kg_source") { + def kgNodeLabelLabel = column[String]("kg_node_label_label") + def kgSourceId = column[String]("kg_source_id") + + def * = (kgNodeLabelLabel, kgSourceId) + + def kgNodeLabel = foreignKey("kg_node_label_fk", kgNodeLabelLabel, kgNodeLabels)(_.label) + def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) + + def pk = primaryKey("pk_kg_node_label_kg_source", (kgNodeLabelLabel, kgSourceId)) + } + private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") From b8a55554e8090cf6089055938aa1af7db5c32798 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 27 Oct 2020 02:18:18 -0400 Subject: [PATCH 08/51] consistent naming --- .../postgres/AbstractPostgresKgStore.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 3f09e47b..913aded5 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -20,17 +20,17 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String, String)](tag, "kg_edge") { def id = column[String]("id", O.PrimaryKey) - def objectNodeId = column[String]("object_node_id") + def objectKgNodeId = column[String]("object_kg_node_id") def predicate = column[String]("predicate") def sentences = column[String]("sentences") - def subjectNodeId = column[String]("subject_node_id") + def subjectKgNodeId = column[String]("subject_node_id") - def * = (id, objectNodeId, predicate, sentences, subjectNodeId) + def * = (id, objectKgNodeId, predicate, sentences, subjectKgNodeId) - def objectNode = foreignKey("object_node_fk", objectNodeId, kgNodes)(_.id) - def subjectNode = foreignKey("subject_node_fk", subjectNodeId, kgNodes)(_.id) + def objectNode = foreignKey("object_kg_node_fk", objectKgNodeId, kgNodes)(_.id) + def subjectNode = foreignKey("subject_kg_node_fk", subjectKgNodeId, kgNodes)(_.id) - def unique_constraint = index("idx_kg_edge_unique", (objectNodeId, subjectNodeId, predicate), unique = true) + def unique_constraint = index("kg_edge_unique_idx", (objectKgNodeId, subjectKgNodeId, predicate), unique = true) } private class KgEdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_label") { @@ -41,7 +41,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) - def pk = primaryKey("pk_kg_edge_label", (kgEdgeId, label)) + def pk = primaryKey("kg_edge_label_pk", (kgEdgeId, label)) } private class KgEdgeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_kg_source") { @@ -52,7 +52,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) - def pk = primaryKey("pk_kg_edge_kg_source", (kgEdgeId, kgSourceId)) + def pk = primaryKey("kg_edge_kg_source_pk", (kgEdgeId, kgSourceId)) } private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Double], Option[Char], Option[Short])](tag, "kg_node") { @@ -74,7 +74,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) - def pk = primaryKey("pk_kg_node_label", (kgNodeId, label)) + def pk = primaryKey("kg_node_label_pk", (kgNodeId, label)) } private class KgNodeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_kg_node_source") { @@ -86,7 +86,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("pk_kg_node_kg_source", (kgNodeId, kgSourceId)) + def pk = primaryKey("kg_node_kg_source_pk", (kgNodeId, kgSourceId)) } private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Double])](tag, "kg_node_label") { @@ -106,7 +106,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def objectKgNodeLabel = foreignKey("object_kg_node_label_fk", objectKgNodeLabelLabel, kgNodeLabels)(_.label) def subjectKgNodeLabel = foreignKey("subject_kg_node_label_fk", subjectKgNodeLabelLabel, kgNodeLabels)(_.label) - def unique_constraint = index("idx_kg_node_label_edge_unique", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) + def unique_constraint = index("kg_node_label_edge_unique_idx", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) } private class KgNodeLabelEdgeKgSourceTable(tag: Tag) extends Table[(Int, String)](tag, "kg_node_label_edge_kg_source") { @@ -118,7 +118,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgNodeLabelEdge = foreignKey("kg_node_label_edge_fk", kgNodeLabelEdgeId, kgNodeLabelEdges)(_.id) def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("pk_kg_node_label_edge_kg_source", (kgNodeLabelEdgeId, kgSourceId)) + def pk = primaryKey("kg_node_label_edge_kg_source_pk", (kgNodeLabelEdgeId, kgSourceId)) } private class KgNodeLabelKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_label_kg_source") { @@ -130,7 +130,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def kgNodeLabel = foreignKey("kg_node_label_fk", kgNodeLabelLabel, kgNodeLabels)(_.label) def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("pk_kg_node_label_kg_source", (kgNodeLabelLabel, kgSourceId)) + def pk = primaryKey("kg_node_label_kg_source_pk", (kgNodeLabelLabel, kgSourceId)) } private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { From b6721f30cf1d1fdb823e5d5b1f8c21e3dcb80022 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 28 Oct 2020 01:28:26 -0400 Subject: [PATCH 09/51] change page_rank to float (postgres real) --- .../lib/kg/stores/postgres/AbstractPostgresKgStore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 913aded5..37e62251 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -55,11 +55,11 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("kg_edge_kg_source_pk", (kgEdgeId, kgSourceId)) } - private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Double], Option[Char], Option[Short])](tag, "kg_node") { + private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Float], Option[Char], Option[Short])](tag, "kg_node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") - def pageRank = column[Option[Double]]("page_rank") + def pageRank = column[Option[Float]]("page_rank") def pos = column[Option[Char]]("pos", O.Length(1)) def wordNetSenseNumber = column[Option[Short]]("word_net_sense_number") @@ -89,9 +89,9 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("kg_node_kg_source_pk", (kgNodeId, kgSourceId)) } - private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Double])](tag, "kg_node_label") { + private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "kg_node_label") { def label = column[String]("label", O.PrimaryKey) - def pageRank = column[Option[Double]]("page_rank") + def pageRank = column[Option[Float]]("page_rank") def * = (label, pageRank) } From c79d3b107c4c3fd4ab229058738866e7b8b96e40 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 28 Oct 2020 01:33:56 -0400 Subject: [PATCH 10/51] remove kg and add x for join tables --- .../postgres/AbstractPostgresKgStore.scala | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 37e62251..d66e42b3 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -18,44 +18,44 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC lazy val kgNodeKgSources = TableQuery[KgNodeKgSourceTable] lazy val kgSources = TableQuery[KgSourceTable] - private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String, String)](tag, "kg_edge") { + private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String, String)](tag, "edge") { def id = column[String]("id", O.PrimaryKey) - def objectKgNodeId = column[String]("object_kg_node_id") + def objectKgNodeId = column[String]("object_node_id") def predicate = column[String]("predicate") def sentences = column[String]("sentences") def subjectKgNodeId = column[String]("subject_node_id") def * = (id, objectKgNodeId, predicate, sentences, subjectKgNodeId) - def objectNode = foreignKey("object_kg_node_fk", objectKgNodeId, kgNodes)(_.id) - def subjectNode = foreignKey("subject_kg_node_fk", subjectKgNodeId, kgNodes)(_.id) + def objectNode = foreignKey("object_node_fk", objectKgNodeId, kgNodes)(_.id) + def subjectNode = foreignKey("subject_node_fk", subjectKgNodeId, kgNodes)(_.id) def unique_constraint = index("kg_edge_unique_idx", (objectKgNodeId, subjectKgNodeId, predicate), unique = true) } - private class KgEdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_label") { - def kgEdgeId = column[String]("kg_edge_id") + private class KgEdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { + def kgEdgeId = column[String]("edge_id") def label = column[String]("label") def * = (kgEdgeId, label) - def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) + def kgEdge = foreignKey("edge_fk", kgEdgeId, kgEdges)(_.id) - def pk = primaryKey("kg_edge_label_pk", (kgEdgeId, label)) + def pk = primaryKey("edge_label_pk", (kgEdgeId, label)) } - private class KgEdgeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_edge_kg_source") { - def kgEdgeId = column[String]("kg_edge_id") - def kgSourceId = column[String]("kg_source_id") + private class KgEdgeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { + def kgEdgeId = column[String]("edge_id") + def kgSourceId = column[String]("source_id") def * = (kgEdgeId, kgSourceId) - def kgEdge = foreignKey("kg_edge_fk", kgEdgeId, kgEdges)(_.id) + def kgEdge = foreignKey("edge_fk", kgEdgeId, kgEdges)(_.id) - def pk = primaryKey("kg_edge_kg_source_pk", (kgEdgeId, kgSourceId)) + def pk = primaryKey("edge_source_pk", (kgEdgeId, kgSourceId)) } - private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Float], Option[Char], Option[Short])](tag, "kg_node") { + private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Float], Option[Char], Option[Short])](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") @@ -66,74 +66,74 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) } - private class KgNodeKgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_kg_node_label") { - def kgNodeId = column[String]("kg_node_id") + private class KgNodeKgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { + def kgNodeId = column[String]("node_id") def label = column[String]("label") def * = (kgNodeId, label) - def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) + def kgNode = foreignKey("node_fk", kgNodeId, kgNodes)(_.id) - def pk = primaryKey("kg_node_label_pk", (kgNodeId, label)) + def pk = primaryKey("node_label_pk", (kgNodeId, label)) } - private class KgNodeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_kg_node_source") { - def kgNodeId = column[String]("kg_node_id") - def kgSourceId = column[String]("kg_source_id") + private class KgNodeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { + def kgNodeId = column[String]("node_id") + def kgSourceId = column[String]("source_id") def * = (kgNodeId, kgSourceId) - def kgNode = foreignKey("kg_node_fk", kgNodeId, kgNodes)(_.id) - def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) + def kgNode = foreignKey("node_fk", kgNodeId, kgNodes)(_.id) + def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("kg_node_kg_source_pk", (kgNodeId, kgSourceId)) + def pk = primaryKey("node_source_pk", (kgNodeId, kgSourceId)) } - private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "kg_node_label") { + private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") def * = (label, pageRank) } - private class KgNodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "kg_node_label_edge") { + private class KgNodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) - def objectKgNodeLabelLabel = column[String]("object_kg_node_label_label") - def subjectKgNodeLabelLabel = column[String]("subject_kg_node_label_label") + def objectKgNodeLabelLabel = column[String]("object_node_label_label") + def subjectKgNodeLabelLabel = column[String]("subject_node_label_label") def * = (id, objectKgNodeLabelLabel, subjectKgNodeLabelLabel) - def objectKgNodeLabel = foreignKey("object_kg_node_label_fk", objectKgNodeLabelLabel, kgNodeLabels)(_.label) - def subjectKgNodeLabel = foreignKey("subject_kg_node_label_fk", subjectKgNodeLabelLabel, kgNodeLabels)(_.label) + def objectKgNodeLabel = foreignKey("object_node_label_fk", objectKgNodeLabelLabel, kgNodeLabels)(_.label) + def subjectKgNodeLabel = foreignKey("subject_node_label_fk", subjectKgNodeLabelLabel, kgNodeLabels)(_.label) - def unique_constraint = index("kg_node_label_edge_unique_idx", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) + def unique_constraint = index("node_label_edge_unique_idx", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) } - private class KgNodeLabelEdgeKgSourceTable(tag: Tag) extends Table[(Int, String)](tag, "kg_node_label_edge_kg_source") { - def kgNodeLabelEdgeId = column[Int]("kg_node_label_edge_id") - def kgSourceId = column[String]("kg_source_id") + private class KgNodeLabelEdgeKgSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { + def kgNodeLabelEdgeId = column[Int]("node_label_edge_id") + def kgSourceId = column[String]("source_id") def * = (kgNodeLabelEdgeId, kgSourceId) - def kgNodeLabelEdge = foreignKey("kg_node_label_edge_fk", kgNodeLabelEdgeId, kgNodeLabelEdges)(_.id) - def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) + def kgNodeLabelEdge = foreignKey("node_label_edge_fk", kgNodeLabelEdgeId, kgNodeLabelEdges)(_.id) + def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("kg_node_label_edge_kg_source_pk", (kgNodeLabelEdgeId, kgSourceId)) + def pk = primaryKey("node_label_edge_source_pk", (kgNodeLabelEdgeId, kgSourceId)) } - private class KgNodeLabelKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_node_label_kg_source") { - def kgNodeLabelLabel = column[String]("kg_node_label_label") - def kgSourceId = column[String]("kg_source_id") + private class KgNodeLabelKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { + def kgNodeLabelLabel = column[String]("node_label_label") + def kgSourceId = column[String]("source_id") def * = (kgNodeLabelLabel, kgSourceId) - def kgNodeLabel = foreignKey("kg_node_label_fk", kgNodeLabelLabel, kgNodeLabels)(_.label) - def kgSource = foreignKey("kg_source_fk", kgSourceId, kgSources)(_.id) + def kgNodeLabel = foreignKey("node_label_fk", kgNodeLabelLabel, kgNodeLabels)(_.label) + def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) - def pk = primaryKey("kg_node_label_kg_source_pk", (kgNodeLabelLabel, kgSourceId)) + def pk = primaryKey("node_label_source_pk", (kgNodeLabelLabel, kgSourceId)) } - private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "kg_source") { + private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") From c2a25db19902df6949b86350dc9526db9b4f8efa Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Thu, 29 Oct 2020 08:44:05 -0400 Subject: [PATCH 11/51] Add case classes for KgNode and KgEdge. Add DBIO run methods --- .../postgres/AbstractPostgresKgStore.scala | 26 ++++++++++++++++--- .../postgres/PostgresKgQueryStore.scala | 5 +--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index d66e42b3..553b3b39 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -3,10 +3,16 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import slick.jdbc.PostgresProfile +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} + abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { import profile.api._ + protected val SentencesDelimChar = '|' + protected val SentencesDelimString = SentencesDelimChar.toString + lazy val kgEdges = TableQuery[KgEdgeTable] lazy val kgEdgeLabels = TableQuery[KgEdgeLabelTable] lazy val kgEdgeKgSources = TableQuery[KgEdgeKgSourceTable] @@ -16,16 +22,28 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC lazy val kgNodeLabelEdgeKgSources = TableQuery[KgNodeLabelEdgeKgSourceTable] lazy val kgNodeLabelKgSource = TableQuery[KgNodeLabelKgSourceTable] lazy val kgNodeKgSources = TableQuery[KgNodeKgSourceTable] + lazy val kgNodeKgNodeLabels = TableQuery[KgNodeKgNodeLabelTable] lazy val kgSources = TableQuery[KgSourceTable] - private class KgEdgeTable(tag: Tag) extends Table[(String, String, String, String, String)](tag, "edge") { + protected def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { + db.run(a.transactionally) + } + + protected def runSyncTransaction[R](a: DBIOAction[R, NoStream, Nothing], duration: Duration = Duration.Inf): R = { + Await.result(runTransaction(a), duration) + } + + protected final case class KgEdgeRow(id: String, objectKgNodeId: String, predicate: String, sentences: String, subjectKgNodeId: String) + protected final case class KgNodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) + + private class KgEdgeTable(tag: Tag) extends Table[KgEdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) def objectKgNodeId = column[String]("object_node_id") def predicate = column[String]("predicate") def sentences = column[String]("sentences") def subjectKgNodeId = column[String]("subject_node_id") - def * = (id, objectKgNodeId, predicate, sentences, subjectKgNodeId) + def * = (id, objectKgNodeId, predicate, sentences, subjectKgNodeId) <> (KgEdgeRow.tupled, KgEdgeRow.unapply) def objectNode = foreignKey("object_node_fk", objectKgNodeId, kgNodes)(_.id) def subjectNode = foreignKey("subject_node_fk", subjectKgNodeId, kgNodes)(_.id) @@ -55,7 +73,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("edge_source_pk", (kgEdgeId, kgSourceId)) } - private class KgNodeTable(tag: Tag) extends Table[(String, Option[Short], Option[Short], Option[Float], Option[Char], Option[Short])](tag, "node") { + private class KgNodeTable(tag: Tag) extends Table[KgNodeRow](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") @@ -63,7 +81,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pos = column[Option[Char]]("pos", O.Length(1)) def wordNetSenseNumber = column[Option[Short]]("word_net_sense_number") - def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) + def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (KgNodeRow.tupled, KgNodeRow.unapply) } private class KgNodeKgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index cb439159..477e65eb 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -10,16 +10,13 @@ import javax.inject.Singleton import play.api.db.slick.DatabaseConfigProvider import scala.concurrent.ExecutionContext -import scala.util.{Failure, Success} -import scala.concurrent.Await -import scala.concurrent.duration._ @Singleton final class PostgresKgQueryStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgQueryStore { import profile.api._ override def getNode(id: String): Option[KgNode] = { - Await.result(db.run((for { node <- KgNodes if node.id === id } yield node).result.headOption), 10 seconds) + runSyncTransaction((for { node <- kgNodes if node.id === id } yield node).result.headOption) } override def getNodeContext(id: String): Option[KgNodeContext] = None From e1faf5820888d4f70f6cbcc256d99a1b237d646c Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Thu, 29 Oct 2020 08:44:53 -0400 Subject: [PATCH 12/51] implementing KgNode and KgEdge methods of PostgresKgCommandStore --- .../postgres/PostgresKgCommandStore.scala | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala new file mode 100644 index 00000000..ee021548 --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -0,0 +1,81 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import com.google.inject.Inject +import io.github.tetherlessworld.mcsapps.lib.kg.formats.kgtk.KgtkEdgeWithNodes +import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode +import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath +import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreTransaction} +import play.api.db.slick.DatabaseConfigProvider + +import scala.concurrent.ExecutionContext + +class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgCommandStore { + import profile.api._ + + private final class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { + private final implicit class KgEdgeWrapper(edge: KgEdge) { + def toRow: KgEdgeRow = KgEdgeRow( + id = edge.id, + objectKgNodeId = edge.`object`, + predicate = edge.predicate, + sentences = edge.sentences.mkString(SentencesDelimString), + subjectKgNodeId = edge.subject + ) + } + + private final implicit class KgNodeWrapper(node: KgNode) { + def toRow: KgNodeRow = KgNodeRow( + id = node.id, + inDegree = node.inDegree.map(_.toShort), + outDegree = node.outDegree.map(_.toShort), + pageRank = node.pageRank.map(_.toFloat), + pos = node.pos, + wordNetSenseNumber = node.wordNetSenseNumber.map(_.toShort) + ) + } + + override def clear(): Unit = { +// TODO + } + + final private def generateEdgeInsert(edge: KgEdge) = + List(kgEdges += edge.toRow) ++ + edge.labels.map(label => kgEdgeLabels += (edge.id, label)) ++ + edge.sourceIds.map(sourceId => kgEdgeKgSources += (edge.id, sourceId)) + + final private def generateNodeInsert(node: KgNode) = + List(kgNodes += node.toRow) ++ + node.labels.map(label => kgNodeKgNodeLabels += (node.id, label)) ++ + node.sourceIds.map(sourceId => kgNodeKgSources += (node.id, sourceId)) + + override def putEdges(edges: Iterator[KgEdge]) = + runSyncTransaction(DBIO.sequence(edges.flatMap(generateEdgeInsert))) + + override def putKgtkEdgesWithNodes(edgesWithNodes: Iterator[KgtkEdgeWithNodes]): Unit = + runSyncTransaction(DBIO.sequence( + edgesWithNodes.flatMap { edge => + edge.nodes.flatMap(generateNodeInsert) ++ + generateEdgeInsert(edge.edge) + } + )) + + override def putNodes(nodes: Iterator[KgNode]): Unit = + runSyncTransaction(DBIO.sequence(nodes.flatMap(generateNodeInsert))) + + override def putPaths(paths: Iterator[KgPath]): Unit = { +// TODO + } + + override def putSources(sources: Iterator[KgSource]): Unit = + runSyncTransaction(DBIO.sequence(sources.map { source => + kgSources += (source.id, source.label) + })) + + override def close(): Unit = Unit + } + + override def beginTransaction: KgCommandStoreTransaction = + new PostgresKgCommandStoreTransaction +} From 7641970f1b3e5bcd493cd02fd7da6e3042d40374 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 00:51:04 -0400 Subject: [PATCH 13/51] remove kg prefixes --- .../postgres/AbstractPostgresKgStore.scala | 140 +++++++++--------- .../postgres/PostgresKgCommandStore.scala | 35 +++-- .../postgres/PostgresKgQueryStore.scala | 2 +- 3 files changed, 88 insertions(+), 89 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 553b3b39..02714d8f 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -11,19 +11,19 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC import profile.api._ protected val SentencesDelimChar = '|' - protected val SentencesDelimString = SentencesDelimChar.toString - - lazy val kgEdges = TableQuery[KgEdgeTable] - lazy val kgEdgeLabels = TableQuery[KgEdgeLabelTable] - lazy val kgEdgeKgSources = TableQuery[KgEdgeKgSourceTable] - lazy val kgNodes = TableQuery[KgNodeTable] - lazy val kgNodeLabels = TableQuery[KgNodeLabelTable] - lazy val kgNodeLabelEdges = TableQuery[KgNodeLabelEdgeTable] - lazy val kgNodeLabelEdgeKgSources = TableQuery[KgNodeLabelEdgeKgSourceTable] - lazy val kgNodeLabelKgSource = TableQuery[KgNodeLabelKgSourceTable] - lazy val kgNodeKgSources = TableQuery[KgNodeKgSourceTable] - lazy val kgNodeKgNodeLabels = TableQuery[KgNodeKgNodeLabelTable] - lazy val kgSources = TableQuery[KgSourceTable] + protected val SentencesDelimString: String = SentencesDelimChar.toString + + lazy val edges = TableQuery[EdgeTable] + lazy val edgeLabels = TableQuery[EdgeLabelTable] + lazy val edgeSources = TableQuery[EdgeSourceTable] + lazy val nodes = TableQuery[NodeTable] + lazy val nodeLabels = TableQuery[NodeLabelTable] + lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] + lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] + lazy val nodeLabelSource = TableQuery[NodeLabelSourceTable] + lazy val nodeSources = TableQuery[NodeSourceTable] + lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] + lazy val sources = TableQuery[SourceTable] protected def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { db.run(a.transactionally) @@ -33,47 +33,47 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC Await.result(runTransaction(a), duration) } - protected final case class KgEdgeRow(id: String, objectKgNodeId: String, predicate: String, sentences: String, subjectKgNodeId: String) - protected final case class KgNodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) + protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) + protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) - private class KgEdgeTable(tag: Tag) extends Table[KgEdgeRow](tag, "edge") { + private class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) - def objectKgNodeId = column[String]("object_node_id") + def objectNodeId = column[String]("object_node_id") def predicate = column[String]("predicate") def sentences = column[String]("sentences") - def subjectKgNodeId = column[String]("subject_node_id") + def subjectNodeId = column[String]("subject_node_id") - def * = (id, objectKgNodeId, predicate, sentences, subjectKgNodeId) <> (KgEdgeRow.tupled, KgEdgeRow.unapply) + def * = (id, objectNodeId, predicate, sentences, subjectNodeId) <> (EdgeRow.tupled, EdgeRow.unapply) - def objectNode = foreignKey("object_node_fk", objectKgNodeId, kgNodes)(_.id) - def subjectNode = foreignKey("subject_node_fk", subjectKgNodeId, kgNodes)(_.id) + def objectNode = foreignKey("object_node_fk", objectNodeId, nodes)(_.id) + def subjectNode = foreignKey("subject_node_fk", subjectNodeId, nodes)(_.id) - def unique_constraint = index("kg_edge_unique_idx", (objectKgNodeId, subjectKgNodeId, predicate), unique = true) + def unique_constraint = index("_edge_unique_idx", (objectNodeId, subjectNodeId, predicate), unique = true) } - private class KgEdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { - def kgEdgeId = column[String]("edge_id") + private class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { + def EdgeId = column[String]("edge_id") def label = column[String]("label") - def * = (kgEdgeId, label) + def * = (EdgeId, label) - def kgEdge = foreignKey("edge_fk", kgEdgeId, kgEdges)(_.id) + def Edge = foreignKey("edge_fk", EdgeId, edges)(_.id) - def pk = primaryKey("edge_label_pk", (kgEdgeId, label)) + def pk = primaryKey("edge_label_pk", (EdgeId, label)) } - private class KgEdgeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { - def kgEdgeId = column[String]("edge_id") - def kgSourceId = column[String]("source_id") + private class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { + def EdgeId = column[String]("edge_id") + def SourceId = column[String]("source_id") - def * = (kgEdgeId, kgSourceId) + def * = (EdgeId, SourceId) - def kgEdge = foreignKey("edge_fk", kgEdgeId, kgEdges)(_.id) + def Edge = foreignKey("edge_fk", EdgeId, edges)(_.id) - def pk = primaryKey("edge_source_pk", (kgEdgeId, kgSourceId)) + def pk = primaryKey("edge_source_pk", (EdgeId, SourceId)) } - private class KgNodeTable(tag: Tag) extends Table[KgNodeRow](tag, "node") { + private class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") @@ -81,77 +81,77 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pos = column[Option[Char]]("pos", O.Length(1)) def wordNetSenseNumber = column[Option[Short]]("word_net_sense_number") - def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (KgNodeRow.tupled, KgNodeRow.unapply) + def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (NodeRow.tupled, NodeRow.unapply) } - private class KgNodeKgNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { - def kgNodeId = column[String]("node_id") + private class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { + def NodeId = column[String]("node_id") def label = column[String]("label") - def * = (kgNodeId, label) + def * = (NodeId, label) - def kgNode = foreignKey("node_fk", kgNodeId, kgNodes)(_.id) + def Node = foreignKey("node_fk", NodeId, nodes)(_.id) - def pk = primaryKey("node_label_pk", (kgNodeId, label)) + def pk = primaryKey("node_label_pk", (NodeId, label)) } - private class KgNodeKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { - def kgNodeId = column[String]("node_id") - def kgSourceId = column[String]("source_id") + private class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { + def NodeId = column[String]("node_id") + def SourceId = column[String]("source_id") - def * = (kgNodeId, kgSourceId) + def * = (NodeId, SourceId) - def kgNode = foreignKey("node_fk", kgNodeId, kgNodes)(_.id) - def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) + def Node = foreignKey("node_fk", NodeId, nodes)(_.id) + def Source = foreignKey("source_fk", SourceId, sources)(_.id) - def pk = primaryKey("node_source_pk", (kgNodeId, kgSourceId)) + def pk = primaryKey("node_source_pk", (NodeId, SourceId)) } - private class KgNodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { + private class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") def * = (label, pageRank) } - private class KgNodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { + private class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) - def objectKgNodeLabelLabel = column[String]("object_node_label_label") - def subjectKgNodeLabelLabel = column[String]("subject_node_label_label") + def objectNodeLabelLabel = column[String]("object_node_label_label") + def subjectNodeLabelLabel = column[String]("subject_node_label_label") - def * = (id, objectKgNodeLabelLabel, subjectKgNodeLabelLabel) + def * = (id, objectNodeLabelLabel, subjectNodeLabelLabel) - def objectKgNodeLabel = foreignKey("object_node_label_fk", objectKgNodeLabelLabel, kgNodeLabels)(_.label) - def subjectKgNodeLabel = foreignKey("subject_node_label_fk", subjectKgNodeLabelLabel, kgNodeLabels)(_.label) + def objectNodeLabel = foreignKey("object_node_label_fk", objectNodeLabelLabel, nodeLabels)(_.label) + def subjectNodeLabel = foreignKey("subject_node_label_fk", subjectNodeLabelLabel, nodeLabels)(_.label) - def unique_constraint = index("node_label_edge_unique_idx", (objectKgNodeLabelLabel, subjectKgNodeLabelLabel), unique = true) + def unique_constraint = index("node_label_edge_unique_idx", (objectNodeLabelLabel, subjectNodeLabelLabel), unique = true) } - private class KgNodeLabelEdgeKgSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { - def kgNodeLabelEdgeId = column[Int]("node_label_edge_id") - def kgSourceId = column[String]("source_id") + private class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { + def NodeLabelEdgeId = column[Int]("node_label_edge_id") + def SourceId = column[String]("source_id") - def * = (kgNodeLabelEdgeId, kgSourceId) + def * = (NodeLabelEdgeId, SourceId) - def kgNodeLabelEdge = foreignKey("node_label_edge_fk", kgNodeLabelEdgeId, kgNodeLabelEdges)(_.id) - def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) + def NodeLabelEdge = foreignKey("node_label_edge_fk", NodeLabelEdgeId, NodeLabelEdges)(_.id) + def Source = foreignKey("source_fk", SourceId, sources)(_.id) - def pk = primaryKey("node_label_edge_source_pk", (kgNodeLabelEdgeId, kgSourceId)) + def pk = primaryKey("node_label_edge_source_pk", (NodeLabelEdgeId, SourceId)) } - private class KgNodeLabelKgSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { - def kgNodeLabelLabel = column[String]("node_label_label") - def kgSourceId = column[String]("source_id") + private class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { + def NodeLabelLabel = column[String]("node_label_label") + def SourceId = column[String]("source_id") - def * = (kgNodeLabelLabel, kgSourceId) + def * = (NodeLabelLabel, SourceId) - def kgNodeLabel = foreignKey("node_label_fk", kgNodeLabelLabel, kgNodeLabels)(_.label) - def kgSource = foreignKey("source_fk", kgSourceId, kgSources)(_.id) + def NodeLabel = foreignKey("node_label_fk", NodeLabelLabel, nodeLabels)(_.label) + def Source = foreignKey("source_fk", SourceId, sources)(_.id) - def pk = primaryKey("node_label_source_pk", (kgNodeLabelLabel, kgSourceId)) + def pk = primaryKey("node_label_source_pk", (NodeLabelLabel, SourceId)) } - private class KgSourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { + private class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index ee021548..9083d9ef 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -16,17 +16,17 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) private final class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { private final implicit class KgEdgeWrapper(edge: KgEdge) { - def toRow: KgEdgeRow = KgEdgeRow( + def toRow: EdgeRow = EdgeRow( id = edge.id, - objectKgNodeId = edge.`object`, + objectNodeId = edge.`object`, predicate = edge.predicate, sentences = edge.sentences.mkString(SentencesDelimString), - subjectKgNodeId = edge.subject + subjectNodeId = edge.subject ) } private final implicit class KgNodeWrapper(node: KgNode) { - def toRow: KgNodeRow = KgNodeRow( + def toRow: NodeRow = NodeRow( id = node.id, inDegree = node.inDegree.map(_.toShort), outDegree = node.outDegree.map(_.toShort), @@ -40,15 +40,18 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) // TODO } - final private def generateEdgeInsert(edge: KgEdge) = - List(kgEdges += edge.toRow) ++ - edge.labels.map(label => kgEdgeLabels += (edge.id, label)) ++ - edge.sourceIds.map(sourceId => kgEdgeKgSources += (edge.id, sourceId)) + private def generateEdgeInsert(edge: KgEdge) = + List(edges += edge.toRow) ++ + edge.labels.map(label => edgeLabels += (edge.id, label)) ++ + edge.sourceIds.map(sourceId => edgeSources += (edge.id, sourceId)) - final private def generateNodeInsert(node: KgNode) = - List(kgNodes += node.toRow) ++ - node.labels.map(label => kgNodeKgNodeLabels += (node.id, label)) ++ - node.sourceIds.map(sourceId => kgNodeKgSources += (node.id, sourceId)) + private def generateNodeInsert(node: KgNode) = + List(nodes += node.toRow) ++ + node.labels.map(label => nodeNodeLabels += (node.id, label)) ++ + node.sourceIds.map(sourceId => nodeSources += (node.id, sourceId)) + + private def generateSourceInsert(source: KgSource) = + List(sources += (source.id, source.label)) override def putEdges(edges: Iterator[KgEdge]) = runSyncTransaction(DBIO.sequence(edges.flatMap(generateEdgeInsert))) @@ -64,14 +67,10 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) override def putNodes(nodes: Iterator[KgNode]): Unit = runSyncTransaction(DBIO.sequence(nodes.flatMap(generateNodeInsert))) - override def putPaths(paths: Iterator[KgPath]): Unit = { -// TODO - } + override def putPaths(paths: Iterator[KgPath]): Unit = Unit override def putSources(sources: Iterator[KgSource]): Unit = - runSyncTransaction(DBIO.sequence(sources.map { source => - kgSources += (source.id, source.label) - })) + runSyncTransaction(DBIO.sequence(sources.flatMap(generateSourceInsert))) override def close(): Unit = Unit } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 477e65eb..4efc3cd9 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -16,7 +16,7 @@ final class PostgresKgQueryStore @Inject()(dbConfigProvider: DatabaseConfigProvi import profile.api._ override def getNode(id: String): Option[KgNode] = { - runSyncTransaction((for { node <- kgNodes if node.id === id } yield node).result.headOption) + runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption) } override def getNodeContext(id: String): Option[KgNodeContext] = None From ae0a1e5e5adcace9ebd038271eac8648000874d1 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 00:51:58 -0400 Subject: [PATCH 14/51] final protected methods --- .../lib/kg/stores/postgres/AbstractPostgresKgStore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 02714d8f..1bcba848 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -25,11 +25,11 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] lazy val sources = TableQuery[SourceTable] - protected def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { + protected final def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { db.run(a.transactionally) } - protected def runSyncTransaction[R](a: DBIOAction[R, NoStream, Nothing], duration: Duration = Duration.Inf): R = { + protected final def runSyncTransaction[R](a: DBIOAction[R, NoStream, Nothing], duration: Duration = Duration.Inf): R = { Await.result(runTransaction(a), duration) } From b9efbed73b2773328f30947ca529291598f9f5c4 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 01:20:26 -0400 Subject: [PATCH 15/51] add final to overrides --- .../postgres/PostgresKgCommandStore.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 9083d9ef..9c088503 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -14,8 +14,8 @@ import scala.concurrent.ExecutionContext class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgCommandStore { import profile.api._ - private final class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { - private final implicit class KgEdgeWrapper(edge: KgEdge) { + private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { + private implicit class KgEdgeWrapper(edge: KgEdge) { def toRow: EdgeRow = EdgeRow( id = edge.id, objectNodeId = edge.`object`, @@ -25,7 +25,7 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) ) } - private final implicit class KgNodeWrapper(node: KgNode) { + private implicit class KgNodeWrapper(node: KgNode) { def toRow: NodeRow = NodeRow( id = node.id, inDegree = node.inDegree.map(_.toShort), @@ -36,7 +36,7 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) ) } - override def clear(): Unit = { + override final def clear(): Unit = { // TODO } @@ -53,10 +53,10 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) private def generateSourceInsert(source: KgSource) = List(sources += (source.id, source.label)) - override def putEdges(edges: Iterator[KgEdge]) = + override final def putEdges(edges: Iterator[KgEdge]) = runSyncTransaction(DBIO.sequence(edges.flatMap(generateEdgeInsert))) - override def putKgtkEdgesWithNodes(edgesWithNodes: Iterator[KgtkEdgeWithNodes]): Unit = + override final def putKgtkEdgesWithNodes(edgesWithNodes: Iterator[KgtkEdgeWithNodes]): Unit = runSyncTransaction(DBIO.sequence( edgesWithNodes.flatMap { edge => edge.nodes.flatMap(generateNodeInsert) ++ @@ -64,17 +64,17 @@ class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider) } )) - override def putNodes(nodes: Iterator[KgNode]): Unit = + override final def putNodes(nodes: Iterator[KgNode]): Unit = runSyncTransaction(DBIO.sequence(nodes.flatMap(generateNodeInsert))) - override def putPaths(paths: Iterator[KgPath]): Unit = Unit + override final def putPaths(paths: Iterator[KgPath]): Unit = Unit - override def putSources(sources: Iterator[KgSource]): Unit = + override final def putSources(sources: Iterator[KgSource]): Unit = runSyncTransaction(DBIO.sequence(sources.flatMap(generateSourceInsert))) - override def close(): Unit = Unit + override final def close(): Unit = Unit } - override def beginTransaction: KgCommandStoreTransaction = + override final def beginTransaction: KgCommandStoreTransaction = new PostgresKgCommandStoreTransaction } From 49437f0e8cd30c9c34013c4e7fe0ef8164fbc9e3 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 01:21:40 -0400 Subject: [PATCH 16/51] singleton PostgresKgCommandStore --- .../lib/kg/stores/postgres/PostgresKgCommandStore.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 9c088503..1d45cc50 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -1,6 +1,6 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import com.google.inject.Inject +import com.google.inject.{Inject, Singleton} import io.github.tetherlessworld.mcsapps.lib.kg.formats.kgtk.KgtkEdgeWithNodes import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode @@ -11,6 +11,7 @@ import play.api.db.slick.DatabaseConfigProvider import scala.concurrent.ExecutionContext +@Singleton class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgCommandStore { import profile.api._ From 5c8ad1fa0b729ff5df9c0fddb894529044a9af15 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 01:30:10 -0400 Subject: [PATCH 17/51] update KgStoresModule with postgres --- .../mcsapps/lib/kg/stores/KgStoresModule.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala index bfef0477..f0cc5c6e 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/KgStoresModule.scala @@ -4,7 +4,7 @@ import com.google.inject.AbstractModule import io.github.tetherlessworld.mcsapps.lib.kg.stores.empty.EmptyKgStore import io.github.tetherlessworld.mcsapps.lib.kg.stores.mem.MemKgStore import io.github.tetherlessworld.mcsapps.lib.kg.stores.neo4j.{Neo4jKgCommandStore, Neo4jKgQueryStore} -import io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres.PostgresKgQueryStore +import io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres.{PostgresKgCommandStore, PostgresKgQueryStore} import io.github.tetherlessworld.mcsapps.lib.kg.stores.test.TestKgStore import org.slf4j.LoggerFactory import play.api.{Configuration, Environment} @@ -23,14 +23,15 @@ final class KgStoresModule(environment: Environment, configuration: Configuratio bind(classOf[KgCommandStore]).to(classOf[MemKgStore]) bind(classOf[KgQueryStore]).to(classOf[MemKgStore]) } + case "postgres" => { + bind(classOf[KgCommandStore]).to(classOf[PostgresKgCommandStore]) + bind(classOf[KgQueryStore]).to(classOf[PostgresKgQueryStore]) + } case "test" => { logger.info("using test stores") bind(classOf[KgCommandStore]).to(classOf[TestKgStore]) bind(classOf[KgQueryStore]).to(classOf[TestKgStore]) } - case "postgres" => { - bind(classOf[KgQueryStore]).to(classOf[PostgresKgQueryStore]) - } case _ => { bind(classOf[KgCommandStore]).to(classOf[Neo4jKgCommandStore]) bind(classOf[KgQueryStore]).to(classOf[Neo4jKgQueryStore]) From 384a91f7197d3d30fd92a5ce74f8d117c71df37c Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 01:56:11 -0400 Subject: [PATCH 18/51] add postgres to docker compose --- docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d619115e..198dee03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,8 +48,24 @@ services: - neo4j-logs:/logs - neo4j-import:/var/lib/neo4j/import - neo4j-plugins:/plugins + postgres: + container_name: mcs-postgres + environment: + POSTGRES_USER: "mcs" + POSTGRES_PASSWORD: "7EAdu7jJvZNxxrNZ" + POSTGRES_DB: "kg" + image: library/postgres:12.4 + networks: + - mcs + ports: + - 5432:5432 + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data + volumes: neo4j-data: neo4j-import: neo4j-logs: neo4j-plugins: + postgres-data: From 3449050ed644ac3cbc420f3a0d3afa2aa96bfa11 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 02:01:44 -0400 Subject: [PATCH 19/51] use protected on Tables and TableQuerys --- .../postgres/AbstractPostgresKgStore.scala | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 1bcba848..f14a05f3 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -13,17 +13,17 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC protected val SentencesDelimChar = '|' protected val SentencesDelimString: String = SentencesDelimChar.toString - lazy val edges = TableQuery[EdgeTable] - lazy val edgeLabels = TableQuery[EdgeLabelTable] - lazy val edgeSources = TableQuery[EdgeSourceTable] - lazy val nodes = TableQuery[NodeTable] - lazy val nodeLabels = TableQuery[NodeLabelTable] - lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] - lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] - lazy val nodeLabelSource = TableQuery[NodeLabelSourceTable] - lazy val nodeSources = TableQuery[NodeSourceTable] - lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] - lazy val sources = TableQuery[SourceTable] + protected lazy val edges = TableQuery[EdgeTable] + protected lazy val edgeLabels = TableQuery[EdgeLabelTable] + protected lazy val edgeSources = TableQuery[EdgeSourceTable] + protected lazy val nodes = TableQuery[NodeTable] + protected lazy val nodeLabels = TableQuery[NodeLabelTable] + protected lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] + protected lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] + protected lazy val nodeLabelSource = TableQuery[NodeLabelSourceTable] + protected lazy val nodeSources = TableQuery[NodeSourceTable] + protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] + protected lazy val sources = TableQuery[SourceTable] protected final def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { db.run(a.transactionally) @@ -36,7 +36,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) - private class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { + protected class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) def objectNodeId = column[String]("object_node_id") def predicate = column[String]("predicate") @@ -51,7 +51,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def unique_constraint = index("_edge_unique_idx", (objectNodeId, subjectNodeId, predicate), unique = true) } - private class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { + protected class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { def EdgeId = column[String]("edge_id") def label = column[String]("label") @@ -62,7 +62,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("edge_label_pk", (EdgeId, label)) } - private class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { + protected class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { def EdgeId = column[String]("edge_id") def SourceId = column[String]("source_id") @@ -73,7 +73,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("edge_source_pk", (EdgeId, SourceId)) } - private class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { + protected class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") @@ -84,7 +84,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (NodeRow.tupled, NodeRow.unapply) } - private class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { + protected class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { def NodeId = column[String]("node_id") def label = column[String]("label") @@ -95,7 +95,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("node_label_pk", (NodeId, label)) } - private class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { + protected class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { def NodeId = column[String]("node_id") def SourceId = column[String]("source_id") @@ -107,14 +107,14 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("node_source_pk", (NodeId, SourceId)) } - private class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { + protected class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") def * = (label, pageRank) } - private class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { + protected class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def objectNodeLabelLabel = column[String]("object_node_label_label") def subjectNodeLabelLabel = column[String]("subject_node_label_label") @@ -127,19 +127,19 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def unique_constraint = index("node_label_edge_unique_idx", (objectNodeLabelLabel, subjectNodeLabelLabel), unique = true) } - private class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { + protected class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { def NodeLabelEdgeId = column[Int]("node_label_edge_id") def SourceId = column[String]("source_id") def * = (NodeLabelEdgeId, SourceId) - def NodeLabelEdge = foreignKey("node_label_edge_fk", NodeLabelEdgeId, NodeLabelEdges)(_.id) + def NodeLabelEdge = foreignKey("node_label_edge_fk", NodeLabelEdgeId, nodeLabelEdges)(_.id) def Source = foreignKey("source_fk", SourceId, sources)(_.id) def pk = primaryKey("node_label_edge_source_pk", (NodeLabelEdgeId, SourceId)) } - private class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { + protected class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { def NodeLabelLabel = column[String]("node_label_label") def SourceId = column[String]("source_id") @@ -151,7 +151,7 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC def pk = primaryKey("node_label_source_pk", (NodeLabelLabel, SourceId)) } - private class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { + protected class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") From d88dcabc9cb2913bb766197f29167476c2992463 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 02:06:46 -0400 Subject: [PATCH 20/51] https://github.com/slick/slick/issues/1425 --- .../lib/kg/stores/postgres/AbstractPostgresKgStore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index f14a05f3..a816624c 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -25,11 +25,11 @@ abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseC protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] - protected final def runTransaction[R](a: DBIOAction[R, NoStream, Nothing]): Future[R] = { + protected final def runTransaction[R](a: DBIOAction[R, NoStream, Effect.All]): Future[R] = { db.run(a.transactionally) } - protected final def runSyncTransaction[R](a: DBIOAction[R, NoStream, Nothing], duration: Duration = Duration.Inf): R = { + protected final def runSyncTransaction[R](a: DBIOAction[R, NoStream, Effect.All], duration: Duration = Duration.Inf): R = { Await.result(runTransaction(a), duration) } From d5b56f3b2ecf6bb76d4bcb8e5c05903028e7a965 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 30 Oct 2020 02:22:07 -0400 Subject: [PATCH 21/51] fix postgres jdbc url --- app/kg/conf/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/kg/conf/application.conf b/app/kg/conf/application.conf index 3cba903e..465f4c03 100644 --- a/app/kg/conf/application.conf +++ b/app/kg/conf/application.conf @@ -12,5 +12,5 @@ play.modules.enabled += "stores.KgStoresModule" slick.dbs.default.db.driver = "org.postgresql.Driver" slick.dbs.default.db.password = "7EAdu7jJvZNxxrNZ" slick.dbs.default.db.user = "mcs" -slick.dbs.default.db.url = "postgres://mcs-postgres:5432/kg" +slick.dbs.default.db.url = "jdbc:postgresql://mcs-postgres:5432/kg" slick.dbs.default.profile = "slick.jdbc.PostgresProfile$" From 033f9d58628775357690a4094b90c0ea9098ab7e Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 00:06:05 -0500 Subject: [PATCH 22/51] An attempt at postgres store config. error exists on PostgresKgCommandStore and PostgresKgQueryStore. --- app/kg/conf/application.conf | 13 +++++++---- build.sbt | 4 ++-- .../stores/SlickDatabaseConfigProvider.scala | 23 +++++++++++++++++++ .../postgres/AbstractPostgresKgStore.scala | 6 ++--- .../postgres/PostgresKgCommandStore.scala | 3 +-- .../postgres/PostgresKgQueryStore.scala | 3 +-- .../PostgresStoreConfigProvider.scala | 8 +++++++ 7 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala diff --git a/app/kg/conf/application.conf b/app/kg/conf/application.conf index 465f4c03..b2e34ae8 100644 --- a/app/kg/conf/application.conf +++ b/app/kg/conf/application.conf @@ -9,8 +9,11 @@ play.filters.enabled += "play.filters.cors.CORSFilter" play.filters.disabled += "play.filters.csrf.CSRFFilter" play.filters.disabled += "play.filters.hosts.AllowedHostsFilter" play.modules.enabled += "stores.KgStoresModule" -slick.dbs.default.db.driver = "org.postgresql.Driver" -slick.dbs.default.db.password = "7EAdu7jJvZNxxrNZ" -slick.dbs.default.db.user = "mcs" -slick.dbs.default.db.url = "jdbc:postgresql://mcs-postgres:5432/kg" -slick.dbs.default.profile = "slick.jdbc.PostgresProfile$" +postgres.connectionPool = "HikariCP" +postgres.dataSourceClass = "org.postgresql.ds.PGSimpleDataSource" +postgres.profile = "slick.jdbc.PostgresProfile$" +postgres.properties.driver = "org.postgresql.Driver" +postgres.properties.password = "7EAdu7jJvZNxxrNZ" +postgres.properties.url = "jdbc:postgresql://mcs-postgres:5432/kg" +postgres.properties.user = "mcs" + diff --git a/build.sbt b/build.sbt index 47087fe6..ab2d49c0 100644 --- a/build.sbt +++ b/build.sbt @@ -47,8 +47,8 @@ lazy val kgLib = libraryDependencies ++= Seq( // Implement search in the MemStore (and thus the TestStore) "com.outr" %% "lucene4s" % "1.9.1", - "com.typesafe.play" %% "play-slick" % "5.0.0", - "com.typesafe.play" %% "play-slick-evolutions" % "5.0.0", + "com.typesafe.slick" %% "slick" % "3.3.3", + "com.typesafe.slick" %% "slick-hikaricp" % "3.3.3", "io.github.tetherless-world" %% "twxplore-base" % twxploreVersion, "io.github.tetherless-world" %% "twxplore-test" % twxploreVersion % Test, "me.tongfei" % "progressbar" % "0.8.1", diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala new file mode 100644 index 00000000..174f8251 --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala @@ -0,0 +1,23 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores + + +import slick.basic.{BasicProfile, DatabaseConfig} + +trait HasDatabaseConfig[T <: BasicProfile] { + val databaseConfig: DatabaseConfig[T] + + val profile: T = databaseConfig.profile + val db: T#Backend#Database = databaseConfig.db +} + +trait HasDatabaseConfigProvider[T <: BasicProfile] extends HasDatabaseConfig[T] { + val databaseConfigProvider: SlickDatabaseConfigProvider[T] + + val databaseConfig = databaseConfigProvider.databaseConfig +} + +class SlickDatabaseConfigProvider[T <: BasicProfile](val databaseConfig: DatabaseConfig[T]) { + def this(path: String) = + this(DatabaseConfig.forConfig[T](path)) + +} diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index a816624c..16b0518d 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,13 +1,13 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} -import slick.jdbc.PostgresProfile +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider, SlickDatabaseConfigProvider} +import slick.jdbc.{PostgresProfile} import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} -abstract class AbstractPostgresKgStore(protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { +abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) extends HasDatabaseConfigProvider[PostgresProfile] { import profile.api._ protected val SentencesDelimChar = '|' diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 1d45cc50..f2e2a460 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -7,12 +7,11 @@ import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreTransaction} -import play.api.db.slick.DatabaseConfigProvider import scala.concurrent.ExecutionContext @Singleton -class PostgresKgCommandStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgCommandStore { +class PostgresKgCommandStore @Inject()(databaseConfigProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(databaseConfigProvider) with KgCommandStore { import profile.api._ private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 4efc3cd9..c4dcc0e2 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -7,12 +7,11 @@ import io.github.tetherlessworld.mcsapps.lib.kg.models.search.{KgSearchFacets, K import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.KgQueryStore import javax.inject.Singleton -import play.api.db.slick.DatabaseConfigProvider import scala.concurrent.ExecutionContext @Singleton -final class PostgresKgQueryStore @Inject()(dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(dbConfigProvider) with KgQueryStore { +final class PostgresKgQueryStore @Inject()(databaseConfigProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(databaseConfigProvider) with KgQueryStore { import profile.api._ override def getNode(id: String): Option[KgNode] = { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala new file mode 100644 index 00000000..7dfb21de --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -0,0 +1,8 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider +import javax.inject.Singleton +import slick.jdbc.PostgresProfile + +@Singleton +final class PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") From 57d0ac41f00bd8d0af6d6821bd30f98c6b029d72 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 00:12:06 -0500 Subject: [PATCH 23/51] docker expose port only to localhost --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 198dee03..01f4e6ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: networks: - mcs ports: - - 5432:5432 + - 127.0.0.1:5432:5432 restart: unless-stopped volumes: - postgres-data:/var/lib/postgresql/data From 9db92079fc1af20661fdd1054e19980b6e64b5dc Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 00:19:27 -0500 Subject: [PATCH 24/51] make tables final --- .../postgres/AbstractPostgresKgStore.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 16b0518d..12fdafeb 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -36,7 +36,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) - protected class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { + protected final class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) def objectNodeId = column[String]("object_node_id") def predicate = column[String]("predicate") @@ -51,7 +51,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def unique_constraint = index("_edge_unique_idx", (objectNodeId, subjectNodeId, predicate), unique = true) } - protected class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { + protected final class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { def EdgeId = column[String]("edge_id") def label = column[String]("label") @@ -62,7 +62,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("edge_label_pk", (EdgeId, label)) } - protected class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { + protected final class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { def EdgeId = column[String]("edge_id") def SourceId = column[String]("source_id") @@ -73,7 +73,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("edge_source_pk", (EdgeId, SourceId)) } - protected class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { + protected final class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") def outDegree = column[Option[Short]]("out_degree") @@ -84,7 +84,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (NodeRow.tupled, NodeRow.unapply) } - protected class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { + protected final class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { def NodeId = column[String]("node_id") def label = column[String]("label") @@ -95,7 +95,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("node_label_pk", (NodeId, label)) } - protected class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { + protected final class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { def NodeId = column[String]("node_id") def SourceId = column[String]("source_id") @@ -107,14 +107,14 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("node_source_pk", (NodeId, SourceId)) } - protected class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { + protected final class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") def * = (label, pageRank) } - protected class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { + protected final class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def objectNodeLabelLabel = column[String]("object_node_label_label") def subjectNodeLabelLabel = column[String]("subject_node_label_label") @@ -127,7 +127,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def unique_constraint = index("node_label_edge_unique_idx", (objectNodeLabelLabel, subjectNodeLabelLabel), unique = true) } - protected class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { + protected final class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { def NodeLabelEdgeId = column[Int]("node_label_edge_id") def SourceId = column[String]("source_id") @@ -139,7 +139,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("node_label_edge_source_pk", (NodeLabelEdgeId, SourceId)) } - protected class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { + protected final class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { def NodeLabelLabel = column[String]("node_label_label") def SourceId = column[String]("source_id") @@ -151,7 +151,7 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf def pk = primaryKey("node_label_source_pk", (NodeLabelLabel, SourceId)) } - protected class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { + protected final class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") From 8b1026977e15e9ba4b6d3a1e9b82de8aee6b43ee Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 00:26:00 -0500 Subject: [PATCH 25/51] Workaround by not using HasDatabaseConfigProvider trait --- .../lib/kg/stores/SlickDatabaseConfigProvider.scala | 4 +++- .../kg/stores/postgres/AbstractPostgresKgStore.scala | 10 ++++++---- .../kg/stores/postgres/PostgresKgCommandStore.scala | 4 ++-- .../lib/kg/stores/postgres/PostgresKgQueryStore.scala | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala index 174f8251..d83fd399 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala @@ -3,6 +3,8 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores import slick.basic.{BasicProfile, DatabaseConfig} +import scala.reflect.ClassTag + trait HasDatabaseConfig[T <: BasicProfile] { val databaseConfig: DatabaseConfig[T] @@ -16,7 +18,7 @@ trait HasDatabaseConfigProvider[T <: BasicProfile] extends HasDatabaseConfig[T] val databaseConfig = databaseConfigProvider.databaseConfig } -class SlickDatabaseConfigProvider[T <: BasicProfile](val databaseConfig: DatabaseConfig[T]) { +class SlickDatabaseConfigProvider[T <: BasicProfile : ClassTag](val databaseConfig: DatabaseConfig[T]) { def this(path: String) = this(DatabaseConfig.forConfig[T](path)) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 12fdafeb..85c5d3ec 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,14 +1,14 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider, SlickDatabaseConfigProvider} -import slick.jdbc.{PostgresProfile} +import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider +import slick.jdbc.PostgresProfile import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} -abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) extends HasDatabaseConfigProvider[PostgresProfile] { - import profile.api._ +abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) { + import databaseConfigProvider.databaseConfig.profile.api._ protected val SentencesDelimChar = '|' protected val SentencesDelimString: String = SentencesDelimChar.toString @@ -25,6 +25,8 @@ abstract class AbstractPostgresKgStore(databaseConfigProvider: SlickDatabaseConf protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] + protected val db = databaseConfigProvider.databaseConfig.db + protected final def runTransaction[R](a: DBIOAction[R, NoStream, Effect.All]): Future[R] = { db.run(a.transactionally) } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index f2e2a460..267b7cd2 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -11,8 +11,8 @@ import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgComman import scala.concurrent.ExecutionContext @Singleton -class PostgresKgCommandStore @Inject()(databaseConfigProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(databaseConfigProvider) with KgCommandStore { - import profile.api._ +class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgCommandStore { + import databaseConfigProvider.databaseConfig.profile.api._ private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { private implicit class KgEdgeWrapper(edge: KgEdge) { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index c4dcc0e2..0954543b 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -11,8 +11,8 @@ import javax.inject.Singleton import scala.concurrent.ExecutionContext @Singleton -final class PostgresKgQueryStore @Inject()(databaseConfigProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(databaseConfigProvider) with KgQueryStore { - import profile.api._ +final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgQueryStore { + import databaseConfigProvider.databaseConfig.profile.api._ override def getNode(id: String): Option[KgNode] = { runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption) From b35e3996ece78288a50882f0f14ea807216434b3 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 01:17:04 -0500 Subject: [PATCH 26/51] add protected to config provider traits --- .../lib/kg/stores/SlickDatabaseConfigProvider.scala | 10 +++++----- .../kg/stores/postgres/AbstractPostgresKgStore.scala | 8 +++----- .../kg/stores/postgres/PostgresKgCommandStore.scala | 2 +- .../lib/kg/stores/postgres/PostgresKgQueryStore.scala | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala index d83fd399..78bf6cfc 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala @@ -6,16 +6,16 @@ import slick.basic.{BasicProfile, DatabaseConfig} import scala.reflect.ClassTag trait HasDatabaseConfig[T <: BasicProfile] { - val databaseConfig: DatabaseConfig[T] + protected val databaseConfig: DatabaseConfig[T] - val profile: T = databaseConfig.profile - val db: T#Backend#Database = databaseConfig.db + protected val profile: T = databaseConfig.profile + protected val db: T#Backend#Database = databaseConfig.db } trait HasDatabaseConfigProvider[T <: BasicProfile] extends HasDatabaseConfig[T] { - val databaseConfigProvider: SlickDatabaseConfigProvider[T] + protected val databaseConfigProvider: SlickDatabaseConfigProvider[T] - val databaseConfig = databaseConfigProvider.databaseConfig + protected val databaseConfig = databaseConfigProvider.databaseConfig } class SlickDatabaseConfigProvider[T <: BasicProfile : ClassTag](val databaseConfig: DatabaseConfig[T]) { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 85c5d3ec..d606b5db 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,14 +1,14 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider, SlickDatabaseConfigProvider} import slick.jdbc.PostgresProfile import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} -abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) { - import databaseConfigProvider.databaseConfig.profile.api._ +abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) extends HasDatabaseConfigProvider[PostgresProfile] { + import profile.api._ protected val SentencesDelimChar = '|' protected val SentencesDelimString: String = SentencesDelimChar.toString @@ -25,8 +25,6 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Sli protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] - protected val db = databaseConfigProvider.databaseConfig.db - protected final def runTransaction[R](a: DBIOAction[R, NoStream, Effect.All]): Future[R] = { db.run(a.transactionally) } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 267b7cd2..29e7d22b 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -12,7 +12,7 @@ import scala.concurrent.ExecutionContext @Singleton class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgCommandStore { - import databaseConfigProvider.databaseConfig.profile.api._ + import profile.api._ private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { private implicit class KgEdgeWrapper(edge: KgEdge) { diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 0954543b..f60f438c 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -12,7 +12,7 @@ import scala.concurrent.ExecutionContext @Singleton final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgQueryStore { - import databaseConfigProvider.databaseConfig.profile.api._ + import profile.api._ override def getNode(id: String): Option[KgNode] = { runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption) From 90eb712c5c355eeb1ba3133902b0f83f0b06130d Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 02:03:38 -0500 Subject: [PATCH 27/51] truncate on clear, createIfNotExists on bootstrap with optional drop if specified through environment variable --- .../postgres/AbstractPostgresKgStore.scala | 9 ++++++-- .../postgres/PostgresKgCommandStore.scala | 23 ++++++++++++++++++- .../PostgresStoreConfigProvider.scala | 9 ++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index d606b5db..23297381 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,13 +1,13 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider, SlickDatabaseConfigProvider} +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider} import slick.jdbc.PostgresProfile import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} -abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: SlickDatabaseConfigProvider[PostgresProfile]) extends HasDatabaseConfigProvider[PostgresProfile] { +abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: PostgresStoreConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { import profile.api._ protected val SentencesDelimChar = '|' @@ -25,6 +25,11 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Sli protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] + private lazy val tables = + List(edges, edgeLabels, edgeSources, nodes, nodeLabels, nodeLabelEdges, nodeLabelEdgeSources, nodeLabelSource, nodeSources, nodeNodeLabels, sources) + + protected lazy val tablesDdlObject = tables.map(_.schema).reduce((left, right) => left ++ right) + protected final def runTransaction[R](a: DBIOAction[R, NoStream, Effect.All]): Future[R] = { db.run(a.transactionally) } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 29e7d22b..d794d9a2 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -14,6 +14,8 @@ import scala.concurrent.ExecutionContext class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgCommandStore { import profile.api._ + private var bootstrapped: Boolean = false + private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { private implicit class KgEdgeWrapper(edge: KgEdge) { def toRow: EdgeRow = EdgeRow( @@ -37,7 +39,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid } override final def clear(): Unit = { -// TODO + runSyncTransaction(tablesDdlObject.truncate) } private def generateEdgeInsert(edge: KgEdge) = @@ -75,6 +77,25 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid override final def close(): Unit = Unit } + bootstrapStore() + + private def bootstrapStore(): Unit = { + this.synchronized { + if (bootstrapped) { + return + } + + runSyncTransaction( + DBIO.sequence( + { if (databaseConfigProvider.dropTables) List(tablesDdlObject.drop) else List() } ++ + List(tablesDdlObject.createIfNotExists) + ) + ) + + bootstrapped = true + } + } + override final def beginTransaction: KgCommandStoreTransaction = new PostgresKgCommandStoreTransaction } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala index 7dfb21de..6edf7ec7 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -1,8 +1,13 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider -import javax.inject.Singleton +import javax.inject.{Inject, Singleton} +import play.api.Configuration import slick.jdbc.PostgresProfile @Singleton -final class PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") +final class PostgresStoreConfigProvider(val dropTables: Boolean) extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") { + @Inject + def this(configuration: Configuration) = + this(configuration.getOptional[Boolean]("drop").getOrElse(false)) +} From fdd2a20246585f7a727ad4f2fb83c3c097a7e705 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 3 Nov 2020 03:11:12 -0500 Subject: [PATCH 28/51] add toKg conversions for node and edge case classes --- .../postgres/AbstractPostgresKgStore.scala | 29 +++++++++++++++++-- .../postgres/PostgresKgCommandStore.scala | 4 +++ .../postgres/PostgresKgQueryStore.scala | 2 +- .../PostgresStoreConfigProvider.scala | 2 +- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 23297381..21440a21 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,6 +1,8 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres -import io.github.tetherlessworld.mcsapps.lib.kg.stores.{HasDatabaseConfigProvider} +import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode +import io.github.tetherlessworld.mcsapps.lib.kg.stores.HasDatabaseConfigProvider import slick.jdbc.PostgresProfile import scala.concurrent.duration.Duration @@ -38,8 +40,29 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos Await.result(runTransaction(a), duration) } - protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) - protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) + protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) { + def toKgEdge(labels: List[String], sourceIds: List[String]) = KgEdge( + id = id, + labels = labels, + `object` = objectNodeId, + predicate = predicate, + sentences = sentences.split(SentencesDelimChar).toList, + sourceIds = sourceIds, + subject = subjectNodeId + ) + } + protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) { + def toKgNode(labels: List[String], sourceIds: List[String]) = KgNode( + id = id, + inDegree = inDegree.map(_.toInt), + labels = labels, + outDegree = outDegree.map(_.toInt), + pageRank = pageRank.map(_.toDouble), + pos = pos, + sourceIds = sourceIds, + wordNetSenseNumber = wordNetSenseNumber.map(_.toInt) + ) + } protected final class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index d794d9a2..bb469172 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -7,6 +7,7 @@ import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreTransaction} +import org.slf4j.LoggerFactory import scala.concurrent.ExecutionContext @@ -15,6 +16,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid import profile.api._ private var bootstrapped: Boolean = false + private val logger = LoggerFactory.getLogger(getClass) private class PostgresKgCommandStoreTransaction extends KgCommandStoreTransaction { private implicit class KgEdgeWrapper(edge: KgEdge) { @@ -82,6 +84,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid private def bootstrapStore(): Unit = { this.synchronized { if (bootstrapped) { + logger.info("Postgres store already bootstrapped, skipping...") return } @@ -93,6 +96,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid ) bootstrapped = true + logger.info("Postgres store bootstrapped") } } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index f60f438c..4baf4b35 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -15,7 +15,7 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr import profile.api._ override def getNode(id: String): Option[KgNode] = { - runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption) + runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption).map(_.toKgNode(List(), List())) } override def getNodeContext(id: String): Option[KgNodeContext] = None diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala index 6edf7ec7..0e0aa7dd 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -9,5 +9,5 @@ import slick.jdbc.PostgresProfile final class PostgresStoreConfigProvider(val dropTables: Boolean) extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") { @Inject def this(configuration: Configuration) = - this(configuration.getOptional[Boolean]("drop").getOrElse(false)) + this(configuration.getOptional[Boolean]("postgres.drop").getOrElse(false)) } From b5471ebadf0f513538957e20459e25fed78d0296 Mon Sep 17 00:00:00 2001 From: Minor Gordon Date: Tue, 3 Nov 2020 09:43:51 -0500 Subject: [PATCH 29/51] use lazy val to get around initialization order issues --- .../mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala index 78bf6cfc..8d08d234 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala @@ -8,8 +8,8 @@ import scala.reflect.ClassTag trait HasDatabaseConfig[T <: BasicProfile] { protected val databaseConfig: DatabaseConfig[T] - protected val profile: T = databaseConfig.profile - protected val db: T#Backend#Database = databaseConfig.db + protected lazy val profile: T = databaseConfig.profile + protected lazy val db: T#Backend#Database = databaseConfig.db } trait HasDatabaseConfigProvider[T <: BasicProfile] extends HasDatabaseConfig[T] { From 3d51ccef20e247d816e4893b69ccda37dab90767 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 4 Nov 2020 02:39:17 -0500 Subject: [PATCH 30/51] Fix postgres config --- app/kg/conf/application.conf | 11 +++++------ .../stores/postgres/PostgresKgCommandStore.scala | 14 ++++++++------ .../postgres/PostgresStoreConfigProvider.scala | 9 ++------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/app/kg/conf/application.conf b/app/kg/conf/application.conf index b2e34ae8..133eba64 100644 --- a/app/kg/conf/application.conf +++ b/app/kg/conf/application.conf @@ -9,11 +9,10 @@ play.filters.enabled += "play.filters.cors.CORSFilter" play.filters.disabled += "play.filters.csrf.CSRFFilter" play.filters.disabled += "play.filters.hosts.AllowedHostsFilter" play.modules.enabled += "stores.KgStoresModule" -postgres.connectionPool = "HikariCP" -postgres.dataSourceClass = "org.postgresql.ds.PGSimpleDataSource" postgres.profile = "slick.jdbc.PostgresProfile$" -postgres.properties.driver = "org.postgresql.Driver" -postgres.properties.password = "7EAdu7jJvZNxxrNZ" -postgres.properties.url = "jdbc:postgresql://mcs-postgres:5432/kg" -postgres.properties.user = "mcs" +postgres.db.connectionPool = "HikariCP" +postgres.db.driver = "org.postgresql.Driver" +postgres.db.password = "7EAdu7jJvZNxxrNZ" +postgres.db.url = "jdbc:postgresql://mcs-postgres:5432/kg" +postgres.db.user = "mcs" diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index bb469172..3cc7b5cd 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -88,12 +88,14 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid return } - runSyncTransaction( - DBIO.sequence( - { if (databaseConfigProvider.dropTables) List(tablesDdlObject.drop) else List() } ++ - List(tablesDdlObject.createIfNotExists) - ) - ) + val tableCount = runSyncTransaction(sql"SELECT COUNT(table_name) FROM information_schema.tables WHERE table_schema='public'".as[Int].head) + + if (tableCount != 0) { + logger.info("Postgres database tables already created, skipping bootstrap...") + return + } + + runSyncTransaction(tablesDdlObject.create) bootstrapped = true logger.info("Postgres store bootstrapped") diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala index 0e0aa7dd..7dfb21de 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -1,13 +1,8 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider -import javax.inject.{Inject, Singleton} -import play.api.Configuration +import javax.inject.Singleton import slick.jdbc.PostgresProfile @Singleton -final class PostgresStoreConfigProvider(val dropTables: Boolean) extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") { - @Inject - def this(configuration: Configuration) = - this(configuration.getOptional[Boolean]("postgres.drop").getOrElse(false)) -} +final class PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") From 349e2988345e58a821acc96acdc296a041f78265 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 4 Nov 2020 03:12:44 -0500 Subject: [PATCH 31/51] override putData --- .../lib/kg/stores/postgres/PostgresKgCommandStore.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 3cc7b5cd..7810c54a 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -1,6 +1,7 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import com.google.inject.{Inject, Singleton} +import io.github.tetherlessworld.mcsapps.lib.kg.data.KgData import io.github.tetherlessworld.mcsapps.lib.kg.formats.kgtk.KgtkEdgeWithNodes import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode @@ -57,6 +58,13 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid private def generateSourceInsert(source: KgSource) = List(sources += (source.id, source.label)) + override final def putData(data: KgData) = + runSyncTransaction(DBIO.sequence( + data.sources.flatMap(generateSourceInsert) ++ + data.nodesUnranked.flatMap(generateNodeInsert) ++ + data.edges.flatMap(generateEdgeInsert) + )) + override final def putEdges(edges: Iterator[KgEdge]) = runSyncTransaction(DBIO.sequence(edges.flatMap(generateEdgeInsert))) From 6b368e671c63c28605b44b52b102ac140184af68 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 4 Nov 2020 03:12:49 -0500 Subject: [PATCH 32/51] Start postgres spec --- .../stores/postgres/PostgresKgStoreSpec.scala | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala diff --git a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala new file mode 100644 index 00000000..af43f6f8 --- /dev/null +++ b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala @@ -0,0 +1,50 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import java.net.InetAddress + +import io.github.tetherlessworld.mcsapps.lib.kg.data.TestKgData +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreBehaviors, KgQueryStore, KgStoreFactory, StoreTestMode} +import org.scalatest.{BeforeAndAfterAll, WordSpec} + +class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommandStoreBehaviors { + import scala.concurrent.ExecutionContext.Implicits.global + + val configProvider = new PostgresStoreConfigProvider() + val command = new PostgresKgCommandStore(configProvider) + val query = new PostgresKgQueryStore(configProvider) + val postgresHostAddress = InetAddress.getByName("mcs-postgres").getHostAddress + val inTestingEnvironment = System.getenv("CI") != null || postgresHostAddress != "128.113.12.49" + + override def beforeAll(): Unit = { + if (!inTestingEnvironment) { + return + } + resetSut() + } + + private def resetSut(): Unit = { + if (!query.isEmpty) { + command.withTransaction { _.clear() } + } + command.withTransaction { _.putData(TestKgData) } + } + + private object PostgresKgStoreFactory extends KgStoreFactory { + override def apply(testMode: StoreTestMode)(f: (KgCommandStore, KgQueryStore) => Unit): Unit = { + try { + f(command, query) + } finally { + if (testMode == StoreTestMode.ReadWrite) { + resetSut() + } + } + } + } + + if (inTestingEnvironment) { + "The neo4j store" can { + behave like commandStore(PostgresKgStoreFactory) +// behave like queryStore(PostgresKgStoreFactory) + } + } +} From 483491143154ae6138c5d63fcaf256e09e4039b1 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 6 Nov 2020 13:34:26 -0500 Subject: [PATCH 33/51] fix table names --- .../postgres/AbstractPostgresKgStore.scala | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 21440a21..c472c327 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -80,25 +80,25 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos } protected final class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { - def EdgeId = column[String]("edge_id") + def edgeId = column[String]("edge_id") def label = column[String]("label") - def * = (EdgeId, label) + def * = (edgeId, label) - def Edge = foreignKey("edge_fk", EdgeId, edges)(_.id) + def edge = foreignKey("edge_fk", edgeId, edges)(_.id) - def pk = primaryKey("edge_label_pk", (EdgeId, label)) + def pk = primaryKey("edge_label_pk", (edgeId, label)) } protected final class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { - def EdgeId = column[String]("edge_id") - def SourceId = column[String]("source_id") + def edgeId = column[String]("edge_id") + def sourceId = column[String]("source_id") - def * = (EdgeId, SourceId) + def * = (edgeId, sourceId) - def Edge = foreignKey("edge_fk", EdgeId, edges)(_.id) + def edge = foreignKey("edge_fk", edgeId, edges)(_.id) - def pk = primaryKey("edge_source_pk", (EdgeId, SourceId)) + def pk = primaryKey("edge_source_pk", (edgeId, sourceId)) } protected final class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { @@ -113,26 +113,26 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos } protected final class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { - def NodeId = column[String]("node_id") + def nodeId = column[String]("node_id") def label = column[String]("label") - def * = (NodeId, label) + def * = (nodeId, label) - def Node = foreignKey("node_fk", NodeId, nodes)(_.id) + def node = foreignKey("node_fk", nodeId, nodes)(_.id) - def pk = primaryKey("node_label_pk", (NodeId, label)) + def pk = primaryKey("node_label_pk", (nodeId, label)) } protected final class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { - def NodeId = column[String]("node_id") - def SourceId = column[String]("source_id") + def nodeId = column[String]("node_id") + def sourceId = column[String]("source_id") - def * = (NodeId, SourceId) + def * = (nodeId, sourceId) - def Node = foreignKey("node_fk", NodeId, nodes)(_.id) - def Source = foreignKey("source_fk", SourceId, sources)(_.id) + def node = foreignKey("node_fk", nodeId, nodes)(_.id) + def source = foreignKey("source_fk", sourceId, sources)(_.id) - def pk = primaryKey("node_source_pk", (NodeId, SourceId)) + def pk = primaryKey("node_source_pk", (nodeId, sourceId)) } protected final class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { @@ -156,27 +156,27 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos } protected final class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { - def NodeLabelEdgeId = column[Int]("node_label_edge_id") - def SourceId = column[String]("source_id") + def nodeLabelEdgeId = column[Int]("node_label_edge_id") + def sourceId = column[String]("source_id") - def * = (NodeLabelEdgeId, SourceId) + def * = (nodeLabelEdgeId, sourceId) - def NodeLabelEdge = foreignKey("node_label_edge_fk", NodeLabelEdgeId, nodeLabelEdges)(_.id) - def Source = foreignKey("source_fk", SourceId, sources)(_.id) + def nodeLabelEdge = foreignKey("node_label_edge_fk", nodeLabelEdgeId, nodeLabelEdges)(_.id) + def source = foreignKey("source_fk", sourceId, sources)(_.id) - def pk = primaryKey("node_label_edge_source_pk", (NodeLabelEdgeId, SourceId)) + def pk = primaryKey("node_label_edge_source_pk", (nodeLabelEdgeId, sourceId)) } protected final class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { - def NodeLabelLabel = column[String]("node_label_label") - def SourceId = column[String]("source_id") + def nodeLabelLabel = column[String]("node_label_label") + def sourceId = column[String]("source_id") - def * = (NodeLabelLabel, SourceId) + def * = (nodeLabelLabel, sourceId) - def NodeLabel = foreignKey("node_label_fk", NodeLabelLabel, nodeLabels)(_.label) - def Source = foreignKey("source_fk", SourceId, sources)(_.id) + def nodeLabel = foreignKey("node_label_fk", nodeLabelLabel, nodeLabels)(_.label) + def source = foreignKey("source_fk", sourceId, sources)(_.id) - def pk = primaryKey("node_label_source_pk", (NodeLabelLabel, SourceId)) + def pk = primaryKey("node_label_source_pk", (nodeLabelLabel, sourceId)) } protected final class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { From e974b15a2f6c530ac5020d412b617f6f6a750249 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 6 Nov 2020 13:35:46 -0500 Subject: [PATCH 34/51] refactor postgres config provider for test --- .../kg/stores/SlickDatabaseConfigProvider.scala | 4 +++- .../postgres/PostgresStoreConfigProvider.scala | 10 +++++++--- .../kg/stores/postgres/PostgresKgStoreSpec.scala | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala index 8d08d234..4e84cc96 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/SlickDatabaseConfigProvider.scala @@ -1,6 +1,6 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores - +import com.typesafe.config.Config import slick.basic.{BasicProfile, DatabaseConfig} import scala.reflect.ClassTag @@ -22,4 +22,6 @@ class SlickDatabaseConfigProvider[T <: BasicProfile : ClassTag](val databaseConf def this(path: String) = this(DatabaseConfig.forConfig[T](path)) + def this(path: String, config: Config) = + this(DatabaseConfig.forConfig[T](path, config)) } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala index 7dfb21de..5a4c7c32 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -1,8 +1,12 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres +import com.typesafe.config.Config import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider -import javax.inject.Singleton import slick.jdbc.PostgresProfile -@Singleton -final class PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile]("postgres") +trait PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile] + +object PostgresStoreConfigProvider { + def apply() = new SlickDatabaseConfigProvider[PostgresProfile]("postgres") with PostgresStoreConfigProvider + def apply(path: String, config: Config) = new SlickDatabaseConfigProvider[PostgresProfile](path, config) with PostgresStoreConfigProvider +} diff --git a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala index af43f6f8..debb8760 100644 --- a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala +++ b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala @@ -2,14 +2,25 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import java.net.InetAddress +import com.typesafe.config.ConfigFactory import io.github.tetherlessworld.mcsapps.lib.kg.data.TestKgData import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreBehaviors, KgQueryStore, KgStoreFactory, StoreTestMode} import org.scalatest.{BeforeAndAfterAll, WordSpec} +import scala.collection.JavaConverters.mapAsJavaMap class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommandStoreBehaviors { import scala.concurrent.ExecutionContext.Implicits.global - val configProvider = new PostgresStoreConfigProvider() + val testConfig = ConfigFactory.parseMap(mapAsJavaMap(Map( + "postgres.profile" -> "slick.jdbc.PostgresProfile$", + "postgres.db.connectionPool" -> "HikariCP", + "postgres.db.driver" -> "org.postgresql.Driver", + "postgres.db.password" -> "7EAdu7jJvZNxxrNZ", + "postgres.db.url" -> "jdbc:postgresql://mcs-postgres:5432/kg", + "postgres.db.user" -> "mcs" + ))) + + val configProvider = PostgresStoreConfigProvider("postgres", testConfig) val command = new PostgresKgCommandStore(configProvider) val query = new PostgresKgQueryStore(configProvider) val postgresHostAddress = InetAddress.getByName("mcs-postgres").getHostAddress @@ -42,7 +53,7 @@ class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommand } if (inTestingEnvironment) { - "The neo4j store" can { + "The postgres store" can { behave like commandStore(PostgresKgStoreFactory) // behave like queryStore(PostgresKgStoreFactory) } From 368c7b8e2bb9b8019c96ae2df95f9c9db320b0a6 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 6 Nov 2020 14:07:23 -0500 Subject: [PATCH 35/51] close db connection after test --- .../mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala index debb8760..438f6e14 100644 --- a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala +++ b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala @@ -26,6 +26,10 @@ class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommand val postgresHostAddress = InetAddress.getByName("mcs-postgres").getHostAddress val inTestingEnvironment = System.getenv("CI") != null || postgresHostAddress != "128.113.12.49" + override def afterAll(): Unit = { + configProvider.databaseConfig.db.close() + } + override def beforeAll(): Unit = { if (!inTestingEnvironment) { return From f5cbab7665165a19e5e6bfde4db7205e8683c9e8 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 6 Nov 2020 14:18:17 -0500 Subject: [PATCH 36/51] truncate all tables at once --- .../lib/kg/stores/postgres/AbstractPostgresKgStore.scala | 2 +- .../lib/kg/stores/postgres/PostgresKgCommandStore.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index c472c327..119454c0 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -27,7 +27,7 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] - private lazy val tables = + protected lazy val tables = List(edges, edgeLabels, edgeSources, nodes, nodeLabels, nodeLabelEdges, nodeLabelEdgeSources, nodeLabelSource, nodeSources, nodeNodeLabels, sources) protected lazy val tablesDdlObject = tables.map(_.schema).reduce((left, right) => left ++ right) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 7810c54a..62f778c2 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -42,7 +42,8 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid } override final def clear(): Unit = { - runSyncTransaction(tablesDdlObject.truncate) + val tableNames = tables.map(_.baseTableRow.tableName).mkString(",") + runSyncTransaction(sqlu"TRUNCATE #$tableNames;") } private def generateEdgeInsert(edge: KgEdge) = From efa2035f5e99a455740b76a0d02fc8738a434273 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 6 Nov 2020 14:29:11 -0500 Subject: [PATCH 37/51] use insertOrUpdate to handle duplicates --- .../stores/postgres/PostgresKgCommandStore.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 62f778c2..35d5f544 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -47,17 +47,17 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid } private def generateEdgeInsert(edge: KgEdge) = - List(edges += edge.toRow) ++ - edge.labels.map(label => edgeLabels += (edge.id, label)) ++ - edge.sourceIds.map(sourceId => edgeSources += (edge.id, sourceId)) + List(edges.insertOrUpdate(edge.toRow)) ++ + edge.labels.map(label => edgeLabels.insertOrUpdate((edge.id, label))) ++ + edge.sourceIds.map(sourceId => edgeSources.insertOrUpdate((edge.id, sourceId))) private def generateNodeInsert(node: KgNode) = - List(nodes += node.toRow) ++ - node.labels.map(label => nodeNodeLabels += (node.id, label)) ++ - node.sourceIds.map(sourceId => nodeSources += (node.id, sourceId)) + List(nodes.insertOrUpdate(node.toRow)) ++ + node.labels.map(label => nodeNodeLabels.insertOrUpdate((node.id, label))) ++ + node.sourceIds.map(sourceId => nodeSources.insertOrUpdate((node.id, sourceId))) private def generateSourceInsert(source: KgSource) = - List(sources += (source.id, source.label)) + List(sources.insertOrUpdate(source.id, source.label)) override final def putData(data: KgData) = runSyncTransaction(DBIO.sequence( From 334113461a40195858ec8797a205756050f32ce4 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Mon, 9 Nov 2020 10:30:24 -0500 Subject: [PATCH 38/51] add extended postgres profile to support native upsert --- build.sbt | 1 + .../postgres/ExtendedPostgresProfile.scala | 16 ++++++++++++++++ .../postgres/PostgresStoreConfigProvider.scala | 7 +++---- .../kg/stores/postgres/PostgresKgStoreSpec.scala | 10 ++++++---- 4 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala diff --git a/build.sbt b/build.sbt index ab2d49c0..72d7a535 100644 --- a/build.sbt +++ b/build.sbt @@ -46,6 +46,7 @@ lazy val kgLib = .settings( libraryDependencies ++= Seq( // Implement search in the MemStore (and thus the TestStore) + "com.github.tminglei" %% "slick-pg" % "0.19.3", "com.outr" %% "lucene4s" % "1.9.1", "com.typesafe.slick" %% "slick" % "3.3.3", "com.typesafe.slick" %% "slick-hikaricp" % "3.3.3", diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala new file mode 100644 index 00000000..f6c22ab3 --- /dev/null +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala @@ -0,0 +1,16 @@ +package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres + +import slick.basic.Capability +import slick.jdbc.{JdbcCapabilities} +import com.github.tminglei.slickpg._ + +trait ExtendedPostgresProfile extends ExPostgresProfile { + override protected def computeCapabilities: Set[Capability] = + super.computeCapabilities + JdbcCapabilities.insertOrUpdate + + override val api = ExtendedAPI + + object ExtendedAPI extends API +} + +object ExtendedPostgresProfile extends ExtendedPostgresProfile \ No newline at end of file diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala index 5a4c7c32..c4885c7f 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresStoreConfigProvider.scala @@ -2,11 +2,10 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import com.typesafe.config.Config import io.github.tetherlessworld.mcsapps.lib.kg.stores.SlickDatabaseConfigProvider -import slick.jdbc.PostgresProfile -trait PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[PostgresProfile] +trait PostgresStoreConfigProvider extends SlickDatabaseConfigProvider[ExtendedPostgresProfile] object PostgresStoreConfigProvider { - def apply() = new SlickDatabaseConfigProvider[PostgresProfile]("postgres") with PostgresStoreConfigProvider - def apply(path: String, config: Config) = new SlickDatabaseConfigProvider[PostgresProfile](path, config) with PostgresStoreConfigProvider + def apply() = new SlickDatabaseConfigProvider[ExtendedPostgresProfile]("postgres") with PostgresStoreConfigProvider + def apply(path: String, config: Config) = new SlickDatabaseConfigProvider[ExtendedPostgresProfile](path, config) with PostgresStoreConfigProvider } diff --git a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala index 438f6e14..7de79ffe 100644 --- a/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala +++ b/lib/scala/kg/src/test/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgStoreSpec.scala @@ -4,15 +4,17 @@ import java.net.InetAddress import com.typesafe.config.ConfigFactory import io.github.tetherlessworld.mcsapps.lib.kg.data.TestKgData -import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreBehaviors, KgQueryStore, KgStoreFactory, StoreTestMode} +import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreBehaviors, KgQueryStore, KgQueryStoreBehaviors, KgStoreFactory, StoreTestMode} import org.scalatest.{BeforeAndAfterAll, WordSpec} + import scala.collection.JavaConverters.mapAsJavaMap -class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommandStoreBehaviors { +class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommandStoreBehaviors with KgQueryStoreBehaviors { import scala.concurrent.ExecutionContext.Implicits.global val testConfig = ConfigFactory.parseMap(mapAsJavaMap(Map( - "postgres.profile" -> "slick.jdbc.PostgresProfile$", +// "postgres.profile" -> "slick.jdbc.PostgresProfile$", + "postgres.profile" -> "io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres.ExtendedPostgresProfile$", "postgres.db.connectionPool" -> "HikariCP", "postgres.db.driver" -> "org.postgresql.Driver", "postgres.db.password" -> "7EAdu7jJvZNxxrNZ", @@ -59,7 +61,7 @@ class PostgresKgStoreSpec extends WordSpec with BeforeAndAfterAll with KgCommand if (inTestingEnvironment) { "The postgres store" can { behave like commandStore(PostgresKgStoreFactory) -// behave like queryStore(PostgresKgStoreFactory) + behave like queryStore(PostgresKgStoreFactory) } } } From c3ec5dd721a4f8547272e2783e83fa55563eb81e Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Mon, 9 Nov 2020 10:31:43 -0500 Subject: [PATCH 39/51] trying out queries for node labels and node --- .../postgres/AbstractPostgresKgStore.scala | 53 ++++++++++++++---- .../postgres/PostgresKgCommandStore.scala | 16 +++++- .../postgres/PostgresKgQueryStore.scala | 55 ++++++++++++++++--- 3 files changed, 103 insertions(+), 21 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 119454c0..2fbfb7dc 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -1,7 +1,8 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge -import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.{KgNode, KgNodeLabel} +import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.HasDatabaseConfigProvider import slick.jdbc.PostgresProfile @@ -9,7 +10,7 @@ import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} -abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: PostgresStoreConfigProvider) extends HasDatabaseConfigProvider[PostgresProfile] { +abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: PostgresStoreConfigProvider) extends HasDatabaseConfigProvider[ExtendedPostgresProfile] { import profile.api._ protected val SentencesDelimChar = '|' @@ -18,8 +19,26 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos protected lazy val edges = TableQuery[EdgeTable] protected lazy val edgeLabels = TableQuery[EdgeLabelTable] protected lazy val edgeSources = TableQuery[EdgeSourceTable] - protected lazy val nodes = TableQuery[NodeTable] - protected lazy val nodeLabels = TableQuery[NodeLabelTable] + protected object nodes extends TableQuery(new NodeTable(_)) { + def withNodeLabels = for { + ((node, _), nodeLabel) <- nodes + .join(nodeNodeLabels).on(_.id === _.nodeId) + .join(nodeLabels).on(_._2.nodeLabelLabel === _.label) + } yield (node, nodeLabel) + + def withSources = for { + ((node, _), source) <- nodes + .join(nodeSources).on(_.id === _.nodeId) + .join(sources).on(_._2.sourceId === _.id) + } yield (node, source) + } + protected object nodeLabels extends TableQuery(new NodeLabelTable(_)) { + def withNodes = for { + ((nodeLabel, _), node) <- nodeLabels + .join(nodeNodeLabels).on(_.label === _.nodeLabelLabel) + .join(nodes).on(_._2.nodeId === _.id) + } yield (nodeLabel, node) + } protected lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] protected lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] protected lazy val nodeLabelSource = TableQuery[NodeLabelSourceTable] @@ -63,6 +82,17 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos wordNetSenseNumber = wordNetSenseNumber.map(_.toInt) ) } + protected final case class NodeLabelRow(label: String, pageRank: Option[Float]) { + def toKgNodeLabel(nodes: List[KgNode], sourceIds: List[String]) = KgNodeLabel( + nodeLabel = label, + nodes = nodes, + pageRank = pageRank.map(_.toDouble), + sourceIds = sourceIds + ) + } + protected final case class SourceRow(id: String, label: String) { + def toKgSource = KgSource(id = id, label = label) + } protected final class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) @@ -114,13 +144,14 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos protected final class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { def nodeId = column[String]("node_id") - def label = column[String]("label") + def nodeLabelLabel = column[String]("label") - def * = (nodeId, label) + def * = (nodeId, nodeLabelLabel) def node = foreignKey("node_fk", nodeId, nodes)(_.id) + def nodeLabel = foreignKey("node_label_fk", nodeLabelLabel, nodeLabels)(_.label) - def pk = primaryKey("node_label_pk", (nodeId, label)) + def pk = primaryKey("node_label_pk", (nodeId, nodeLabelLabel)) } protected final class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { @@ -135,11 +166,11 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def pk = primaryKey("node_source_pk", (nodeId, sourceId)) } - protected final class NodeLabelTable(tag: Tag) extends Table[(String, Option[Float])](tag, "node_x_label") { + protected final class NodeLabelTable(tag: Tag) extends Table[NodeLabelRow](tag, "node_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") - def * = (label, pageRank) + def * = (label, pageRank) <> (NodeLabelRow.tupled, NodeLabelRow.unapply) } protected final class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { @@ -179,10 +210,10 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def pk = primaryKey("node_label_source_pk", (nodeLabelLabel, sourceId)) } - protected final class SourceTable(tag: Tag) extends Table[(String, String)](tag, "source") { + protected final class SourceTable(tag: Tag) extends Table[SourceRow](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") - def * = (id, label) + def * = (id, label) <> (SourceRow.tupled, SourceRow.unapply) } } diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 35d5f544..57964ee8 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -4,7 +4,7 @@ import com.google.inject.{Inject, Singleton} import io.github.tetherlessworld.mcsapps.lib.kg.data.KgData import io.github.tetherlessworld.mcsapps.lib.kg.formats.kgtk.KgtkEdgeWithNodes import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge -import io.github.tetherlessworld.mcsapps.lib.kg.models.node.KgNode +import io.github.tetherlessworld.mcsapps.lib.kg.models.node.{KgNode, KgNodeLabel} import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.{KgCommandStore, KgCommandStoreTransaction} @@ -41,6 +41,13 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid ) } + private implicit class KgSourceWrapper(source: KgSource) { + def toRow: SourceRow = SourceRow( + id = source.id, + label = source.label + ) + } + override final def clear(): Unit = { val tableNames = tables.map(_.baseTableRow.tableName).mkString(",") runSyncTransaction(sqlu"TRUNCATE #$tableNames;") @@ -53,11 +60,14 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid private def generateNodeInsert(node: KgNode) = List(nodes.insertOrUpdate(node.toRow)) ++ - node.labels.map(label => nodeNodeLabels.insertOrUpdate((node.id, label))) ++ + node.labels.flatMap { label => List( + nodeLabels.insertOrUpdate(NodeLabelRow(label, None)), + nodeNodeLabels.insertOrUpdate((node.id, label)) + )} ++ node.sourceIds.map(sourceId => nodeSources.insertOrUpdate((node.id, sourceId))) private def generateSourceInsert(source: KgSource) = - List(sources.insertOrUpdate(source.id, source.label)) + List(sources.insertOrUpdate(source.toRow)) override final def putData(data: KgData) = runSyncTransaction(DBIO.sequence( diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 4baf4b35..381759af 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -15,12 +15,51 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr import profile.api._ override def getNode(id: String): Option[KgNode] = { - runSyncTransaction((for { node <- nodes if node.id === id } yield node).result.headOption).map(_.toKgNode(List(), List())) + runSyncTransaction((for { + ((node, source), (_, nodeLabel)) <- nodes + .withSources + .join(nodes.withNodeLabels) + .on(_._1.id === _._1.id) + .filter(_._1._1.id === id) + } yield (node, nodeLabel.label, source.id)).result) + .groupBy(_._1.id) + .values + .map { rows => + rows.head._1.toKgNode( + labels = rows.map(_._2).distinct.toList, + sourceIds = rows.map(_._3).distinct.toList + ) + } + .headOption } - + override def getNodeContext(id: String): Option[KgNodeContext] = None - override def getNodeLabel(label: String): Option[KgNodeLabel] = None + override def getNodeLabel(label: String): Option[KgNodeLabel] = { + runSyncTransaction((for { + nodeLabel <- nodeLabels if nodeLabel.label === label + nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeLabelLabel === nodeLabel.label + nodeLabelSource <- nodeLabelSource if nodeLabelSource.nodeLabelLabel === nodeLabel.label + node <- nodeNodeLabel.node + source <- nodeLabelSource.source + nodeSource <- nodeSources if nodeSource.nodeId === node.id + nodeLabel <- nodeNodeLabel.nodeLabel + nodeSourceSource <- nodeSource.source + } yield (nodeLabel, source.id, node, nodeSourceSource.id, nodeLabel.label)).result) + .groupBy(_._1.label) + .values.map { + rows => rows.head._1.toKgNodeLabel( + sourceIds = rows.map(_._2).distinct.toList, + nodes = rows.groupBy(_._3.id).values.map { + nodeRows => nodeRows.head._3.toKgNode( + labels = nodeRows.map(_._4).distinct.toList, + sourceIds = nodeRows.map(_._5).distinct.toList + ) + }.toList + ) + } + .headOption + } override def getNodeLabelContext(label: String): Option[KgNodeLabelContext] = None @@ -28,13 +67,15 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr override def getRandomNode: KgNode = KgNode("", None, List(), None, None, None, List(), None) - override def getSourcesById: Map[String, KgSource] = Map() + override def getSourcesById: Map[String, KgSource] = { + runSyncTransaction(sources.result).map(source => (source.id, source.toKgSource)).toMap + } - override def getTotalEdgesCount: Int = 0 + override def getTotalEdgesCount: Int = runSyncTransaction(edges.size.result) - override def getTotalNodesCount: Int = 0 + override def getTotalNodesCount: Int = runSyncTransaction(nodes.size.result) - override def isEmpty: Boolean = true + override def isEmpty: Boolean = getTotalNodesCount == 0 override def search(limit: Int, offset: Int, query: KgSearchQuery, sorts: Option[List[KgSearchSort]]): List[KgSearchResult] = List() From 67e02d56793b50ec26e04ad8854608e9d90445fd Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 10 Nov 2020 01:24:02 -0500 Subject: [PATCH 40/51] Update nodeLabelSources --- .../kg/stores/postgres/PostgresKgCommandStore.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 57964ee8..bff929a7 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -60,10 +60,14 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid private def generateNodeInsert(node: KgNode) = List(nodes.insertOrUpdate(node.toRow)) ++ - node.labels.flatMap { label => List( - nodeLabels.insertOrUpdate(NodeLabelRow(label, None)), - nodeNodeLabels.insertOrUpdate((node.id, label)) - )} ++ + node.labels.flatMap { label => + List( + nodeLabels.insertOrUpdate(NodeLabelRow(label, None)), + nodeNodeLabels.insertOrUpdate((node.id, label)) + ) ++ node.sourceIds.map( + sourceId => nodeLabelSources.insertOrUpdate((label, sourceId)) + ) + } ++ node.sourceIds.map(sourceId => nodeSources.insertOrUpdate((node.id, sourceId))) private def generateSourceInsert(source: KgSource) = From 36f6655fc8fd547cdcfc6571716ab8fbc2b7a3d1 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Tue, 17 Nov 2020 15:48:10 -0500 Subject: [PATCH 41/51] add writeNodeLabelEdges and writeNodeLabelEdgeSources --- .../postgres/AbstractPostgresKgStore.scala | 14 ++++-- .../postgres/PostgresKgCommandStore.scala | 47 ++++++++++++++++++- .../postgres/PostgresKgQueryStore.scala | 23 +++++---- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 2fbfb7dc..770cd3af 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -38,16 +38,22 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos .join(nodeNodeLabels).on(_.label === _.nodeLabelLabel) .join(nodes).on(_._2.nodeId === _.id) } yield (nodeLabel, node) + + def withSources = for { + ((nodeLabel, _), source) <- nodeLabels + .join(nodeLabelSources).on(_.label === _.nodeLabelLabel) + .join(sources).on(_._2.sourceId === _.id) + } yield (nodeLabel, source) } protected lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] protected lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] - protected lazy val nodeLabelSource = TableQuery[NodeLabelSourceTable] + protected lazy val nodeLabelSources = TableQuery[NodeLabelSourceTable] protected lazy val nodeSources = TableQuery[NodeSourceTable] protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] protected lazy val sources = TableQuery[SourceTable] protected lazy val tables = - List(edges, edgeLabels, edgeSources, nodes, nodeLabels, nodeLabelEdges, nodeLabelEdgeSources, nodeLabelSource, nodeSources, nodeNodeLabels, sources) + List(edges, edgeLabels, edgeSources, nodes, nodeLabels, nodeLabelEdges, nodeLabelEdgeSources, nodeLabelSources, nodeSources, nodeNodeLabels, sources) protected lazy val tablesDdlObject = tables.map(_.schema).reduce((left, right) => left ++ right) @@ -173,12 +179,12 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def * = (label, pageRank) <> (NodeLabelRow.tupled, NodeLabelRow.unapply) } - protected final class NodeLabelEdgeTable(tag: Tag) extends Table[(Int, String, String)](tag, "node_label_edge") { + protected final class NodeLabelEdgeTable(tag: Tag) extends Table[(Option[Int], String, String)](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def objectNodeLabelLabel = column[String]("object_node_label_label") def subjectNodeLabelLabel = column[String]("subject_node_label_label") - def * = (id, objectNodeLabelLabel, subjectNodeLabelLabel) + def * = (id.?, objectNodeLabelLabel, subjectNodeLabelLabel) def objectNodeLabel = foreignKey("object_node_label_fk", objectNodeLabelLabel, nodeLabels)(_.label) def subjectNodeLabel = foreignKey("subject_node_label_fk", subjectNodeLabelLabel, nodeLabels)(_.label) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index bff929a7..5b65e643 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -53,6 +53,11 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid runSyncTransaction(sqlu"TRUNCATE #$tableNames;") } + override final def close(): Unit = { + writeNodeLabelEdges + writeNodeLabelEdgeSources + } + private def generateEdgeInsert(edge: KgEdge) = List(edges.insertOrUpdate(edge.toRow)) ++ edge.labels.map(label => edgeLabels.insertOrUpdate((edge.id, label))) ++ @@ -99,7 +104,47 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid override final def putSources(sources: Iterator[KgSource]): Unit = runSyncTransaction(DBIO.sequence(sources.flatMap(generateSourceInsert))) - override final def close(): Unit = Unit + private def writeNodeLabelEdges: Unit = { + val nodeLabelEdgeNodeLabelsQuery = (for { + edge <- edges + objectNode <- edge.objectNode + subjectNode <- edge.subjectNode + objectNodeNodeLabel <- nodeNodeLabels if objectNodeNodeLabel.nodeId === objectNode.id + subjectNodeNodeLabel <- nodeNodeLabels if subjectNodeNodeLabel.nodeId === subjectNode.id + objectNodeLabel <- objectNodeNodeLabel.nodeLabel + subjectNodeLabel <- subjectNodeNodeLabel.nodeLabel + } yield (objectNodeLabel.label, subjectNodeLabel.label)).result + + val nodeLabelEdgeInserts = runSyncTransaction(nodeLabelEdgeNodeLabelsQuery).map { + case (objectNodeLabelLabel, subjectNodeLabelLabel) => + sqlu"INSERT INTO #${nodeLabelEdges.baseTableRow.tableName} (object_node_label_label, subject_node_label_label) VALUES ($objectNodeLabelLabel, $subjectNodeLabelLabel) ON CONFLICT DO NOTHING;" + } + + runSyncTransaction(DBIO.sequence(nodeLabelEdgeInserts)) + } + + private def writeNodeLabelEdgeSources: Unit = { + val objectNodeLabelEdgeSourcesQuery = (for { + nodeLabelEdge <- nodeLabelEdges + objectNodeLabel <- nodeLabelEdge.objectNodeLabel + objectNodeLabelSourceSource <- nodeLabelSources if objectNodeLabelSourceSource.nodeLabelLabel === objectNodeLabel.label + objectNodeLabelSource <- objectNodeLabelSourceSource.source + } yield (nodeLabelEdge.id, objectNodeLabelSource.id)) + + val subjectNodeLabelEdgeSourcesQuery = (for { + nodeLabelEdge <- nodeLabelEdges + subjectNodeLabel <- nodeLabelEdge.subjectNodeLabel + subjectNodeLabelSourceSource <- nodeLabelSources if subjectNodeLabelSourceSource.nodeLabelLabel === subjectNodeLabel.label + subjectNodeLabelSource <- subjectNodeLabelSourceSource.source + } yield (nodeLabelEdge.id, subjectNodeLabelSource.id)) + + val nodeLabelEdgeSourcesAction = (objectNodeLabelEdgeSourcesQuery ++ subjectNodeLabelEdgeSourcesQuery).result + + runSyncTransaction(for { + nodeLabelEdgeSourcesResult <- nodeLabelEdgeSourcesAction + _ <- nodeLabelEdgeSources.insertOrUpdateAll(nodeLabelEdgeSourcesResult) + } yield ()) + } } bootstrapStore() diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 381759af..963209ba 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -15,13 +15,15 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr import profile.api._ override def getNode(id: String): Option[KgNode] = { - runSyncTransaction((for { + val query = (for { ((node, source), (_, nodeLabel)) <- nodes .withSources .join(nodes.withNodeLabels) .on(_._1.id === _._1.id) .filter(_._1._1.id === id) - } yield (node, nodeLabel.label, source.id)).result) + } yield (node, nodeLabel.label, source.id)).result + + runSyncTransaction(query) .groupBy(_._1.id) .values .map { rows => @@ -32,22 +34,25 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr } .headOption } - + override def getNodeContext(id: String): Option[KgNodeContext] = None override def getNodeLabel(label: String): Option[KgNodeLabel] = { - runSyncTransaction((for { + val query = (for { nodeLabel <- nodeLabels if nodeLabel.label === label nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeLabelLabel === nodeLabel.label - nodeLabelSource <- nodeLabelSource if nodeLabelSource.nodeLabelLabel === nodeLabel.label + nodeLabelSource <- nodeLabelSources if nodeLabelSource.nodeLabelLabel === nodeLabel.label node <- nodeNodeLabel.node source <- nodeLabelSource.source nodeSource <- nodeSources if nodeSource.nodeId === node.id - nodeLabel <- nodeNodeLabel.nodeLabel - nodeSourceSource <- nodeSource.source - } yield (nodeLabel, source.id, node, nodeSourceSource.id, nodeLabel.label)).result) + nodeNodeLabel <- nodeNodeLabel.nodeLabel + nodeNodeSource <- nodeSource.source + } yield (nodeLabel, source.id, node, nodeNodeSource.id, nodeNodeLabel.label)).result + + runSyncTransaction(query) .groupBy(_._1.label) - .values.map { + .values + .map { rows => rows.head._1.toKgNodeLabel( sourceIds = rows.map(_._2).distinct.toList, nodes = rows.groupBy(_._3.id).values.map { From 78b6e8d0897f809941bdb0c79a08177c60fb2813 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 20 Nov 2020 14:16:10 -0500 Subject: [PATCH 42/51] use insertOrUpdateAll --- .../postgres/PostgresKgCommandStore.scala | 119 +++++++++--------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 5b65e643..8533c228 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -48,102 +48,107 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid ) } + private def batchedEdgeInserts(kgEdges: Iterator[KgEdge]) = { + val stream = kgEdges.toStream + List( + edges.insertOrUpdateAll(stream.map(_.toRow)), + edgeLabels.insertOrUpdateAll(stream.flatMap(edge => edge.labels.map(label => EdgeLabelRow(edge.id, label)))), + edgeSources.insertOrUpdateAll(stream.flatMap(edge => edge.sourceIds.map(sourceId => EdgeSourceRow(edge.id, sourceId)))) + ) + } + + private def batchedNodeInserts(kgNodes: Iterator[KgNode]) = { + val stream = kgNodes.toStream + List( + nodes.insertOrUpdateAll(stream.map(_.toRow)), + nodeLabels.insertOrUpdateAll(stream.flatMap(_.labels.map(NodeLabelRow(_, None)))), + nodeNodeLabels.insertOrUpdateAll(stream.flatMap(node => node.labels.map(label => NodeNodeLabelRow(node.id, label)))), + nodeLabelSources.insertOrUpdateAll(stream.flatMap(node => node.labels.flatMap(label => node.sourceIds.map(NodeLabelSourceRow(label, _))))), + nodeSources.insertOrUpdateAll(stream.flatMap(node => node.sourceIds.map(NodeSourceRow(node.id, _)))) + ) + } + + private def batchedSourceInserts(kgSources: Iterator[KgSource]) = + List(sources.insertOrUpdateAll(kgSources.map(_.toRow).toIterable)) + override final def clear(): Unit = { val tableNames = tables.map(_.baseTableRow.tableName).mkString(",") runSyncTransaction(sqlu"TRUNCATE #$tableNames;") } override final def close(): Unit = { - writeNodeLabelEdges - writeNodeLabelEdgeSources + runSyncTransaction(DBIO.seq( + writeNodeLabelEdgesAction, + writeNodeLabelEdgeSourcesAction + )) } - private def generateEdgeInsert(edge: KgEdge) = - List(edges.insertOrUpdate(edge.toRow)) ++ - edge.labels.map(label => edgeLabels.insertOrUpdate((edge.id, label))) ++ - edge.sourceIds.map(sourceId => edgeSources.insertOrUpdate((edge.id, sourceId))) - - private def generateNodeInsert(node: KgNode) = - List(nodes.insertOrUpdate(node.toRow)) ++ - node.labels.flatMap { label => - List( - nodeLabels.insertOrUpdate(NodeLabelRow(label, None)), - nodeNodeLabels.insertOrUpdate((node.id, label)) - ) ++ node.sourceIds.map( - sourceId => nodeLabelSources.insertOrUpdate((label, sourceId)) - ) - } ++ - node.sourceIds.map(sourceId => nodeSources.insertOrUpdate((node.id, sourceId))) - - private def generateSourceInsert(source: KgSource) = - List(sources.insertOrUpdate(source.toRow)) - override final def putData(data: KgData) = - runSyncTransaction(DBIO.sequence( - data.sources.flatMap(generateSourceInsert) ++ - data.nodesUnranked.flatMap(generateNodeInsert) ++ - data.edges.flatMap(generateEdgeInsert) - )) + runSyncTransaction( + DBIO.sequence( + batchedSourceInserts(data.sources.iterator) ++ + batchedNodeInserts(data.nodesUnranked.iterator) ++ + batchedEdgeInserts(data.edges.iterator) + ) + ) - override final def putEdges(edges: Iterator[KgEdge]) = - runSyncTransaction(DBIO.sequence(edges.flatMap(generateEdgeInsert))) + override final def putEdges(kgEdges: Iterator[KgEdge]) = + runSyncTransaction(DBIO.sequence(batchedEdgeInserts(kgEdges))) override final def putKgtkEdgesWithNodes(edgesWithNodes: Iterator[KgtkEdgeWithNodes]): Unit = runSyncTransaction(DBIO.sequence( - edgesWithNodes.flatMap { edge => - edge.nodes.flatMap(generateNodeInsert) ++ - generateEdgeInsert(edge.edge) - } + batchedNodeInserts(edgesWithNodes.flatMap(_.nodes)) ++ + batchedEdgeInserts(edgesWithNodes.map(_.edge)) )) override final def putNodes(nodes: Iterator[KgNode]): Unit = - runSyncTransaction(DBIO.sequence(nodes.flatMap(generateNodeInsert))) + runSyncTransaction(DBIO.sequence(batchedNodeInserts(nodes))) override final def putPaths(paths: Iterator[KgPath]): Unit = Unit override final def putSources(sources: Iterator[KgSource]): Unit = - runSyncTransaction(DBIO.sequence(sources.flatMap(generateSourceInsert))) + runSyncTransaction(DBIO.sequence(batchedSourceInserts(sources))) - private def writeNodeLabelEdges: Unit = { - val nodeLabelEdgeNodeLabelsQuery = (for { - edge <- edges + private def writeNodeLabelEdgesAction = { + val nodeLabelEdgePairsAction = (for { + (subjectNodeLabel, node) <- nodeLabels.withNodes + edge <- edges if edge.subjectNodeId === node.id objectNode <- edge.objectNode - subjectNode <- edge.subjectNode - objectNodeNodeLabel <- nodeNodeLabels if objectNodeNodeLabel.nodeId === objectNode.id - subjectNodeNodeLabel <- nodeNodeLabels if subjectNodeNodeLabel.nodeId === subjectNode.id + objectNodeNodeLabel <- nodeNodeLabels if objectNodeNodeLabel.nodeId === objectNode.id && objectNodeNodeLabel.nodeLabelLabel =!= subjectNodeLabel.label objectNodeLabel <- objectNodeNodeLabel.nodeLabel - subjectNodeLabel <- subjectNodeNodeLabel.nodeLabel - } yield (objectNodeLabel.label, subjectNodeLabel.label)).result - - val nodeLabelEdgeInserts = runSyncTransaction(nodeLabelEdgeNodeLabelsQuery).map { - case (objectNodeLabelLabel, subjectNodeLabelLabel) => - sqlu"INSERT INTO #${nodeLabelEdges.baseTableRow.tableName} (object_node_label_label, subject_node_label_label) VALUES ($objectNodeLabelLabel, $subjectNodeLabelLabel) ON CONFLICT DO NOTHING;" - } + } yield (subjectNodeLabel.label, objectNodeLabel.label)).result - runSyncTransaction(DBIO.sequence(nodeLabelEdgeInserts)) + for { + nodeLabelEdgePairs <- nodeLabelEdgePairsAction + _ <- nodeLabelEdges ++= nodeLabelEdgePairs.distinct.map { case (objectNodeLabelLabel, subjectNodeLabelLabel) => + NodeLabelEdgeRow(Some(0), objectNodeLabelLabel, subjectNodeLabelLabel) + } + } yield () } - private def writeNodeLabelEdgeSources: Unit = { - val objectNodeLabelEdgeSourcesQuery = (for { + private def writeNodeLabelEdgeSourcesAction = { + val objectNodeLabelEdgeSourcesQuery = for { nodeLabelEdge <- nodeLabelEdges objectNodeLabel <- nodeLabelEdge.objectNodeLabel objectNodeLabelSourceSource <- nodeLabelSources if objectNodeLabelSourceSource.nodeLabelLabel === objectNodeLabel.label objectNodeLabelSource <- objectNodeLabelSourceSource.source - } yield (nodeLabelEdge.id, objectNodeLabelSource.id)) + } yield (nodeLabelEdge.id, objectNodeLabelSource.id) - val subjectNodeLabelEdgeSourcesQuery = (for { + val subjectNodeLabelEdgeSourcesQuery = for { nodeLabelEdge <- nodeLabelEdges subjectNodeLabel <- nodeLabelEdge.subjectNodeLabel subjectNodeLabelSourceSource <- nodeLabelSources if subjectNodeLabelSourceSource.nodeLabelLabel === subjectNodeLabel.label subjectNodeLabelSource <- subjectNodeLabelSourceSource.source - } yield (nodeLabelEdge.id, subjectNodeLabelSource.id)) + } yield (nodeLabelEdge.id, subjectNodeLabelSource.id) val nodeLabelEdgeSourcesAction = (objectNodeLabelEdgeSourcesQuery ++ subjectNodeLabelEdgeSourcesQuery).result - runSyncTransaction(for { + for { nodeLabelEdgeSourcesResult <- nodeLabelEdgeSourcesAction - _ <- nodeLabelEdgeSources.insertOrUpdateAll(nodeLabelEdgeSourcesResult) - } yield ()) + _ <- nodeLabelEdgeSources.insertOrUpdateAll(nodeLabelEdgeSourcesResult.map { + case (edgeId, sourceId) => NodeLabelEdgeSourceRow(edgeId, sourceId) + }) + } yield () } } From 57f20ea5e4ec82a8932c49a8e59afa3bddcb5d85 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 20 Nov 2020 14:16:31 -0500 Subject: [PATCH 43/51] use case classes for all tables --- .../postgres/AbstractPostgresKgStore.scala | 127 ++++++++++-------- 1 file changed, 68 insertions(+), 59 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 770cd3af..73a9b941 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -4,7 +4,6 @@ import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge import io.github.tetherlessworld.mcsapps.lib.kg.models.node.{KgNode, KgNodeLabel} import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.HasDatabaseConfigProvider -import slick.jdbc.PostgresProfile import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} @@ -48,8 +47,8 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos protected lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] protected lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] protected lazy val nodeLabelSources = TableQuery[NodeLabelSourceTable] - protected lazy val nodeSources = TableQuery[NodeSourceTable] protected lazy val nodeNodeLabels = TableQuery[NodeNodeLabelTable] + protected lazy val nodeSources = TableQuery[NodeSourceTable] protected lazy val sources = TableQuery[SourceTable] protected lazy val tables = @@ -65,6 +64,10 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos Await.result(runTransaction(a), duration) } + protected final def runSync[R](a: DBIOAction[R, NoStream, Effect.All], duration: Duration = Duration.Inf): R = { + Await.result(db.run(a), duration) + } + protected final case class EdgeRow(id: String, objectNodeId: String, predicate: String, sentences: String, subjectNodeId: String) { def toKgEdge(labels: List[String], sourceIds: List[String]) = KgEdge( id = id, @@ -76,30 +79,6 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos subject = subjectNodeId ) } - protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) { - def toKgNode(labels: List[String], sourceIds: List[String]) = KgNode( - id = id, - inDegree = inDegree.map(_.toInt), - labels = labels, - outDegree = outDegree.map(_.toInt), - pageRank = pageRank.map(_.toDouble), - pos = pos, - sourceIds = sourceIds, - wordNetSenseNumber = wordNetSenseNumber.map(_.toInt) - ) - } - protected final case class NodeLabelRow(label: String, pageRank: Option[Float]) { - def toKgNodeLabel(nodes: List[KgNode], sourceIds: List[String]) = KgNodeLabel( - nodeLabel = label, - nodes = nodes, - pageRank = pageRank.map(_.toDouble), - sourceIds = sourceIds - ) - } - protected final case class SourceRow(id: String, label: String) { - def toKgSource = KgSource(id = id, label = label) - } - protected final class EdgeTable(tag: Tag) extends Table[EdgeRow](tag, "edge") { def id = column[String]("id", O.PrimaryKey) def objectNodeId = column[String]("object_node_id") @@ -115,28 +94,42 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def unique_constraint = index("_edge_unique_idx", (objectNodeId, subjectNodeId, predicate), unique = true) } - protected final class EdgeLabelTable(tag: Tag) extends Table[(String, String)](tag, "edge_label") { + protected final case class EdgeLabelRow(edgeId: String, label: String) + protected final class EdgeLabelTable(tag: Tag) extends Table[EdgeLabelRow](tag, "edge_label") { def edgeId = column[String]("edge_id") def label = column[String]("label") - def * = (edgeId, label) + def * = (edgeId, label) <> (EdgeLabelRow.tupled, EdgeLabelRow.unapply) def edge = foreignKey("edge_fk", edgeId, edges)(_.id) def pk = primaryKey("edge_label_pk", (edgeId, label)) } - protected final class EdgeSourceTable(tag: Tag) extends Table[(String, String)](tag, "edge_x_source") { + protected final case class EdgeSourceRow(edgeId: String, sourceId: String) + protected final class EdgeSourceTable(tag: Tag) extends Table[EdgeSourceRow](tag, "edge_x_source") { def edgeId = column[String]("edge_id") def sourceId = column[String]("source_id") - def * = (edgeId, sourceId) + def * = (edgeId, sourceId) <> (EdgeSourceRow.tupled, EdgeSourceRow.unapply) def edge = foreignKey("edge_fk", edgeId, edges)(_.id) def pk = primaryKey("edge_source_pk", (edgeId, sourceId)) } + protected final case class NodeRow(id: String, inDegree: Option[Short], outDegree: Option[Short], pageRank: Option[Float], pos: Option[Char], wordNetSenseNumber: Option[Short]) { + def toKgNode(labels: List[String], sourceIds: List[String]) = KgNode( + id = id, + inDegree = inDegree.map(_.toInt), + labels = labels, + outDegree = outDegree.map(_.toInt), + pageRank = pageRank.map(_.toDouble), + pos = pos, + sourceIds = sourceIds, + wordNetSenseNumber = wordNetSenseNumber.map(_.toInt) + ) + } protected final class NodeTable(tag: Tag) extends Table[NodeRow](tag, "node") { def id = column[String]("id", O.PrimaryKey) def inDegree = column[Option[Short]]("in_degree") @@ -148,30 +141,14 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def * = (id, inDegree, outDegree, pageRank, pos, wordNetSenseNumber) <> (NodeRow.tupled, NodeRow.unapply) } - protected final class NodeNodeLabelTable(tag: Tag) extends Table[(String, String)](tag, "node_x_node_label") { - def nodeId = column[String]("node_id") - def nodeLabelLabel = column[String]("label") - - def * = (nodeId, nodeLabelLabel) - - def node = foreignKey("node_fk", nodeId, nodes)(_.id) - def nodeLabel = foreignKey("node_label_fk", nodeLabelLabel, nodeLabels)(_.label) - - def pk = primaryKey("node_label_pk", (nodeId, nodeLabelLabel)) - } - - protected final class NodeSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_x_source") { - def nodeId = column[String]("node_id") - def sourceId = column[String]("source_id") - - def * = (nodeId, sourceId) - - def node = foreignKey("node_fk", nodeId, nodes)(_.id) - def source = foreignKey("source_fk", sourceId, sources)(_.id) - - def pk = primaryKey("node_source_pk", (nodeId, sourceId)) + protected final case class NodeLabelRow(label: String, pageRank: Option[Float]) { + def toKgNodeLabel(nodes: List[KgNode], sourceIds: List[String]) = KgNodeLabel( + nodeLabel = label, + nodes = nodes, + pageRank = pageRank.map(_.toDouble), + sourceIds = sourceIds + ) } - protected final class NodeLabelTable(tag: Tag) extends Table[NodeLabelRow](tag, "node_label") { def label = column[String]("label", O.PrimaryKey) def pageRank = column[Option[Float]]("page_rank") @@ -179,12 +156,13 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def * = (label, pageRank) <> (NodeLabelRow.tupled, NodeLabelRow.unapply) } - protected final class NodeLabelEdgeTable(tag: Tag) extends Table[(Option[Int], String, String)](tag, "node_label_edge") { + protected final case class NodeLabelEdgeRow(pageRank: Option[Int], objectNodeLabelLabel: String, subjectNodeLabelLabel: String) + protected final class NodeLabelEdgeTable(tag: Tag) extends Table[NodeLabelEdgeRow](tag, "node_label_edge") { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def objectNodeLabelLabel = column[String]("object_node_label_label") def subjectNodeLabelLabel = column[String]("subject_node_label_label") - def * = (id.?, objectNodeLabelLabel, subjectNodeLabelLabel) + def * = (id.?, objectNodeLabelLabel, subjectNodeLabelLabel) <> (NodeLabelEdgeRow.tupled, NodeLabelEdgeRow.unapply) def objectNodeLabel = foreignKey("object_node_label_fk", objectNodeLabelLabel, nodeLabels)(_.label) def subjectNodeLabel = foreignKey("subject_node_label_fk", subjectNodeLabelLabel, nodeLabels)(_.label) @@ -192,11 +170,12 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def unique_constraint = index("node_label_edge_unique_idx", (objectNodeLabelLabel, subjectNodeLabelLabel), unique = true) } - protected final class NodeLabelEdgeSourceTable(tag: Tag) extends Table[(Int, String)](tag, "node_label_edge_x_source") { + protected final case class NodeLabelEdgeSourceRow(nodeLabelEdgeId: Int, sourceId: String) + protected final class NodeLabelEdgeSourceTable(tag: Tag) extends Table[NodeLabelEdgeSourceRow](tag, "node_label_edge_x_source") { def nodeLabelEdgeId = column[Int]("node_label_edge_id") def sourceId = column[String]("source_id") - def * = (nodeLabelEdgeId, sourceId) + def * = (nodeLabelEdgeId, sourceId) <> (NodeLabelEdgeSourceRow.tupled, NodeLabelEdgeSourceRow.unapply) def nodeLabelEdge = foreignKey("node_label_edge_fk", nodeLabelEdgeId, nodeLabelEdges)(_.id) def source = foreignKey("source_fk", sourceId, sources)(_.id) @@ -204,11 +183,12 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def pk = primaryKey("node_label_edge_source_pk", (nodeLabelEdgeId, sourceId)) } - protected final class NodeLabelSourceTable(tag: Tag) extends Table[(String, String)](tag, "node_label_x_source") { + protected final case class NodeLabelSourceRow(nodeLabelLabel: String, sourceId: String) + protected final class NodeLabelSourceTable(tag: Tag) extends Table[NodeLabelSourceRow](tag, "node_label_x_source") { def nodeLabelLabel = column[String]("node_label_label") def sourceId = column[String]("source_id") - def * = (nodeLabelLabel, sourceId) + def * = (nodeLabelLabel, sourceId) <> (NodeLabelSourceRow.tupled, NodeLabelSourceRow.unapply) def nodeLabel = foreignKey("node_label_fk", nodeLabelLabel, nodeLabels)(_.label) def source = foreignKey("source_fk", sourceId, sources)(_.id) @@ -216,6 +196,35 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def pk = primaryKey("node_label_source_pk", (nodeLabelLabel, sourceId)) } + protected final case class NodeNodeLabelRow(nodeId: String, nodeLabelLabel: String) + protected final class NodeNodeLabelTable(tag: Tag) extends Table[NodeNodeLabelRow](tag, "node_x_node_label") { + def nodeId = column[String]("node_id") + def nodeLabelLabel = column[String]("label") + + def * = (nodeId, nodeLabelLabel) <> (NodeNodeLabelRow.tupled, NodeNodeLabelRow.unapply) + + def node = foreignKey("node_fk", nodeId, nodes)(_.id) + def nodeLabel = foreignKey("node_label_fk", nodeLabelLabel, nodeLabels)(_.label) + + def pk = primaryKey("node_label_pk", (nodeId, nodeLabelLabel)) + } + + protected final case class NodeSourceRow(nodeId: String, sourceId: String) + protected final class NodeSourceTable(tag: Tag) extends Table[NodeSourceRow](tag, "node_x_source") { + def nodeId = column[String]("node_id") + def sourceId = column[String]("source_id") + + def * = (nodeId, sourceId) <> (NodeSourceRow.tupled, NodeSourceRow.unapply) + + def node = foreignKey("node_fk", nodeId, nodes)(_.id) + def source = foreignKey("source_fk", sourceId, sources)(_.id) + + def pk = primaryKey("node_source_pk", (nodeId, sourceId)) + } + + protected final case class SourceRow(id: String, label: String) { + def toKgSource = KgSource(id = id, label = label) + } protected final class SourceTable(tag: Tag) extends Table[SourceRow](tag, "source") { def id = column[String]("id", O.PrimaryKey) def label = column[String]("label") From a713f39cbbc213556b898ebbeaee99c9b627212e Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 25 Nov 2020 01:30:45 -0500 Subject: [PATCH 44/51] use composite primary key for node_label_edge --- .../postgres/AbstractPostgresKgStore.scala | 18 ++++++++++-------- .../postgres/PostgresKgCommandStore.scala | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 73a9b941..496b7ac1 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -156,31 +156,33 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos def * = (label, pageRank) <> (NodeLabelRow.tupled, NodeLabelRow.unapply) } - protected final case class NodeLabelEdgeRow(pageRank: Option[Int], objectNodeLabelLabel: String, subjectNodeLabelLabel: String) + protected final case class NodeLabelEdgeRow(objectNodeLabelLabel: String, subjectNodeLabelLabel: String) protected final class NodeLabelEdgeTable(tag: Tag) extends Table[NodeLabelEdgeRow](tag, "node_label_edge") { - def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def objectNodeLabelLabel = column[String]("object_node_label_label") def subjectNodeLabelLabel = column[String]("subject_node_label_label") - def * = (id.?, objectNodeLabelLabel, subjectNodeLabelLabel) <> (NodeLabelEdgeRow.tupled, NodeLabelEdgeRow.unapply) + def * = (objectNodeLabelLabel, subjectNodeLabelLabel) <> (NodeLabelEdgeRow.tupled, NodeLabelEdgeRow.unapply) def objectNodeLabel = foreignKey("object_node_label_fk", objectNodeLabelLabel, nodeLabels)(_.label) def subjectNodeLabel = foreignKey("subject_node_label_fk", subjectNodeLabelLabel, nodeLabels)(_.label) + def pk = primaryKey("node_label_edge_pk", (objectNodeLabelLabel, subjectNodeLabelLabel)) + def unique_constraint = index("node_label_edge_unique_idx", (objectNodeLabelLabel, subjectNodeLabelLabel), unique = true) } - protected final case class NodeLabelEdgeSourceRow(nodeLabelEdgeId: Int, sourceId: String) + protected final case class NodeLabelEdgeSourceRow(nodeLabelEdgeObjectNodeLabelLabel: String, nodeLabelEdgeSubjectNodeLabelLabel: String, sourceId: String) protected final class NodeLabelEdgeSourceTable(tag: Tag) extends Table[NodeLabelEdgeSourceRow](tag, "node_label_edge_x_source") { - def nodeLabelEdgeId = column[Int]("node_label_edge_id") + def nodeLabelEdgeObjectNodeLabelLabel = column[String]("node_node_label_edge_object_node_label_label") + def nodeLabelEdgeSubjectNodeLabelLabel = column[String]("subject_node_label_edge_subject_node_label_label") def sourceId = column[String]("source_id") - def * = (nodeLabelEdgeId, sourceId) <> (NodeLabelEdgeSourceRow.tupled, NodeLabelEdgeSourceRow.unapply) + def * = (nodeLabelEdgeObjectNodeLabelLabel, nodeLabelEdgeSubjectNodeLabelLabel, sourceId) <> (NodeLabelEdgeSourceRow.tupled, NodeLabelEdgeSourceRow.unapply) - def nodeLabelEdge = foreignKey("node_label_edge_fk", nodeLabelEdgeId, nodeLabelEdges)(_.id) + def nodeLabelEdge = foreignKey("node_label_edge_fk", (nodeLabelEdgeObjectNodeLabelLabel, nodeLabelEdgeSubjectNodeLabelLabel), nodeLabelEdges)(nodeLabelEdgeTable => (nodeLabelEdgeTable.objectNodeLabelLabel, nodeLabelEdgeTable.subjectNodeLabelLabel)) def source = foreignKey("source_fk", sourceId, sources)(_.id) - def pk = primaryKey("node_label_edge_source_pk", (nodeLabelEdgeId, sourceId)) + def pk = primaryKey("node_label_edge_source_pk", (nodeLabelEdgeObjectNodeLabelLabel, nodeLabelEdgeSubjectNodeLabelLabel, sourceId)) } protected final case class NodeLabelSourceRow(nodeLabelLabel: String, sourceId: String) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 8533c228..19ace30e 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -121,7 +121,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid for { nodeLabelEdgePairs <- nodeLabelEdgePairsAction _ <- nodeLabelEdges ++= nodeLabelEdgePairs.distinct.map { case (objectNodeLabelLabel, subjectNodeLabelLabel) => - NodeLabelEdgeRow(Some(0), objectNodeLabelLabel, subjectNodeLabelLabel) + NodeLabelEdgeRow(objectNodeLabelLabel, subjectNodeLabelLabel) } } yield () } @@ -132,21 +132,21 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid objectNodeLabel <- nodeLabelEdge.objectNodeLabel objectNodeLabelSourceSource <- nodeLabelSources if objectNodeLabelSourceSource.nodeLabelLabel === objectNodeLabel.label objectNodeLabelSource <- objectNodeLabelSourceSource.source - } yield (nodeLabelEdge.id, objectNodeLabelSource.id) + } yield (nodeLabelEdge.objectNodeLabelLabel, nodeLabelEdge.subjectNodeLabelLabel, objectNodeLabelSource.id) val subjectNodeLabelEdgeSourcesQuery = for { nodeLabelEdge <- nodeLabelEdges subjectNodeLabel <- nodeLabelEdge.subjectNodeLabel subjectNodeLabelSourceSource <- nodeLabelSources if subjectNodeLabelSourceSource.nodeLabelLabel === subjectNodeLabel.label subjectNodeLabelSource <- subjectNodeLabelSourceSource.source - } yield (nodeLabelEdge.id, subjectNodeLabelSource.id) + } yield (nodeLabelEdge.objectNodeLabelLabel, nodeLabelEdge.subjectNodeLabelLabel, subjectNodeLabelSource.id) val nodeLabelEdgeSourcesAction = (objectNodeLabelEdgeSourcesQuery ++ subjectNodeLabelEdgeSourcesQuery).result for { nodeLabelEdgeSourcesResult <- nodeLabelEdgeSourcesAction _ <- nodeLabelEdgeSources.insertOrUpdateAll(nodeLabelEdgeSourcesResult.map { - case (edgeId, sourceId) => NodeLabelEdgeSourceRow(edgeId, sourceId) + case (edgeObjectLabel, edgeSubjectLabel, sourceId) => NodeLabelEdgeSourceRow(edgeObjectLabel, edgeSubjectLabel, sourceId) }) } yield () } From 687eb9866e6bab99eef61ef663c0e18a36e88054 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 25 Nov 2020 17:15:21 -0500 Subject: [PATCH 45/51] add getNodeContext with id sorting --- .../postgres/AbstractPostgresKgStore.scala | 46 +++--- .../postgres/ExtendedPostgresProfile.scala | 6 +- .../postgres/PostgresKgCommandStore.scala | 2 +- .../postgres/PostgresKgQueryStore.scala | 144 +++++++++++++----- 4 files changed, 141 insertions(+), 57 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala index 496b7ac1..be5c5815 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/AbstractPostgresKgStore.scala @@ -15,34 +15,44 @@ abstract class AbstractPostgresKgStore(protected val databaseConfigProvider: Pos protected val SentencesDelimChar = '|' protected val SentencesDelimString: String = SentencesDelimChar.toString + protected val NodeContextTopEdgesLimit = 10 + protected val NodeLabelContextTopEdgesLimit = 10 + protected lazy val edges = TableQuery[EdgeTable] protected lazy val edgeLabels = TableQuery[EdgeLabelTable] protected lazy val edgeSources = TableQuery[EdgeSourceTable] protected object nodes extends TableQuery(new NodeTable(_)) { - def withNodeLabels = for { - ((node, _), nodeLabel) <- nodes - .join(nodeNodeLabels).on(_.id === _.nodeId) - .join(nodeLabels).on(_._2.nodeLabelLabel === _.label) - } yield (node, nodeLabel) - - def withSources = for { - ((node, _), source) <- nodes - .join(nodeSources).on(_.id === _.nodeId) - .join(sources).on(_._2.sourceId === _.id) - } yield (node, source) + def getById(id: String) = + nodes.filter(_.id === id).result.headOption + + def withLabelSource(nodeQuery: Query[NodeTable, NodeRow, Seq] = nodes) = + for { + node <- nodeQuery + nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeId === node.id + nodeLabel <- nodeNodeLabel.nodeLabel + nodeSource <- nodeSources if nodeSource.nodeId === node.id + source <- nodeSource.source + } yield (node, nodeLabel, source) + } protected object nodeLabels extends TableQuery(new NodeLabelTable(_)) { - def withNodes = for { - ((nodeLabel, _), node) <- nodeLabels + def withNodes(nodeLabelQuery: Query[NodeLabelTable, NodeLabelRow, Seq] = nodeLabels) = for { + ((nodeLabel, _), node) <- nodeLabelQuery .join(nodeNodeLabels).on(_.label === _.nodeLabelLabel) .join(nodes).on(_._2.nodeId === _.id) } yield (nodeLabel, node) - def withSources = for { - ((nodeLabel, _), source) <- nodeLabels - .join(nodeLabelSources).on(_.label === _.nodeLabelLabel) - .join(sources).on(_._2.sourceId === _.id) - } yield (nodeLabel, source) + def withSourceNode(nodeLabelQuery: Query[NodeLabelTable, NodeLabelRow, Seq] = nodeLabels) = + for { + nodeLabel <- nodeLabelQuery + nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeLabelLabel === nodeLabel.label + nodeLabelSource <- nodeLabelSources if nodeLabelSource.nodeLabelLabel === nodeLabel.label + node <- nodeNodeLabel.node + source <- nodeLabelSource.source + nodeSource <- nodeSources if nodeSource.nodeId === node.id + nodeNodeLabel <- nodeNodeLabel.nodeLabel + nodeNodeSource <- nodeSource.source + } yield (nodeLabel, source, node, nodeNodeSource, nodeNodeLabel) } protected lazy val nodeLabelEdges = TableQuery[NodeLabelEdgeTable] protected lazy val nodeLabelEdgeSources = TableQuery[NodeLabelEdgeSourceTable] diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala index f6c22ab3..ca52f4f7 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/ExtendedPostgresProfile.scala @@ -4,13 +4,15 @@ import slick.basic.Capability import slick.jdbc.{JdbcCapabilities} import com.github.tminglei.slickpg._ -trait ExtendedPostgresProfile extends ExPostgresProfile { +trait ExtendedPostgresProfile extends ExPostgresProfile with PgArraySupport { override protected def computeCapabilities: Set[Capability] = super.computeCapabilities + JdbcCapabilities.insertOrUpdate override val api = ExtendedAPI - object ExtendedAPI extends API + object ExtendedAPI extends API with ArrayImplicits { + implicit val strListTypeMapper = new SimpleArrayJdbcType[String]("text").to(_.toList) + } } object ExtendedPostgresProfile extends ExtendedPostgresProfile \ No newline at end of file diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 19ace30e..182a1c8f 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -111,7 +111,7 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid private def writeNodeLabelEdgesAction = { val nodeLabelEdgePairsAction = (for { - (subjectNodeLabel, node) <- nodeLabels.withNodes + (subjectNodeLabel, node) <- nodeLabels.withNodes() edge <- edges if edge.subjectNodeId === node.id objectNode <- edge.objectNode objectNodeNodeLabel <- nodeNodeLabels if objectNodeNodeLabel.nodeId === objectNode.id && objectNodeNodeLabel.nodeLabelLabel =!= subjectNodeLabel.label diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 963209ba..311a8e90 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -1,12 +1,14 @@ package io.github.tetherlessworld.mcsapps.lib.kg.stores.postgres import com.google.inject.Inject +import io.github.tetherlessworld.mcsapps.lib.kg.models.edge.KgEdge import io.github.tetherlessworld.mcsapps.lib.kg.models.node.{KgNode, KgNodeContext, KgNodeLabel, KgNodeLabelContext} import io.github.tetherlessworld.mcsapps.lib.kg.models.path.KgPath import io.github.tetherlessworld.mcsapps.lib.kg.models.search.{KgSearchFacets, KgSearchQuery, KgSearchResult, KgSearchSort} import io.github.tetherlessworld.mcsapps.lib.kg.models.source.KgSource import io.github.tetherlessworld.mcsapps.lib.kg.stores.KgQueryStore import javax.inject.Singleton +import slick.jdbc.GetResult import scala.concurrent.ExecutionContext @@ -14,16 +16,49 @@ import scala.concurrent.ExecutionContext final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgQueryStore { import profile.api._ + private implicit val getStringList = GetResult[List[String]] (r => + r.rs.getArray(r.skip.currentPos) + .getArray + .asInstanceOf[Array[Any]] + .toList + .map(_.toString()) + ) + + private implicit val getKgEdge = GetResult(r => KgEdge( + id = r.<<[String], + labels = r.<<[List[String]], + `object` = r.<<[String], + predicate = r.<<[String], + sentences = (r.<<[String]).split(SentencesDelimChar).toList, + sourceIds = r.<<[List[String]], + subject = r.<<[String] + )) + + private def toKgNodeLabels(rows: Seq[(NodeLabelRow, String, NodeRow, String, String)]) = + rows + .groupBy(_._1.label) + .values + .map { rows => + rows.head._1.toKgNodeLabel( + sourceIds = rows.map(_._2).distinct.toList, + nodes = rows.groupBy(_._3.id).values.map { + nodeRows => + nodeRows.head._3.toKgNode( + labels = nodeRows.map(_._4).distinct.toList, + sourceIds = nodeRows.map(_._5).distinct.toList + ) + }.toList + ) + } + override def getNode(id: String): Option[KgNode] = { - val query = (for { - ((node, source), (_, nodeLabel)) <- nodes - .withSources - .join(nodes.withNodeLabels) - .on(_._1.id === _._1.id) - .filter(_._1._1.id === id) - } yield (node, nodeLabel.label, source.id)).result - - runSyncTransaction(query) + val nodeQuery = nodes.filter(_.id === id) + + val nodeAction = nodes.withLabelSource(nodeQuery).map { + case (node, nodeLabel, source) => (node, nodeLabel.label, source.id) + }.result + + runSyncTransaction(nodeAction) .groupBy(_._1.id) .values .map { rows => @@ -35,35 +70,72 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr .headOption } - override def getNodeContext(id: String): Option[KgNodeContext] = None + override def getNodeContext(id: String): Option[KgNodeContext] = + runSyncTransaction(nodes.getById(id)).map { _ => + val nodeLabelQuery = for { + nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeId === id + nodeLabel <- nodeNodeLabel.nodeLabel + } yield (nodeLabel) + + val relatedObjectNodeLabelQuery = (for { + nodeLabel <- nodeLabelQuery + relatedObjectNodeLabelEdge <- nodeLabelEdges if relatedObjectNodeLabelEdge.subjectNodeLabelLabel === nodeLabel.label + relatedObjectNodeLabel <- relatedObjectNodeLabelEdge.objectNodeLabel + } yield (relatedObjectNodeLabel)) + + val relatedSubjectNodeLabelQuery = (for { + nodeLabel <- nodeLabelQuery + relatedSubjectNodeLabelEdge <- nodeLabelEdges if relatedSubjectNodeLabelEdge.objectNodeLabelLabel === nodeLabel.label + relatedSubjectNodeLabel <- relatedSubjectNodeLabelEdge.subjectNodeLabel + } yield (relatedSubjectNodeLabel)) + + val relatedNodeLabelQuery = (relatedObjectNodeLabelQuery ++ relatedSubjectNodeLabelQuery) + val relatedNodeLabelWithNodeSourceAction = nodeLabels.withSourceNode(relatedNodeLabelQuery).map { + case (nodeLabel, source, nodeLabelNode, nodeLabelNodeSource, nodeLabelNodeLabel) => + (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeSource.id, nodeLabelNodeLabel.label) + }.result + + val relatedNodeLabels = toKgNodeLabels(runSyncTransaction(relatedNodeLabelWithNodeSourceAction)).toList + + // TODO replace inner id order by with pageRank + val topEdgesQuery = + sql""" + SELECT + e_top.id, + array_agg(DISTINCT el.label), + e_top.object_node_id, + e_outer.predicate, + e_top.sentences, + array_agg(DISTINCT s.id), + e_top.subject_node_id + FROM edge e_outer + JOIN LATERAL ( + SELECT * FROM edge e_inner + WHERE e_inner.subject_node_id = ${id} AND e_inner.predicate = e_outer.predicate + ORDER BY e_inner.id + LIMIT #$NodeContextTopEdgesLimit + ) e_top ON e_outer.subject_node_id = ${id} + JOIN edge_x_source es ON es.edge_id = e_top.id + JOIN source s ON s.id = es.source_id + JOIN edge_label el ON el.edge_id = e_top.id + GROUP BY e_outer.predicate, e_top.id, e_top.object_node_id, e_top.sentences, e_top.subject_node_id + ORDER BY e_outer.predicate + """.as[KgEdge] + + val topEdges = runSyncTransaction(topEdgesQuery).toList + + KgNodeContext( + topEdges = topEdges, + relatedNodeLabels = relatedNodeLabels + ) + } override def getNodeLabel(label: String): Option[KgNodeLabel] = { - val query = (for { - nodeLabel <- nodeLabels if nodeLabel.label === label - nodeNodeLabel <- nodeNodeLabels if nodeNodeLabel.nodeLabelLabel === nodeLabel.label - nodeLabelSource <- nodeLabelSources if nodeLabelSource.nodeLabelLabel === nodeLabel.label - node <- nodeNodeLabel.node - source <- nodeLabelSource.source - nodeSource <- nodeSources if nodeSource.nodeId === node.id - nodeNodeLabel <- nodeNodeLabel.nodeLabel - nodeNodeSource <- nodeSource.source - } yield (nodeLabel, source.id, node, nodeNodeSource.id, nodeNodeLabel.label)).result - - runSyncTransaction(query) - .groupBy(_._1.label) - .values - .map { - rows => rows.head._1.toKgNodeLabel( - sourceIds = rows.map(_._2).distinct.toList, - nodes = rows.groupBy(_._3.id).values.map { - nodeRows => nodeRows.head._3.toKgNode( - labels = nodeRows.map(_._4).distinct.toList, - sourceIds = nodeRows.map(_._5).distinct.toList - ) - }.toList - ) - } - .headOption + val nodeLabelQuery = nodeLabels.filter(_.label === label) + + val nodeLabelAction = nodeLabels.withNodeSource(nodeLabelQuery).result + + toKgNodeLabels(runSyncTransaction(nodeLabelAction)).headOption } override def getNodeLabelContext(label: String): Option[KgNodeLabelContext] = None From 7ec32fd13a9973c84dbaaa732dde0a457576984a Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 25 Nov 2020 17:26:41 -0500 Subject: [PATCH 46/51] remove random node and path --- .../lib/kg/stores/postgres/PostgresKgCommandStore.scala | 2 -- .../lib/kg/stores/postgres/PostgresKgQueryStore.scala | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala index 182a1c8f..4133ac8b 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgCommandStore.scala @@ -104,8 +104,6 @@ class PostgresKgCommandStore @Inject()(configProvider: PostgresStoreConfigProvid override final def putNodes(nodes: Iterator[KgNode]): Unit = runSyncTransaction(DBIO.sequence(batchedNodeInserts(nodes))) - override final def putPaths(paths: Iterator[KgPath]): Unit = Unit - override final def putSources(sources: Iterator[KgSource]): Unit = runSyncTransaction(DBIO.sequence(batchedSourceInserts(sources))) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 311a8e90..80c1e280 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -139,11 +139,7 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr } override def getNodeLabelContext(label: String): Option[KgNodeLabelContext] = None - - override def getPath(id: String): Option[KgPath] = None - - override def getRandomNode: KgNode = KgNode("", None, List(), None, None, None, List(), None) - + override def getSourcesById: Map[String, KgSource] = { runSyncTransaction(sources.result).map(source => (source.id, source.toKgSource)).toMap } From 9fd706fd731d0f8f8ed6231ecca3f17ab8aa97ea Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 25 Nov 2020 17:31:32 -0500 Subject: [PATCH 47/51] fix getNodeLabel --- .../lib/kg/stores/postgres/PostgresKgQueryStore.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 80c1e280..1225128f 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -133,13 +133,16 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr override def getNodeLabel(label: String): Option[KgNodeLabel] = { val nodeLabelQuery = nodeLabels.filter(_.label === label) - val nodeLabelAction = nodeLabels.withNodeSource(nodeLabelQuery).result + val nodeLabelAction = nodeLabels.withSourceNode(nodeLabelQuery).map { + case (nodeLabel, source, nodeLabelNode, nodeLabelNodeSource, nodeLabelNodeLabel) => + (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeSource.id, nodeLabelNodeLabel.label) + }.result toKgNodeLabels(runSyncTransaction(nodeLabelAction)).headOption } override def getNodeLabelContext(label: String): Option[KgNodeLabelContext] = None - + override def getSourcesById: Map[String, KgSource] = { runSyncTransaction(sources.result).map(source => (source.id, source.toKgSource)).toMap } From 1853a98fba34e70168b627a7dbc6778ff42ad1e4 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Wed, 25 Nov 2020 17:46:38 -0500 Subject: [PATCH 48/51] add postgres to circleci --- .circleci/config.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ee36dcc..d4c3d17b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,12 +74,12 @@ jobs: - run: name: Build command: | - docker-compose build kg-app neo4j + docker-compose build kg-app neo4j postgres - run: name: Run background: true command: | - docker-compose up --abort-on-container-exit kg-app neo4j + docker-compose up --abort-on-container-exit kg-app neo4j postgres - run: name: Wait for the server to start command: | @@ -103,6 +103,15 @@ jobs: environment: NEO4J_AUTH: neo4j/nC1aB4mji623s2Zs NEO4JLABS_PLUGINS: '["apoc","graph-data-science"]' + - image: library/postgres:12.4 + auth: + username: $DOCKER_ID + password: $DOCKER_PASSWORD + name: mcs-postgres + environment: + POSTGRES_USER: "mcs" + POSTGRES_PASSWORD: "7EAdu7jJvZNxxrNZ" + POSTGRES_DB: "kg" steps: - attach_workspace: at: /tmp/workspace From 9cc6881ae3b5c61d828c375af73c832afb41f669 Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 27 Nov 2020 01:17:25 -0500 Subject: [PATCH 49/51] fix ordering kg node label query result --- .../mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 1225128f..f3344cb7 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -92,7 +92,7 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr val relatedNodeLabelQuery = (relatedObjectNodeLabelQuery ++ relatedSubjectNodeLabelQuery) val relatedNodeLabelWithNodeSourceAction = nodeLabels.withSourceNode(relatedNodeLabelQuery).map { case (nodeLabel, source, nodeLabelNode, nodeLabelNodeSource, nodeLabelNodeLabel) => - (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeSource.id, nodeLabelNodeLabel.label) + (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeLabel.label, nodeLabelNodeSource.id) }.result val relatedNodeLabels = toKgNodeLabels(runSyncTransaction(relatedNodeLabelWithNodeSourceAction)).toList @@ -135,7 +135,7 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr val nodeLabelAction = nodeLabels.withSourceNode(nodeLabelQuery).map { case (nodeLabel, source, nodeLabelNode, nodeLabelNodeSource, nodeLabelNodeLabel) => - (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeSource.id, nodeLabelNodeLabel.label) + (nodeLabel, source.id, nodeLabelNode, nodeLabelNodeLabel.label, nodeLabelNodeSource.id) }.result toKgNodeLabels(runSyncTransaction(nodeLabelAction)).headOption From 0cc8a01c306fa02551f310c68db95564593c034d Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 27 Nov 2020 01:47:45 -0500 Subject: [PATCH 50/51] use column names instead of index --- .../postgres/PostgresKgQueryStore.scala | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index f3344cb7..2797589d 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -16,22 +16,14 @@ import scala.concurrent.ExecutionContext final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigProvider)(implicit executionContext: ExecutionContext) extends AbstractPostgresKgStore(configProvider) with KgQueryStore { import profile.api._ - private implicit val getStringList = GetResult[List[String]] (r => - r.rs.getArray(r.skip.currentPos) - .getArray - .asInstanceOf[Array[Any]] - .toList - .map(_.toString()) - ) - private implicit val getKgEdge = GetResult(r => KgEdge( - id = r.<<[String], - labels = r.<<[List[String]], - `object` = r.<<[String], - predicate = r.<<[String], - sentences = (r.<<[String]).split(SentencesDelimChar).toList, - sourceIds = r.<<[List[String]], - subject = r.<<[String] + id = r.rs.getString("id"), + labels = r.rs.getArray("labels").asInstanceOf[Array[Any]].toList.map(_.toString), + `object` = r.rs.getArray("object"), + predicate = r.rs.getString("predicate"), + sentences = (r.rs.getString("sentences")).split(SentencesDelimChar).toList, + sourceIds = r.rs.getArray("sourceIds").asInstanceOf[Array[Any]].toList.map(_.toString), + subject = r.rs.getString("subject") )) private def toKgNodeLabels(rows: Seq[(NodeLabelRow, String, NodeRow, String, String)]) = @@ -101,13 +93,13 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr val topEdgesQuery = sql""" SELECT - e_top.id, - array_agg(DISTINCT el.label), - e_top.object_node_id, - e_outer.predicate, - e_top.sentences, - array_agg(DISTINCT s.id), - e_top.subject_node_id + e_top.id AS id, + array_agg(DISTINCT el.label) AS labels, + e_top.object_node_id AS object, + e_outer.predicate AS predicate, + e_top.sentences AS sentences, + array_agg(DISTINCT s.id) AS sourceIds, + e_top.subject_node_id AS subject FROM edge e_outer JOIN LATERAL ( SELECT * FROM edge e_inner From fef1963f2040385527e50709eea5bc574fd922dd Mon Sep 17 00:00:00 2001 From: 123joshuawu Date: Fri, 27 Nov 2020 01:52:50 -0500 Subject: [PATCH 51/51] typo on column type --- .../mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala index 2797589d..40314fc4 100644 --- a/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala +++ b/lib/scala/kg/src/main/scala/io/github/tetherlessworld/mcsapps/lib/kg/stores/postgres/PostgresKgQueryStore.scala @@ -19,7 +19,7 @@ final class PostgresKgQueryStore @Inject()(configProvider: PostgresStoreConfigPr private implicit val getKgEdge = GetResult(r => KgEdge( id = r.rs.getString("id"), labels = r.rs.getArray("labels").asInstanceOf[Array[Any]].toList.map(_.toString), - `object` = r.rs.getArray("object"), + `object` = r.rs.getString("object"), predicate = r.rs.getString("predicate"), sentences = (r.rs.getString("sentences")).split(SentencesDelimChar).toList, sourceIds = r.rs.getArray("sourceIds").asInstanceOf[Array[Any]].toList.map(_.toString),