From e4a17b8c919199a9c7dcf0619aee120f9c5cca11 Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 22 Aug 2022 16:07:00 +0200 Subject: [PATCH 01/18] Refactoring DAO. WIP --- .gitignore | 8 + build.sbt | 1 + docker-compose.yml | 6 + .../ing/wbaa/rokku/sts/StsServiceItTest.scala | 8 +- ...{MariaDbItTest.scala => RedisItTest.scala} | 15 +- .../service/db/dao/STSTokenDAOItTest.scala | 106 ++++---- .../db/dao/STSUserAndGroupDAOItTest.scala | 60 +++-- src/main/resources/application.conf | 7 + src/main/resources/reference.conf | 7 + .../scala/com/ing/wbaa/rokku/sts/Server.scala | 14 +- .../wbaa/rokku/sts/config/RedisSettings.scala | 17 ++ .../sts/service/ExpiredTokenCleaner.scala | 29 --- .../ing/wbaa/rokku/sts/service/db/Redis.scala | 114 +++++++++ .../sts/service/db/dao/STSTokenDAO.scala | 166 +++++-------- .../service/db/dao/STSUserAndGroupDAO.scala | 235 ++++++++---------- 15 files changed, 450 insertions(+), 343 deletions(-) rename src/it/scala/com/ing/wbaa/rokku/sts/service/db/{MariaDbItTest.scala => RedisItTest.scala} (53%) create mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/config/RedisSettings.scala delete mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/service/ExpiredTokenCleaner.scala create mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala diff --git a/.gitignore b/.gitignore index 7eea5fd..cba27d1 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,11 @@ snapshots # Log files *.log /bin/ + +.bloop/ +.bsp/ +.metals/ +.scalafmt.conf +project/.bloop/ +project/metals.sbt + diff --git a/build.sbt b/build.sbt index 1ce6fbf..7537812 100644 --- a/build.sbt +++ b/build.sbt @@ -48,6 +48,7 @@ libraryDependencies ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.3", "com.auth0" % "java-jwt" % "4.0.0", "com.bettercloud" % "vault-java-driver" % "5.1.0", + "redis.clients" % "jedis" % "4.3.0-m1", "org.scalatest" %% "scalatest" % "3.2.13" % "test, it", "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % Test, "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test, diff --git a/docker-compose.yml b/docker-compose.yml index 536714e..9a9fa93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,12 @@ services: ports: - 3307:3306 + redis: + image: redislabs/redisearch + command: "redis-server --requirepass password --loadmodule '/usr/lib/redis/modules/redisearch.so'" + ports: + - 6379:6379 + vault: image: vault:1.4.2 environment: diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala index a0e9978..9cc1778 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala @@ -6,13 +6,13 @@ import akka.actor.ActorSystem import akka.http.scaladsl.model.Uri.{Authority, Host} import com.amazonaws.services.securitytoken.AWSSecurityTokenService import com.amazonaws.services.securitytoken.model.{AWSSecurityTokenServiceException, AssumeRoleRequest, GetSessionTokenRequest} -import com.ing.wbaa.rokku.sts.config.{HttpSettings, KeycloakSettings, MariaDBSettings, StsSettings, VaultSettings} +import com.ing.wbaa.rokku.sts.config.{HttpSettings, KeycloakSettings, RedisSettings, StsSettings, VaultSettings} import com.ing.wbaa.rokku.sts.data.aws._ import com.ing.wbaa.rokku.sts.data.{UserAssumeRole, UserName} import com.ing.wbaa.rokku.sts.helper.{KeycloackToken, OAuth2TokenRequest} import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } import com.ing.wbaa.rokku.sts.service.UserTokenDbService -import com.ing.wbaa.rokku.sts.service.db.MariaDb +import com.ing.wbaa.rokku.sts.service.db.Redis import com.ing.wbaa.rokku.sts.service.db.dao.STSUserAndGroupDAO import com.ing.wbaa.rokku.sts.vault.VaultService import org.scalatest.Assertion @@ -55,7 +55,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO - with MariaDb + with Redis with VaultService with KeycloakClient { override implicit def system: ActorSystem = testSystem @@ -66,7 +66,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams override protected[this] def stsSettings: StsSettings = StsSettings(testSystem) - override protected[this] def mariaDBSettings: MariaDBSettings = new MariaDBSettings(testSystem.settings.config) + override protected[this] def redisSettings: RedisSettings = new RedisSettings(testSystem.settings.config) override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/MariaDbItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala similarity index 53% rename from src/it/scala/com/ing/wbaa/rokku/sts/service/db/MariaDbItTest.scala rename to src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala index dfcc4ab..595009e 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/MariaDbItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala @@ -1,21 +1,21 @@ package com.ing.wbaa.rokku.sts.service.db import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{MariaDBSettings, StsSettings} +import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} import org.scalatest.wordspec.AsyncWordSpec import scala.util.{Failure, Success} -class MariaDbItTest extends AsyncWordSpec with MariaDb { +class RedisItTest extends AsyncWordSpec with Redis { val system: ActorSystem = ActorSystem.create("test-system") - protected[this] def mariaDBSettings: MariaDBSettings = MariaDBSettings(system) + protected[this] def redisSettings: RedisSettings = RedisSettings(system) protected[this] def stsSettings: StsSettings = StsSettings(system) override lazy val dbExecutionContext = executionContext - "MariaDB" should { + "Redis" should { "be reachable" in { checkDbConnection() transform { @@ -24,6 +24,13 @@ class MariaDbItTest extends AsyncWordSpec with MariaDb { } } + "create index upon forceInitRedisConnectionPool call" in { + forceInitRedisConnectionPool() + val info = redisConnectionPool.ftInfo("users-index") + println(info) + assert(info.containsValue("users-index")) + } + } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala index 34f7b53..1616276 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala @@ -1,33 +1,53 @@ package com.ing.wbaa.rokku.sts.service.db.dao import java.time.Instant -import java.time.temporal.ChronoUnit import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{MariaDBSettings, StsSettings} +import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} import com.ing.wbaa.rokku.sts.data.{UserAssumeRole, UserName} import com.ing.wbaa.rokku.sts.data.aws.{AwsCredential, AwsSessionToken, AwsSessionTokenExpiration} import com.ing.wbaa.rokku.sts.service.TokenGeneration -import com.ing.wbaa.rokku.sts.service.db.MariaDb +import com.ing.wbaa.rokku.sts.service.db.Redis import org.scalatest.Assertion import org.scalatest.wordspec.AsyncWordSpec - +import org.scalatest.{BeforeAndAfterAll} import scala.concurrent.Future import scala.util.Random +import scala.jdk.CollectionConverters._ + -class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGroupDAO with MariaDb with TokenGeneration { +class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGroupDAO with Redis with TokenGeneration with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") - override protected[this] def mariaDBSettings: MariaDBSettings = MariaDBSettings(system) + override protected[this] def redisSettings: RedisSettings = RedisSettings(system) override protected[this] def stsSettings: StsSettings = StsSettings(system) override lazy val dbExecutionContext = executionContext + override protected def beforeAll(): Unit = { + // forceInitRedisConnectionPool() + List("users:", "sessionTokens:").foreach(pattern => { + val keys = redisConnectionPool.keys(pattern) + keys.asScala.foreach(key => { + println(s"WTFFFF $key") + redisConnectionPool.del(key)}) + }) + } + override protected def afterAll(): Unit = { + List("users:", "sessionTokens:").foreach(pattern => { + val keys = redisConnectionPool.keys(pattern) + keys.asScala.foreach(key => { + println(key) + redisConnectionPool.del(key)}) + }) + } + + private class TestObject { val testAwsSessionToken: AwsSessionToken = AwsSessionToken(Random.alphanumeric.take(32).mkString) - val userName: UserName = UserName(Random.alphanumeric.take(32).mkString) + val username: UserName = UserName(Random.alphanumeric.take(32).mkString) val testExpirationDate: AwsSessionTokenExpiration = AwsSessionTokenExpiration(Instant.now()) val cred: AwsCredential = generateAwsCredential val testAwsSessionTokenValid1 = AwsSessionToken(Random.alphanumeric.take(32).mkString) @@ -37,26 +57,26 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr private def withInsertedUser(testCode: UserName => Future[Assertion]): Future[Assertion] = { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) - testCode(testObject.userName) + insertAwsCredentials(testObject.username, testObject.cred, isNPA = false) + testCode(testObject.username) } "STS Token DAO" should { "get Token" that { - "exists" in withInsertedUser { userName => + "exists" in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, userName, testObject.testExpirationDate) - getToken(testObject.testAwsSessionToken, userName).map { o => + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) + getToken(testObject.testAwsSessionToken, username).map { o => assert(o.isDefined) - assert(o.get._1 == userName) + assert(o.get._1 == username) //is off by milliseconds, because we truncate it, so we match be epoch seconds assert(o.get._3.value.getEpochSecond == testObject.testExpirationDate.value.getEpochSecond) } } - "doesn't exist" in withInsertedUser { userName => - getToken(AwsSessionToken("DOESNTEXIST"), userName).map { o => + "doesn't exist" in withInsertedUser { username => + getToken(AwsSessionToken("DOESNTEXIST"), username).map { o => assert(o.isEmpty) } } @@ -64,56 +84,56 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr } "insert Token" that { - "new to the db" in withInsertedUser { userName => + "new to the db" in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, userName, testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) .map(r => assert(r)) } - "token with same session token already exists " in withInsertedUser { userName => + "token with same session token already exists " in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, userName, testObject.testExpirationDate) - insertToken(testObject.testAwsSessionToken, userName, testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) .map(r => assert(!r)) } } "insert Token for a role" that { - "new to the db" in withInsertedUser { userName => + "new to the db" in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, userName, testObject.assumeRole , testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole , testObject.testExpirationDate) .map(r => assert(r)) - getToken(testObject.testAwsSessionToken, userName).map { o => + getToken(testObject.testAwsSessionToken, username).map { o => assert(o.isDefined) - assert(o.get._1 == userName) + assert(o.get._1 == username) assert(o.get._2 == testObject.assumeRole) assert(o.get._3.value.getEpochSecond == testObject.testExpirationDate.value.getEpochSecond) } } - "token with same session token already exists " in withInsertedUser { userName => + "token with same session token already exists " in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, userName, testObject.assumeRole, testObject.testExpirationDate) - insertToken(testObject.testAwsSessionToken, userName, testObject.assumeRole, testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) .map(r => assert(!r)) } } - "clean expired tokens" in { - val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) - insertToken(testObject.testAwsSessionTokenValid1, testObject.userName, AwsSessionTokenExpiration(Instant.now())) - insertToken(testObject.testAwsSessionToken, testObject.userName, AwsSessionTokenExpiration(Instant.now().minus(2, ChronoUnit.DAYS))) - insertToken(testObject.testAwsSessionTokenValid2, testObject.userName, AwsSessionTokenExpiration(Instant.now())) - for { - notOldTokenYet <- getToken(testObject.testAwsSessionToken, testObject.userName).map(_.isDefined) - notArchTokenYet <- getToken(testObject.testAwsSessionToken, testObject.userName, TOKENS_ARCH_TABLE).map(_.isEmpty) - cleanedTokens <- cleanExpiredTokens(AwsSessionTokenExpiration(Instant.now().minus(1, ChronoUnit.DAYS))).map(_ == 1) - tokenOneValid <- getToken(testObject.testAwsSessionTokenValid1, testObject.userName).map(_.isDefined) - oldTokenGone <- getToken(testObject.testAwsSessionToken, testObject.userName).map(_.isEmpty) - tokenTwoValid <- getToken(testObject.testAwsSessionTokenValid2, testObject.userName).map(_.isDefined) - archToken <- getToken(testObject.testAwsSessionToken, testObject.userName, TOKENS_ARCH_TABLE).map(_.isDefined) - } yield assert(notOldTokenYet && notArchTokenYet && cleanedTokens && tokenOneValid && oldTokenGone && tokenTwoValid && archToken) - } + // "clean expired tokens" in { + // val testObject = new TestObject + // insertAwsCredentials(testObject.username, testObject.cred, isNPA = false) + // insertToken(testObject.testAwsSessionTokenValid1, testObject.username, AwsSessionTokenExpiration(Instant.now())) + // insertToken(testObject.testAwsSessionToken, testObject.username, AwsSessionTokenExpiration(Instant.now().minus(2, ChronoUnit.DAYS))) + // insertToken(testObject.testAwsSessionTokenValid2, testObject.username, AwsSessionTokenExpiration(Instant.now())) + // for { + // notOldTokenYet <- getToken(testObject.testAwsSessionToken, testObject.username).map(_.isDefined) + // notArchTokenYet <- getToken(testObject.testAwsSessionToken, testObject.username, TOKENS_ARCH_TABLE).map(_.isEmpty) + // cleanedTokens <- cleanExpiredTokens(AwsSessionTokenExpiration(Instant.now().minus(1, ChronoUnit.DAYS))).map(_ == 1) + // tokenOneValid <- getToken(testObject.testAwsSessionTokenValid1, testObject.username).map(_.isDefined) + // oldTokenGone <- getToken(testObject.testAwsSessionToken, testObject.username).map(_.isEmpty) + // tokenTwoValid <- getToken(testObject.testAwsSessionTokenValid2, testObject.username).map(_.isDefined) + // archToken <- getToken(testObject.testAwsSessionToken, testObject.username, TOKENS_ARCH_TABLE).map(_.isDefined) + // } yield assert(notOldTokenYet && notArchTokenYet && cleanedTokens && tokenOneValid && oldTokenGone && tokenTwoValid && archToken) + // } } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala index 308bc8d..f6fe44c 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala @@ -1,24 +1,46 @@ package com.ing.wbaa.rokku.sts.service.db.dao import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{MariaDBSettings, StsSettings} +import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} import com.ing.wbaa.rokku.sts.data.{AccountStatus, NPA, UserGroup, UserName} import com.ing.wbaa.rokku.sts.data.aws.{AwsAccessKey, AwsCredential} import com.ing.wbaa.rokku.sts.service.TokenGeneration -import com.ing.wbaa.rokku.sts.service.db.MariaDb import org.scalatest.wordspec.AsyncWordSpec - +import org.scalatest.{BeforeAndAfterAll} +import scala.jdk.CollectionConverters._ import scala.util.Random +import com.ing.wbaa.rokku.sts.service.db.Redis -class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO with MariaDb with TokenGeneration { +class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO with Redis with TokenGeneration with BeforeAndAfterAll{ val system: ActorSystem = ActorSystem.create("test-system") override protected[this] def stsSettings: StsSettings = StsSettings(system) - override protected[this] def mariaDBSettings: MariaDBSettings = MariaDBSettings(system) + override protected[this] def redisSettings: RedisSettings = RedisSettings(system) override lazy val dbExecutionContext = executionContext + + override protected def beforeAll(): Unit = { + // forceInitRedisConnectionPool() + List("users:", "sessionTokens:").foreach(pattern => { + val keys = redisConnectionPool.keys(pattern) + keys.asScala.foreach(key => { + println(s"WTFFFF $key") + redisConnectionPool.del(key)}) + }) + } + + override protected def afterAll(): Unit = { + println("###### AFTERALLL") + List("users:", "sessionTokens:").foreach(pattern => { + val keys = redisConnectionPool.keys(pattern) + keys.asScala.foreach(key => { + println(s"WTFFFF $key") + redisConnectionPool.del(key)}) + }) + } + private class TestObject { val cred: AwsCredential = generateAwsCredential val userName: UserName = UserName(Random.alphanumeric.take(32).mkString) @@ -29,7 +51,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "insert AwsCredentials with User" that { "are new in the db and have a unique accesskey" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false).map(r => assert(r)) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).map(r => assert(r)) getAwsCredentialAndStatus(testObject.userName).map{ case (c, _) => assert(c.contains(testObject.cred)) } getUserSecretWithExtInfo(testObject.cred.accessKey).map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) } @@ -38,14 +60,14 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit val testObject = new TestObject val newCred = generateAwsCredential - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false).flatMap { inserted => + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { inserted => getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => assert(c.contains(testObject.cred)) assert(inserted) } } - insertAwsCredentials(testObject.userName, newCred, isNpa = false).flatMap(inserted => + insertAwsCredentials(testObject.userName, newCred, isNPA = false).flatMap(inserted => getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => assert(c.contains(testObject.cred)) assert(!inserted) @@ -56,7 +78,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "have an already existing accesskey" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false).flatMap { inserted => + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { inserted => getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => assert(c.contains(testObject.cred)) assert(inserted) @@ -64,7 +86,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit } val anotherTestObject = new TestObject - insertAwsCredentials(anotherTestObject.userName, testObject.cred, isNpa = false).flatMap(inserted => + insertAwsCredentials(anotherTestObject.userName, testObject.cred, isNPA = false).flatMap(inserted => getAwsCredentialAndStatus(anotherTestObject.userName).map { case (c, _) => assert(c.isEmpty) assert(!inserted) @@ -76,7 +98,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "get User, Secret and isNPA" that { "exists" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) getUserSecretWithExtInfo(testObject.cred.accessKey).map { o => assert(o.isDefined) assert(o.get._1 == testObject.userName) @@ -95,7 +117,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "get AwsCredential" that { "exists" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) getAwsCredentialAndStatus(testObject.userName).map { case (o, _) => assert(o.isDefined) assert(o.get.accessKey == testObject.cred.accessKey) @@ -114,7 +136,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "username is different and access key is the same" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObject.cred.accessKey).map(r => assert(r)) } @@ -122,7 +144,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "username is different and access key is different" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObjectVerification.cred.accessKey) .map(r => assert(!r)) @@ -131,7 +153,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "username is same and access key is different" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObjectVerification.cred.accessKey) .map(r => assert(!r)) @@ -139,7 +161,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "username is same and access key is same" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObject.cred.accessKey) .map(r => assert(!r)) @@ -149,7 +171,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "verify groups" that { "user has two groups then one and then zero" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNpa = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) insertUserGroups(testObject.userName, testObject.userGroups) getUserSecretWithExtInfo(testObject.cred.accessKey) .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), testObject.userGroups)))) @@ -167,7 +189,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit val testObject = new TestObject val newUser = testObject.userName val newCred = testObject.cred - insertAwsCredentials(newUser, newCred, isNpa = false).map(r => assert(r)) + insertAwsCredentials(newUser, newCred, isNPA = false).map(r => assert(r)) setAccountStatus(newUser, false) getAwsCredentialAndStatus(newUser).map { case (_, AccountStatus(isEnabled)) => assert(!isEnabled) } @@ -185,7 +207,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit val testObject = new TestObject val newUser = testObject.userName val newCred = testObject.cred - insertAwsCredentials(newUser, newCred, isNpa = true) + insertAwsCredentials(newUser, newCred, isNPA = true) getAllNPAAccounts.map(l=> assert(l.data.head.accountName == newUser.value)) } } diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 5694f1b..6216f57 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -50,6 +50,13 @@ mariadb { password = ${?MARIADB_PASSWORD} } +redis { + host = ${?REDIS_HOST} + port = ${?REDIS_PORT} + username = ${?REDIS_USER} + password = ${?REDIS_PASSWORD} +} + db-dispatcher { type = Dispatcher executor = "fork-join-executor" diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index b91914c..2a368e9 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -36,6 +36,13 @@ mariadb { password = "admin" } +redis { + host = "localhost" + port = 6379 + username = "default" + password = "password" +} + db-dispatcher { type = Dispatcher executor = "fork-join-executor" diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala index af6693a..73b4b01 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala @@ -3,13 +3,13 @@ package com.ing.wbaa.rokku.sts import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config._ import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } -import com.ing.wbaa.rokku.sts.service.{ ExpiredTokenCleaner, UserTokenDbService } -import com.ing.wbaa.rokku.sts.service.db.MariaDb +import com.ing.wbaa.rokku.sts.service.{ UserTokenDbService } +import com.ing.wbaa.rokku.sts.service.db.Redis import com.ing.wbaa.rokku.sts.service.db.dao.{ STSTokenDAO, STSUserAndGroupDAO } import com.ing.wbaa.rokku.sts.vault.VaultService object Server extends App { - new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO with STSTokenDAO with MariaDb with ExpiredTokenCleaner with VaultService with KeycloakClient { + new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO with STSTokenDAO with Redis with VaultService with KeycloakClient { override implicit lazy val system: ActorSystem = ActorSystem.create("rokku-sts") override protected[this] def httpSettings: HttpSettings = HttpSettings(system) @@ -18,11 +18,11 @@ object Server extends App { override protected[this] def stsSettings: StsSettings = StsSettings(system) - override protected[this] def mariaDBSettings: MariaDBSettings = MariaDBSettings(system) - override protected[this] def vaultSettings: VaultSettings = VaultSettings(system) - //Connects to Maria DB on startup - forceInitMariaDbConnectionPool() + override protected[this] def redisSettings: RedisSettings = RedisSettings(system) + + //Connects to Redis on startup and initializes indeces + forceInitRedisConnectionPool() }.startup } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/config/RedisSettings.scala b/src/main/scala/com/ing/wbaa/rokku/sts/config/RedisSettings.scala new file mode 100644 index 0000000..4ab540c --- /dev/null +++ b/src/main/scala/com/ing/wbaa/rokku/sts/config/RedisSettings.scala @@ -0,0 +1,17 @@ +package com.ing.wbaa.rokku.sts.config + +import akka.actor.{ ExtendedActorSystem, Extension, ExtensionId, ExtensionIdProvider } +import com.typesafe.config.Config + +class RedisSettings(config: Config) extends Extension { + val host: String = config.getString("redis.host") + val port: Int = config.getInt("redis.port") + val username: String = config.getString("redis.username") + val password: String = config.getString("redis.password") +} + +object RedisSettings extends ExtensionId[RedisSettings] with ExtensionIdProvider { + override def createExtension(system: ExtendedActorSystem): RedisSettings = new RedisSettings(system.settings.config) + + override def lookup: ExtensionId[RedisSettings] = RedisSettings +} diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/ExpiredTokenCleaner.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/ExpiredTokenCleaner.scala deleted file mode 100644 index 97aed4d..0000000 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/ExpiredTokenCleaner.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.ing.wbaa.rokku.sts.service - -import java.time.Instant -import java.time.temporal.ChronoUnit - -import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.data.aws.AwsSessionTokenExpiration -import com.typesafe.scalalogging.LazyLogging - -import scala.concurrent.{ ExecutionContextExecutor, Future } -import scala.concurrent.duration._ - -trait ExpiredTokenCleaner extends Runnable with LazyLogging { - - protected[this] implicit def system: ActorSystem - - protected[this] def cleanExpiredTokens(expirationDate: AwsSessionTokenExpiration): Future[Int] - - protected[this] implicit val exContext: ExecutionContextExecutor = system.dispatcher - - system.scheduler.scheduleWithFixedDelay(10.seconds, 1.day)(this) - - override def run(): Unit = { - logger.debug("start clean expired tokens") - cleanExpiredTokens(AwsSessionTokenExpiration(Instant.now().minus(1, ChronoUnit.DAYS))).andThen { - case result => logger.debug("removed {} tokens", result) - } - } -} diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala new file mode 100644 index 0000000..1f6be51 --- /dev/null +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala @@ -0,0 +1,114 @@ +package com.ing.wbaa.rokku.sts.service.db + +import akka.actor.ActorSystem +import com.ing.wbaa.rokku.sts.config.RedisSettings +import com.typesafe.scalalogging.LazyLogging +import redis.clients.jedis.{ JedisPooled, Connection, Jedis } +import redis.clients.jedis.exceptions.JedisDataException +import redis.clients.jedis.search.{ Schema, IndexDefinition, IndexOptions } +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.{ Failure, Success, Try } + +trait Redis extends LazyLogging { + + protected[this] implicit def system: ActorSystem + + protected[this] def redisSettings: RedisSettings + + protected[this] implicit lazy val dbExecutionContext: ExecutionContext = + Try { + system.dispatchers.lookup("db-dispatcher") + } match { + case Success(dispatcher) => dispatcher + case Failure(ex) => + logger.error( + "Failed to configure dedicated db dispatcher, using default one, " + ex.getMessage + ) + system.dispatcher + } + + private val DUPLICATE_KEY_EXCEPTION_MESSAGE = "Index already exists" + + protected[this] lazy val redisConnectionPool: JedisPooled = new JedisPooled( + redisSettings.host, + redisSettings.port, + redisSettings.username, + redisSettings.password, + ) + + /** + * Force initialization of the Redis client. This ensures we get + * connection errors on startup instead of when the first call is made. + */ + protected[this] def forceInitRedisConnectionPool(): Unit = { + val client = redisConnectionPool + val usersIndex = "users-idx" + val schema = new Schema() + .addTagField("accessKey") + .addTagField("isNPA") + + val prefixDefinition = new IndexDefinition() + .setPrefixes("users:") + + // @TODO Check return value + try { + client.ftCreate( + "users-index", + IndexOptions.defaultOptions().setDefinition(prefixDefinition), schema) + logger.info(s"Created index ${usersIndex}") + } catch { + case exc: JedisDataException => + exc.getMessage() match { + case DUPLICATE_KEY_EXCEPTION_MESSAGE => + logger.info(s"Index ${usersIndex} already exists. Continuing...") + case _ => + logger.error(s"Unable to create index $usersIndex. Error: ${exc.getMessage()}") + } + case exc: Exception => + logger.error(s"Unable to create index $usersIndex. Error: ${exc.getMessage()}") + } + } + + protected[this] def withRedisConnection[T]( + databaseOperation: Connection => Future[T] + ): Future[T] = { + Try(redisConnectionPool.getPool().getResource()) match { + case Success(connection) => + val result = databaseOperation(connection) + connection.close() + result + case Failure(exc) => + logger.error("Error when getting a connection from the pool", exc) + Future.failed(exc) + } + } + + protected[this] def withRedisPool[T]( + databaseOperation: JedisPooled => Future[T] + ): Future[T] = { + try { + val result = databaseOperation(redisConnectionPool) + result + } catch { + case exc: Exception => + logger.error("Error when performing database operation", exc) + Future.failed(exc) + } + } + + private[this] def ping(connection: Connection): Future[Unit] = Future { + val response = new Jedis(connection).ping() + + assert(response.toLowerCase().equals("pong")) + } + + /** + * Performs a simple query to check the connectivity with the database/ + * + * @return + * A future that is completed when the query returns or the failure + * otherwise. + */ + protected[this] final def checkDbConnection(): Future[Unit] = + withRedisConnection(ping) +} diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala index c4992f5..8bbc12f 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala @@ -1,12 +1,12 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import java.sql._ - import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } import com.ing.wbaa.rokku.sts.data.aws.{ AwsSessionToken, AwsSessionTokenExpiration } import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging -import org.mariadb.jdbc.MariaDbPoolDataSource +import redis.clients.jedis.{ JedisPooled, Jedis } +import java.time.Instant +import scala.jdk.CollectionConverters._ import scala.concurrent.{ ExecutionContext, Future } @@ -14,13 +14,9 @@ trait STSTokenDAO extends LazyLogging with Encryption { protected[this] implicit def dbExecutionContext: ExecutionContext - protected[this] def mariaDbConnectionPool: MariaDbPoolDataSource - - protected[this] def withMariaDbConnection[T](f: Connection => Future[T]): Future[T] + protected[this] def withRedisPool[T](f: JedisPooled => Future[T]): Future[T] - private[this] val TOKENS_TABLE = "tokens" - private[this] val MYSQL_DUPLICATE__KEY_ERROR_CODE = 1062 - val TOKENS_ARCH_TABLE = "tokens_arch" + private[this] val TOKENS_PREFIX = "sessionTokens:" /** * Get Token from database against the token session identifier @@ -29,32 +25,18 @@ trait STSTokenDAO extends LazyLogging with Encryption { * @param userName * @return */ - def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - getToken(awsSessionToken, userName, TOKENS_TABLE) - - /** - * overloaded getToken method to allow get token form different table - * eg from tokens_arch for integration tests - * - * @param awsSessionToken - * @param userName - * @param table - table the token is taken - * @return - */ - def getToken(awsSessionToken: AwsSessionToken, userName: UserName, table: String = TOKENS_TABLE): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - withMariaDbConnection[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] { - connection => + def getToken(awsSessionToken: AwsSessionToken, username: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = + withRedisPool[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] { + client => { - val sqlQuery = s"SELECT * FROM $table WHERE sessiontoken = ?" Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, encryptSecret(awsSessionToken.value, userName.value)) - val results = preparedStatement.executeQuery() - if (results.first()) { - val username = UserName(results.getString("username")) - val assumeRole = getAssumeRole(results.getString("assumerole")) - val expirationDate = AwsSessionTokenExpiration(results.getTimestamp("expirationtime").toInstant) - logger.debug("getToken {} expire {} (table {})", awsSessionToken, expirationDate, table) + val values = client + .hgetAll(s"${TOKENS_PREFIX}${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}") + + if (values.size() > 0) { + val assumeRole = getAssumeRole(values.get("assumeRole")) + val expirationDate = AwsSessionTokenExpiration(Instant.parse(values.get("expirationTime"))) + logger.debug("getToken {} expire {}", awsSessionToken, expirationDate) Some((username, assumeRole, expirationDate)) } else None } @@ -70,28 +52,7 @@ trait STSTokenDAO extends LazyLogging with Encryption { * @return */ def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = - withMariaDbConnection[Boolean] { - connection => - { - val sqlQuery = s"INSERT INTO $TOKENS_TABLE (sessiontoken, username, expirationtime) VALUES (?, ?, ?)" - - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, encryptSecret(awsSessionToken.value, username.value)) - preparedStatement.setString(2, username.value) - preparedStatement.setTimestamp(3, Timestamp.from(expirationDate.value)) - preparedStatement.execute() - true - } recoverWith { - //A SQL Exception could be thrown as a result of the column sessiontoken containing a duplicate value - //return a successful future with a false result indicating it did not insert and needs to be retried with a new sessiontoken - case sqlEx: SQLException if (sqlEx.isInstanceOf[SQLIntegrityConstraintViolationException] - && sqlEx.getErrorCode.equals(MYSQL_DUPLICATE__KEY_ERROR_CODE)) => - logger.error(sqlEx.getMessage, sqlEx) - Future.successful(false) - } - } - } + insertToken(awsSessionToken, username, UserAssumeRole(""), expirationDate) /** * Insert a token item into the database @@ -103,57 +64,62 @@ trait STSTokenDAO extends LazyLogging with Encryption { * @return */ def insertToken(awsSessionToken: AwsSessionToken, username: UserName, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = - withMariaDbConnection[Boolean] { - connection => + withRedisPool[Boolean] { + client => { - val sqlQuery = s"INSERT INTO $TOKENS_TABLE (sessiontoken, username, expirationtime, assumerole) VALUES (?, ?, ?, ?)" - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, encryptSecret(awsSessionToken.value, username.value)) - preparedStatement.setString(2, username.value) - preparedStatement.setTimestamp(3, Timestamp.from(expirationDate.value)) - preparedStatement.setString(4, role.value) - preparedStatement.execute() - true - } recoverWith { - //A SQL Exception could be thrown as a result of the column sessiontoken containing a duplicate value - //return a successful future with a false result indicating it did not insert and needs to be retried with a new sessiontoken - case sqlEx: SQLException if (sqlEx.isInstanceOf[SQLIntegrityConstraintViolationException] - && sqlEx.getErrorCode.equals(MYSQL_DUPLICATE__KEY_ERROR_CODE)) => - logger.error(sqlEx.getMessage, sqlEx) - Future.successful(false) + val connection = client.getPool().getResource() + val key = s"${TOKENS_PREFIX}${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}" + + if (!client.exists(key)) { + val trx = new Jedis(connection).multi() + trx.hset(key, Map( + "username" -> username.value, + "assumeRole" -> role.value, + "expirationTime" -> expirationDate.value.toString(), + ).asJava) + + trx.expireAt(key, expirationDate.value.toEpochMilli()) + // @TODO check what exec returns + trx.exec() + connection.close() + true + } else { + //A SQL Exception could be thrown as a result of the column sessiontoken containing a duplicate value + //return a successful future with a false result indicating it did not insert and needs to be retried with a new sessiontoken + false + } } } } - /** - * Remove all expired tokens (after the expirationDate) - * - move to archive table - * - delete from original table - * @param expirationDate after the date the tokens are removed - * @return how many tokens have been removed - */ - def cleanExpiredTokens(expirationDate: AwsSessionTokenExpiration): Future[Int] = { - withMariaDbConnection[Int] { - connection => - val archiveTokensQuery = s"insert into $TOKENS_ARCH_TABLE select *, now() from $TOKENS_TABLE where expirationtime < ?" - val deleteOldTokenQuery = s"delete from $TOKENS_TABLE where expirationtime < ?" - Future { - val preparedStmArch = connection.prepareStatement(archiveTokensQuery) - preparedStmArch.setTimestamp(1, Timestamp.from(expirationDate.value)) - val archRecords = preparedStmArch.executeUpdate() - preparedStmArch.close() - - val preparedStmDel = connection.prepareStatement(deleteOldTokenQuery) - preparedStmDel.setTimestamp(1, Timestamp.from(expirationDate.value)) - val delRecords = preparedStmDel.executeUpdate() - preparedStmDel.close() - logger.info(s"archived {} tokens and deleted {}", archRecords, delRecords) - delRecords - } - } - } + // /** + // * Remove all expired tokens (after the expirationDate) + // * - move to archive table + // * - delete from original table + // * @param expirationDate after the date the tokens are removed + // * @return how many tokens have been removed + // */ + // def cleanExpiredTokens(expirationDate: AwsSessionTokenExpiration): Future[Int] = { + // withMariaDbConnection[Int] { + // connection => + // val archiveTokensQuery = s"insert into $TOKENS_ARCH_TABLE select *, now() from $TOKENS_TABLE where expirationtime < ?" + // val deleteOldTokenQuery = s"delete from $TOKENS_TABLE where expirationtime < ?" + // Future { + // val preparedStmArch = connection.prepareStatement(archiveTokensQuery) + // preparedStmArch.setTimestamp(1, Timestamp.from(expirationDate.value)) + // val archRecords = preparedStmArch.executeUpdate() + // preparedStmArch.close() + + // val preparedStmDel = connection.prepareStatement(deleteOldTokenQuery) + // preparedStmDel.setTimestamp(1, Timestamp.from(expirationDate.value)) + // val delRecords = preparedStmDel.executeUpdate() + // preparedStmDel.close() + // logger.info(s"archived {} tokens and deleted {}", archRecords, delRecords) + // delRecords + // } + // } + // } def getAssumeRole(role: String): UserAssumeRole = { if (role == null || role.equals("null")) UserAssumeRole("") else UserAssumeRole(role) diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala index 050f88a..382336c 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala @@ -1,61 +1,56 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import java.sql.{ Connection, PreparedStatement, SQLException, SQLIntegrityConstraintViolationException } - import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential, AwsSecretKey } import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, NPAAccount, NPAAccountList, UserGroup, UserName } import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging -import org.mariadb.jdbc.MariaDbPoolDataSource +import redis.clients.jedis.search.{ Query } +import scala.jdk.CollectionConverters._ -import scala.collection.mutable.ListBuffer import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success, Try } +import redis.clients.jedis.JedisPooled trait STSUserAndGroupDAO extends LazyLogging with Encryption { protected[this] implicit def dbExecutionContext: ExecutionContext - protected[this] def mariaDbConnectionPool: MariaDbPoolDataSource - - protected[this] def withMariaDbConnection[T](f: Connection => Future[T]): Future[T] + protected[this] def withRedisPool[T](f: JedisPooled => Future[T]): Future[T] - private[this] val MYSQL_DUPLICATE__KEY_ERROR_CODE = 1062 - private[this] val USER_TABLE = "users" - private[this] val USER_GROUP_TABLE = "user_groups" + // private[this] val MYSQL_DUPLICATE__KEY_ERROR_CODE = 1062 + private[this] val USERS_PREFIX = "users:" + private[this] val GROUPNAME_SEPARATOR = "," + // private[this] val USER_GROUP_TABLE = "user_groups" /** * Retrieves AWS user credentials based on the username * * @param userName The username to search an entry against */ - def getAwsCredentialAndStatus(userName: UserName): Future[(Option[AwsCredential], AccountStatus)] = - withMariaDbConnection[(Option[AwsCredential], AccountStatus)] { - - connection => + def getAwsCredentialAndStatus(username: UserName): Future[(Option[AwsCredential], AccountStatus)] = + withRedisPool[(Option[AwsCredential], AccountStatus)] { + client => { - val sqlQuery = s"SELECT * FROM $USER_TABLE WHERE username = ?" Future { Try { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, userName.value) - val results = preparedStatement.executeQuery() - if (results.first()) { + val values = client + .hgetAll(s"${USERS_PREFIX}${username.value}") + + if (values.size() > 0) { + val accessKey = AwsAccessKey(values.get("accessKey")) + val secretKey = AwsSecretKey(decryptSecret(values.get("secretKey").trim(), username.value.trim())) + val isEnabled = values.get("isEnabled").toBooleanOption.getOrElse(false) - val accessKey = AwsAccessKey(results.getString("accesskey")) - val secretKey = AwsSecretKey(decryptSecret(results.getString("secretkey"), userName.value)) - val isEnabled = results.getBoolean("isEnabled") (Some(AwsCredential(accessKey, secretKey)), AccountStatus(isEnabled)) } else (None, AccountStatus(false)) } match { case Success(r) => r case Failure(ex) => - logger.error("Cannot find user credentials for ({}), {} ", userName, ex.getMessage) + logger.error("Cannot find user credentials for ({}), {} ", username, ex.getMessage) throw ex } } } - } /** @@ -65,29 +60,25 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { * @return */ def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = - withMariaDbConnection[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] { - - connection => + withRedisPool[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] { + client => { - val separator = "," - val sqlQuery = "select username, accesskey, secretkey, isNPA, isEnabled, " + - s"(select GROUP_CONCAT(groupname SEPARATOR '$separator') from $USER_GROUP_TABLE g where g.username = u.username) as groups " + - s"from $USER_TABLE as u where u.accesskey = ? group by username, accesskey, secretkey, isNPA" - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, awsAccessKey.value) - val results = preparedStatement.executeQuery() - if (results.first()) { - val username = UserName(results.getString("username")) - val secretKey = AwsSecretKey(decryptSecret(results.getString("secretkey"), username.value)) - val isNpa = NPA(results.getBoolean("isNPA")) - val isEnabled = AccountStatus(results.getBoolean("isEnabled")) - val groupsAsString = results.getString("groups") - val groups = if (groupsAsString != null) groupsAsString.split(separator) - .map(_.trim).map(UserGroup).toSet - else Set.empty[UserGroup] - Some((username, secretKey, isNpa, isEnabled, groups)) + val query = new Query(s"@accessKey:{${awsAccessKey.value}}") + println(s" GET @accessKey:{${awsAccessKey.value}}") + val results = client.ftSearch("users-index", query); + if (results.getDocuments().size == 1) { + val document = results.getDocuments().get(0) + val username = UserName(document.getId()) + val secretKey = AwsSecretKey(decryptSecret(document.getString("secretKey").trim(), username.value.trim())) + val isEnabled = Try(document.getString("isEnabled").toBoolean).getOrElse(false) + val isNPA = Try(document.getString("isNPA").toBoolean).getOrElse(false) + val groups = Option(document.getString("groups") + .split(GROUPNAME_SEPARATOR) + .map(g => UserGroup(g.trim)).toSet) + .getOrElse(Set.empty[UserGroup]) + + Some((username, secretKey, NPA(isNPA), AccountStatus(isEnabled), groups)) } else None } } @@ -101,30 +92,31 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { * @param isNpa * @return A future with a boolean if the operation was successful or not */ - def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = - withMariaDbConnection[Boolean] { - connection => + def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNPA: Boolean): Future[Boolean] = + withRedisPool[Boolean] { + client => { - val sqlQuery = s"INSERT INTO $USER_TABLE (username, accesskey, secretkey, isNPA, isEnabled) VALUES (?, ?, ?, ?, ?)" - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(sqlQuery) - preparedStatement.setString(1, username.value) - preparedStatement.setString(2, awsCredential.accessKey.value) - preparedStatement.setString(3, encryptSecret(awsCredential.secretKey.value, username.value)) - preparedStatement.setBoolean(4, isNpa) - preparedStatement.setBoolean(5, true) - - preparedStatement.execute() + // @TODO check return value and how to handle it + println(s"Inserting : ${awsCredential.accessKey.value} ${username.value}") + client.hset(s"users:${username.value}", Map( + "accessKey" -> awsCredential.accessKey.value, + "secretKey" -> encryptSecret(awsCredential.secretKey.value.trim(), username.value.trim()), + "isNPA" -> isNPA.toString(), + "isEnabled" -> "true", + "groups" -> "", + ).asJava) true - } recoverWith { - //A SQL Exception could be thrown as a result of the column accesskey containing a duplicate value - //return a successful future with a false result indicating it did not insert and needs to be retried with a new accesskey - case sqlEx: SQLException if (sqlEx.isInstanceOf[SQLIntegrityConstraintViolationException] - && sqlEx.getErrorCode.equals(MYSQL_DUPLICATE__KEY_ERROR_CODE)) => - logger.error(sqlEx.getMessage, sqlEx) - Future.successful(false) } + + // recoverWith { + // //A SQL Exception could be thrown as a result of the column accesskey containing a duplicate value + // //return a successful future with a false result indicating it did not insert and needs to be retried with a new accesskey + // case sqlEx: SQLException if (sqlEx.isInstanceOf[SQLIntegrityConstraintViolationException] + // && sqlEx.getErrorCode.equals(MYSQL_DUPLICATE__KEY_ERROR_CODE)) => + // logger.error(sqlEx.getMessage, sqlEx) + // Future.successful(false) + // } } } @@ -134,38 +126,14 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { * @param userGroups * @return true if succeeded */ - def insertUserGroups(userName: UserName, userGroups: Set[UserGroup]): Future[Boolean] = - - withMariaDbConnection[Boolean] { - connection => + def insertUserGroups(username: UserName, userGroups: Set[UserGroup]): Future[Boolean] = + withRedisPool[Boolean] { + client => { - val deleteQuery = s"delete from $USER_GROUP_TABLE where username = ?" - val insertQuery = s"insert into $USER_GROUP_TABLE (username, groupname) values (?, ?)" Future { - Try { - val preparedDeleteStatement: PreparedStatement = connection.prepareStatement(deleteQuery) - preparedDeleteStatement.setString(1, userName.value) - val preparedInsertStatement: PreparedStatement = connection.prepareStatement(insertQuery) - userGroups.foreach { group => - preparedInsertStatement.setString(1, userName.value) - preparedInsertStatement.setString(2, group.value) - preparedInsertStatement.addBatch() - } - connection.setAutoCommit(false) - preparedDeleteStatement.executeUpdate() - preparedInsertStatement.executeBatch() - } match { - case Success(_) => - connection.commit() - connection.setAutoCommit(true) - true - case Failure(ex) => - logger.error("Cannot insert user ({}) groups {}", userName, ex) - connection.rollback() - connection.setAutoCommit(true) - throw ex - false - } + // @TODO handle errors + client.hset(s"users:${username.value}", "groups", userGroups.mkString(GROUPNAME_SEPARATOR)) + true } } } @@ -178,16 +146,13 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { * @return */ def setAccountStatus(username: UserName, enabled: Boolean): Future[Boolean] = - withMariaDbConnection[Boolean] { - connection => + withRedisPool[Boolean] { + client => { - val updateQuery = s"update $USER_TABLE set isEnabled = ? where username = ?" Future { Try { - val updateQueryStatement: PreparedStatement = connection.prepareStatement(updateQuery) - updateQueryStatement.setBoolean(1, enabled) - updateQueryStatement.setString(2, username.value) - updateQueryStatement.execute() + client.hset(s"users:${username.value}", "isEnabled", enabled.toString()) + true } match { case Success(r) => r case Failure(ex) => @@ -206,57 +171,53 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { * @param awsAccessKey * @return */ - def doesUsernameNotExistAndAccessKeyExist(userName: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = { - Future.sequence(List(doesUsernameExist(userName), doesAccessKeyExist(awsAccessKey))).map { + def doesUsernameNotExistAndAccessKeyExist(username: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = { + Future.sequence(List(doesUsernameExist(username), doesAccessKeyExist(awsAccessKey))).map { case List(false, true) => true case _ => false } } def getAllNPAAccounts: Future[NPAAccountList] = { - withMariaDbConnection { connection => - val selectNPAs = s"SELECT username, isEnabled FROM $USER_TABLE where isNPA ='1'" - - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(selectNPAs) - val listBuffer = new ListBuffer[NPAAccount] - val result = preparedStatement.executeQuery() - while (result.next()) { - listBuffer += NPAAccount(result.getString("username"), result.getBoolean("isEnabled")) + withRedisPool { + client => + Future { + val query = new Query(s"@isNPA:{true}") + // @TODO HANDLE ERRORS + val results = client.ftSearch("users-index", query) + + val npaAccounts = results.getDocuments().asScala + .map(doc => NPAAccount( + doc.getString("username"), + Try(doc.getString("isEnabled").toBoolean).getOrElse(false) + )) + + println(npaAccounts) + NPAAccountList(npaAccounts.toList) } - NPAAccountList(listBuffer.toList) - } } } - private[this] def doesUsernameExist(userName: UserName): Future[Boolean] = - withMariaDbConnection { connection => - { - val countUsersQuery = s"SELECT count(*) FROM $USER_TABLE WHERE username = ?" - - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(countUsersQuery) - preparedStatement.setString(1, userName.value) - val results = preparedStatement.executeQuery() - if (results.first()) { - results.getInt(1) > 0 - } else false + private[this] def doesUsernameExist(username: UserName): Future[Boolean] = + withRedisPool { + client => + { + Future { + // @TODO handle errors + client.exists(s"users:$username") + } } - } } private[this] def doesAccessKeyExist(awsAccessKey: AwsAccessKey): Future[Boolean] = - withMariaDbConnection { connection => + withRedisPool { client => { - val countAccesskeysQuery = s"SELECT count(*) FROM $USER_TABLE WHERE accesskey = ?" - Future { - val preparedStatement: PreparedStatement = connection.prepareStatement(countAccesskeysQuery) - preparedStatement.setString(1, awsAccessKey.value) - val results = preparedStatement.executeQuery() - if (results.first()) { - results.getInt(1) > 0 - } else false + val query = new Query(s"@accessKey:{${awsAccessKey.value}}") + // @TODO HANDLE ERRORS + val results = client.ftSearch("users-index", query) + val accessKeyExists = results.getTotalResults() != 0 + accessKeyExists } } } From 4c2f76ec5f9429192fca851a75be73708e9ff29a Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 29 Aug 2022 10:46:54 +0200 Subject: [PATCH 02/18] Fixed integration tests --- build.sbt | 8 +- docker-compose.yml | 2 + .../com/ing/wbaa/rokku/sts/AWSSTSClient.scala | 5 +- .../ing/wbaa/rokku/sts/StsServiceItTest.scala | 26 +-- .../rokku/sts/helper/OAuth2TokenRequest.scala | 10 +- .../keycloak/KeycloakTokenVerifierTest.scala | 22 +-- .../rokku/sts/service/db/RedisItTest.scala | 11 +- .../service/db/dao/STSTokenDAOItTest.scala | 97 +++++----- .../db/dao/STSUserAndGroupDAOItTest.scala | 173 ++++++++++-------- .../ing/wbaa/rokku/sts/service/db/Redis.scala | 24 +-- .../sts/service/db/dao/STSTokenDAO.scala | 51 +----- .../service/db/dao/STSUserAndGroupDAO.scala | 89 +++++---- 12 files changed, 236 insertions(+), 282 deletions(-) diff --git a/build.sbt b/build.sbt index 7537812..688a731 100644 --- a/build.sbt +++ b/build.sbt @@ -55,12 +55,9 @@ libraryDependencies ++= Seq( "com.amazonaws" % "aws-java-sdk-sts" % "1.12.278" % IntegrationTest, ) - configs(IntegrationTest) - Defaults.itSettings - -parallelExecution in IntegrationTest := false +Global / lintUnusedKeysOnLoad := false javaOptions in Universal ++= Seq( "-Dlogback.configurationFile=/rokku/logback.xml", @@ -84,4 +81,5 @@ scalariformPreferences := scalariformPreferences.value .setPreference(SingleCasePatternOnNewline, false) scalastyleFailOnError := true - +scalariformItSettings +scalariformAutoformat := true diff --git a/docker-compose.yml b/docker-compose.yml index 9a9fa93..7a7a15e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,8 @@ services: redis: image: redislabs/redisearch + environment: + - TZ=Europe/Amsterdam command: "redis-server --requirepass password --loadmodule '/usr/lib/redis/modules/redisearch.so'" ports: - 6379:6379 diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala b/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala index a5c799d..d1341ce 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala @@ -1,11 +1,10 @@ package com.ing.wbaa.rokku.sts import akka.http.scaladsl.model.Uri.Authority -import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials } import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.regions.Regions -import com.amazonaws.services.securitytoken.{AWSSecurityTokenService, AWSSecurityTokenServiceClientBuilder} - +import com.amazonaws.services.securitytoken.{ AWSSecurityTokenService, AWSSecurityTokenServiceClientBuilder } trait AWSSTSClient { diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala index 9cc1778..91ece6f 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala @@ -3,13 +3,13 @@ package com.ing.wbaa.rokku.sts import java.time.Instant import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri.{Authority, Host} +import akka.http.scaladsl.model.Uri.{ Authority, Host } import com.amazonaws.services.securitytoken.AWSSecurityTokenService -import com.amazonaws.services.securitytoken.model.{AWSSecurityTokenServiceException, AssumeRoleRequest, GetSessionTokenRequest} -import com.ing.wbaa.rokku.sts.config.{HttpSettings, KeycloakSettings, RedisSettings, StsSettings, VaultSettings} +import com.amazonaws.services.securitytoken.model.{ AWSSecurityTokenServiceException, AssumeRoleRequest, GetSessionTokenRequest } +import com.ing.wbaa.rokku.sts.config.{ HttpSettings, KeycloakSettings, RedisSettings, StsSettings, VaultSettings } import com.ing.wbaa.rokku.sts.data.aws._ -import com.ing.wbaa.rokku.sts.data.{UserAssumeRole, UserName} -import com.ing.wbaa.rokku.sts.helper.{KeycloackToken, OAuth2TokenRequest} +import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } +import com.ing.wbaa.rokku.sts.helper.{ KeycloackToken, OAuth2TokenRequest } import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } import com.ing.wbaa.rokku.sts.service.UserTokenDbService import com.ing.wbaa.rokku.sts.service.db.Redis @@ -20,7 +20,7 @@ import org.scalatest.diagrams.Diagrams import org.scalatest.wordspec.AsyncWordSpec import scala.concurrent.duration.Duration -import scala.concurrent.{ExecutionContextExecutor, Future} +import scala.concurrent.{ ExecutionContextExecutor, Future } import scala.util.Random class StsServiceItTest extends AsyncWordSpec with Diagrams @@ -51,13 +51,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams // Fixture for starting and stopping a test proxy that tests can interact with. def withTestStsService(testCode: Authority => Future[Assertion]): Future[Assertion] = { - val sts = new RokkuStsService - with KeycloakTokenVerifier - with UserTokenDbService - with STSUserAndGroupDAO - with Redis - with VaultService - with KeycloakClient { + val sts = new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO with Redis with VaultService with KeycloakClient { override implicit def system: ActorSystem = testSystem override protected[this] def httpSettings: HttpSettings = rokkuHttpSettings @@ -101,8 +95,8 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams "return credentials for valid token" in withAwsClient { stsAwsClient => withOAuth2TokenRequest(validCredentials) { keycloakToken => val credentials = stsAwsClient.getSessionToken(new GetSessionTokenRequest() - .withTokenCode(keycloakToken.access_token)) - .getCredentials + .withTokenCode(keycloakToken.access_token)) + .getCredentials assert(!credentials.getAccessKeyId.isEmpty) assert(!credentials.getSecretAccessKey.isEmpty) @@ -150,7 +144,6 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams } } - "throw AWSSecurityTokenServiceException because there is invalid token" in withAwsClient { stsAwsClient => withOAuth2TokenRequest(invalidCredentials) { keycloakToken => assertThrows[AWSSecurityTokenServiceException] { @@ -163,6 +156,5 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams } } - } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/helper/OAuth2TokenRequest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/helper/OAuth2TokenRequest.scala index e1387d4..0d91871 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/helper/OAuth2TokenRequest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/helper/OAuth2TokenRequest.scala @@ -7,13 +7,13 @@ import akka.http.scaladsl.model.headers.RawHeader import akka.stream.scaladsl.Sink import com.ing.wbaa.rokku.sts.config.KeycloakSettings -import scala.concurrent.{ExecutionContextExecutor, Future} +import scala.concurrent.{ ExecutionContextExecutor, Future } case class KeycloackToken(access_token: String) /** - * OAuth2 request for token - */ + * OAuth2 request for token + */ trait OAuth2TokenRequest { protected implicit def testSystem: ActorSystem @@ -21,13 +21,11 @@ trait OAuth2TokenRequest { protected[this] def keycloakSettings: KeycloakSettings - import spray.json._ import DefaultJsonProtocol._ private implicit val keycloakTokenJson: RootJsonFormat[KeycloackToken] = jsonFormat1(KeycloackToken) - private def getTokenResponse(formData: Map[String, String]): Future[HttpResponse] = { val contentType = RawHeader("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") Http().singleRequest(HttpRequest( @@ -40,6 +38,6 @@ trait OAuth2TokenRequest { def keycloackToken(formData: Map[String, String]): Future[KeycloackToken] = getTokenResponse(formData).map(_.entity.dataBytes.map(_.utf8String) .map(_.parseJson.convertTo[KeycloackToken]) - .runWith(Sink.seq)).flatMap(_.map(_.head)).recover{case _ => KeycloackToken("invalid")} + .runWith(Sink.seq)).flatMap(_.map(_.head)).recover { case _ => KeycloackToken("invalid") } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala index 3607df1..770a466 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala @@ -2,15 +2,15 @@ package com.ing.wbaa.rokku.sts.keycloak import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.KeycloakSettings -import com.ing.wbaa.rokku.sts.data.{BearerToken, UserGroup, UserName} -import com.ing.wbaa.rokku.sts.helper.{KeycloackToken, OAuth2TokenRequest} +import com.ing.wbaa.rokku.sts.data.{ BearerToken, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.helper.{ KeycloackToken, OAuth2TokenRequest } import org.keycloak.common.VerificationException import org.keycloak.representations.JsonWebToken import org.scalatest.Assertion import org.scalatest.diagrams.Diagrams import org.scalatest.wordspec.AsyncWordSpec -import scala.concurrent.{ExecutionContextExecutor, Future} +import scala.concurrent.{ ExecutionContextExecutor, Future } class KeycloakTokenVerifierTest extends AsyncWordSpec with Diagrams with OAuth2TokenRequest with KeycloakTokenVerifier { @@ -29,14 +29,14 @@ class KeycloakTokenVerifierTest extends AsyncWordSpec with Diagrams with OAuth2T "password" -> "password", "client_id" -> keycloakSettings.resource, "client_secret" -> keycloakSettings.clientSecret, - ) - private val validCredentialsUser2 = Map( - "grant_type" -> "password", - "username" -> "testuser", - "password" -> "password", - "client_id" -> keycloakSettings.resource, - "client_secret" -> keycloakSettings.clientSecret, - ) + ) + private val validCredentialsUser2 = Map( + "grant_type" -> "password", + "username" -> "testuser", + "password" -> "password", + "client_id" -> keycloakSettings.resource, + "client_secret" -> keycloakSettings.clientSecret, + ) "Keycloak verifier" should { "return verified token for user 1" in withOAuth2TokenRequest(validCredentialsUser1) { keycloakToken => diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala index 595009e..a7775ce 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala @@ -1,10 +1,10 @@ package com.ing.wbaa.rokku.sts.service.db import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} +import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } import org.scalatest.wordspec.AsyncWordSpec -import scala.util.{Failure, Success} +import scala.util.{ Failure, Success } class RedisItTest extends AsyncWordSpec with Redis { val system: ActorSystem = ActorSystem.create("test-system") @@ -19,16 +19,15 @@ class RedisItTest extends AsyncWordSpec with Redis { "be reachable" in { checkDbConnection() transform { - case Success(_) => Success(succeed) + case Success(_) => Success(succeed) case Failure(err) => Failure(fail(err)) } } "create index upon forceInitRedisConnectionPool call" in { forceInitRedisConnectionPool() - val info = redisConnectionPool.ftInfo("users-index") - println(info) - assert(info.containsValue("users-index")) + val info = redisConnectionPool.ftInfo(UsersIndex) + assert(info.containsValue(UsersIndex)) } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala index 1616276..6280244 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala @@ -3,19 +3,18 @@ package com.ing.wbaa.rokku.sts.service.db.dao import java.time.Instant import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} -import com.ing.wbaa.rokku.sts.data.{UserAssumeRole, UserName} -import com.ing.wbaa.rokku.sts.data.aws.{AwsCredential, AwsSessionToken, AwsSessionTokenExpiration} +import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } +import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } +import com.ing.wbaa.rokku.sts.data.aws.{ AwsCredential, AwsSessionToken, AwsSessionTokenExpiration } import com.ing.wbaa.rokku.sts.service.TokenGeneration import com.ing.wbaa.rokku.sts.service.db.Redis import org.scalatest.Assertion import org.scalatest.wordspec.AsyncWordSpec -import org.scalatest.{BeforeAndAfterAll} +import org.scalatest.{ BeforeAndAfterAll } import scala.concurrent.Future import scala.util.Random import scala.jdk.CollectionConverters._ - class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGroupDAO with Redis with TokenGeneration with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") @@ -27,28 +26,22 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr override lazy val dbExecutionContext = executionContext override protected def beforeAll(): Unit = { - // forceInitRedisConnectionPool() - List("users:", "sessionTokens:").foreach(pattern => { - val keys = redisConnectionPool.keys(pattern) - keys.asScala.foreach(key => { - println(s"WTFFFF $key") - redisConnectionPool.del(key)}) - }) + forceInitRedisConnectionPool() } - override protected def afterAll(): Unit = { - List("users:", "sessionTokens:").foreach(pattern => { + + override protected def afterAll(): Unit = { + List("users:*", "sessionTokens:*").foreach(pattern => { val keys = redisConnectionPool.keys(pattern) keys.asScala.foreach(key => { - println(key) - redisConnectionPool.del(key)}) + redisConnectionPool.del(key) + }) }) } - private class TestObject { val testAwsSessionToken: AwsSessionToken = AwsSessionToken(Random.alphanumeric.take(32).mkString) val username: UserName = UserName(Random.alphanumeric.take(32).mkString) - val testExpirationDate: AwsSessionTokenExpiration = AwsSessionTokenExpiration(Instant.now()) + val testExpirationDate: AwsSessionTokenExpiration = AwsSessionTokenExpiration(Instant.now().plusSeconds(120)) val cred: AwsCredential = generateAwsCredential val testAwsSessionTokenValid1 = AwsSessionToken(Random.alphanumeric.take(32).mkString) val testAwsSessionTokenValid2 = AwsSessionToken(Random.alphanumeric.take(32).mkString) @@ -57,8 +50,9 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr private def withInsertedUser(testCode: UserName => Future[Assertion]): Future[Assertion] = { val testObject = new TestObject - insertAwsCredentials(testObject.username, testObject.cred, isNPA = false) - testCode(testObject.username) + insertAwsCredentials(testObject.username, testObject.cred, isNPA = false).flatMap { _ => + testCode(testObject.username) + } } "STS Token DAO" should { @@ -84,6 +78,19 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr } "insert Token" that { + "that expires after 1 millisecond" in { + val testObject = new TestObject + insertToken(testObject.testAwsSessionToken, UserName("boom"), testObject.assumeRole, + AwsSessionTokenExpiration(Instant.now().plusMillis(1))).flatMap { inserted => + assert(inserted) + getToken(testObject.testAwsSessionToken, UserName("boom")).map { o => + assert(o.isEmpty) + } + } + } + } + + "insert token" that { "new to the db" in withInsertedUser { username => val testObject = new TestObject insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) @@ -92,48 +99,36 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr "token with same session token already exists " in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) - insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) - .map(r => assert(!r)) + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate).flatMap { _ => + insertToken(testObject.testAwsSessionToken, username, testObject.testExpirationDate) + .map(r => assert(!r)) + } } } - "insert Token for a role" that { + "insert token for a role" that { "new to the db" in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole , testObject.testExpirationDate) - .map(r => assert(r)) - getToken(testObject.testAwsSessionToken, username).map { o => - assert(o.isDefined) - assert(o.get._1 == username) - assert(o.get._2 == testObject.assumeRole) - assert(o.get._3.value.getEpochSecond == testObject.testExpirationDate.value.getEpochSecond) - } + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) + .flatMap { r => + assert(r) + getToken(testObject.testAwsSessionToken, username).map { o => + assert(o.isDefined) + assert(o.get._1 == username) + assert(o.get._2 == testObject.assumeRole) + assert(o.get._3.value.getEpochSecond == testObject.testExpirationDate.value.getEpochSecond) + } + } } "token with same session token already exists " in withInsertedUser { username => val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) - insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) - .map(r => assert(!r)) + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate).flatMap { _ => + insertToken(testObject.testAwsSessionToken, username, testObject.assumeRole, testObject.testExpirationDate) + .map(r => assert(!r)) + } } } - // "clean expired tokens" in { - // val testObject = new TestObject - // insertAwsCredentials(testObject.username, testObject.cred, isNPA = false) - // insertToken(testObject.testAwsSessionTokenValid1, testObject.username, AwsSessionTokenExpiration(Instant.now())) - // insertToken(testObject.testAwsSessionToken, testObject.username, AwsSessionTokenExpiration(Instant.now().minus(2, ChronoUnit.DAYS))) - // insertToken(testObject.testAwsSessionTokenValid2, testObject.username, AwsSessionTokenExpiration(Instant.now())) - // for { - // notOldTokenYet <- getToken(testObject.testAwsSessionToken, testObject.username).map(_.isDefined) - // notArchTokenYet <- getToken(testObject.testAwsSessionToken, testObject.username, TOKENS_ARCH_TABLE).map(_.isEmpty) - // cleanedTokens <- cleanExpiredTokens(AwsSessionTokenExpiration(Instant.now().minus(1, ChronoUnit.DAYS))).map(_ == 1) - // tokenOneValid <- getToken(testObject.testAwsSessionTokenValid1, testObject.username).map(_.isDefined) - // oldTokenGone <- getToken(testObject.testAwsSessionToken, testObject.username).map(_.isEmpty) - // tokenTwoValid <- getToken(testObject.testAwsSessionTokenValid2, testObject.username).map(_.isDefined) - // archToken <- getToken(testObject.testAwsSessionToken, testObject.username, TOKENS_ARCH_TABLE).map(_.isDefined) - // } yield assert(notOldTokenYet && notArchTokenYet && cleanedTokens && tokenOneValid && oldTokenGone && tokenTwoValid && archToken) - // } } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala index f6fe44c..16ec8c8 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala @@ -1,17 +1,21 @@ package com.ing.wbaa.rokku.sts.service.db.dao import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{RedisSettings, StsSettings} -import com.ing.wbaa.rokku.sts.data.{AccountStatus, NPA, UserGroup, UserName} -import com.ing.wbaa.rokku.sts.data.aws.{AwsAccessKey, AwsCredential} +import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } +import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential } import com.ing.wbaa.rokku.sts.service.TokenGeneration import org.scalatest.wordspec.AsyncWordSpec -import org.scalatest.{BeforeAndAfterAll} +import org.scalatest.{ BeforeAndAfterAll } import scala.jdk.CollectionConverters._ import scala.util.Random import com.ing.wbaa.rokku.sts.service.db.Redis -class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO with Redis with TokenGeneration with BeforeAndAfterAll{ +class STSUserAndGroupDAOItTest extends AsyncWordSpec + with STSUserAndGroupDAO + with Redis + with TokenGeneration + with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") override protected[this] def stsSettings: StsSettings = StsSettings(system) @@ -20,24 +24,14 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit override lazy val dbExecutionContext = executionContext - override protected def beforeAll(): Unit = { - // forceInitRedisConnectionPool() - List("users:", "sessionTokens:").foreach(pattern => { - val keys = redisConnectionPool.keys(pattern) - keys.asScala.foreach(key => { - println(s"WTFFFF $key") - redisConnectionPool.del(key)}) - }) + forceInitRedisConnectionPool() } override protected def afterAll(): Unit = { - println("###### AFTERALLL") - List("users:", "sessionTokens:").foreach(pattern => { - val keys = redisConnectionPool.keys(pattern) - keys.asScala.foreach(key => { - println(s"WTFFFF $key") - redisConnectionPool.del(key)}) + val keys = redisConnectionPool.keys("users:*") + keys.asScala.foreach(key => { + redisConnectionPool.del(key) }) } @@ -51,9 +45,11 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "insert AwsCredentials with User" that { "are new in the db and have a unique accesskey" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).map(r => assert(r)) - getAwsCredentialAndStatus(testObject.userName).map{ case (c, _) => assert(c.contains(testObject.cred)) } - getUserSecretWithExtInfo(testObject.cred.accessKey).map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { r => + assert(r) + getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => assert(c.contains(testObject.cred)) } + getUserSecretWithExtInfo(testObject.cred.accessKey).map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + } } "user is already present in the db" in { @@ -65,14 +61,15 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit assert(c.contains(testObject.cred)) assert(inserted) } + + insertAwsCredentials(testObject.userName, newCred, isNPA = false).flatMap(inserted => + getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => + assert(c.contains(testObject.cred)) + assert(!inserted) + } + ) } - insertAwsCredentials(testObject.userName, newCred, isNPA = false).flatMap(inserted => - getAwsCredentialAndStatus(testObject.userName).map { case (c, _) => - assert(c.contains(testObject.cred)) - assert(!inserted) - } - ) } "have an already existing accesskey" in { @@ -83,27 +80,29 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit assert(c.contains(testObject.cred)) assert(inserted) } + + val anotherTestObject = new TestObject + insertAwsCredentials(anotherTestObject.userName, testObject.cred, isNPA = false).flatMap(inserted => + getAwsCredentialAndStatus(anotherTestObject.userName).map { case (c, _) => + assert(c.isEmpty) + assert(!inserted) + } + ) } - val anotherTestObject = new TestObject - insertAwsCredentials(anotherTestObject.userName, testObject.cred, isNPA = false).flatMap(inserted => - getAwsCredentialAndStatus(anotherTestObject.userName).map { case (c, _) => - assert(c.isEmpty) - assert(!inserted) - } - ) } } "get User, Secret and isNPA" that { "exists" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - getUserSecretWithExtInfo(testObject.cred.accessKey).map { o => - assert(o.isDefined) - assert(o.get._1 == testObject.userName) - assert(o.get._2 == testObject.cred.secretKey) - assert(!o.get._3.value) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + getUserSecretWithExtInfo(testObject.cred.accessKey).map { o => + assert(o.isDefined) + assert(o.get._1 == testObject.userName) + assert(o.get._2 == testObject.cred.secretKey) + assert(!o.get._3.value) + } } } @@ -117,11 +116,12 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "get AwsCredential" that { "exists" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - getAwsCredentialAndStatus(testObject.userName).map { case (o, _) => - assert(o.isDefined) - assert(o.get.accessKey == testObject.cred.accessKey) - assert(o.get.secretKey == testObject.cred.secretKey) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + getAwsCredentialAndStatus(testObject.userName).map { case (o, _) => + assert(o.isDefined) + assert(o.get.accessKey == testObject.cred.accessKey) + assert(o.get.secretKey == testObject.cred.secretKey) + } } } @@ -136,51 +136,61 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit "username is different and access key is the same" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - - doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObject.cred.accessKey).map(r => assert(r)) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObject.cred.accessKey) + .map(r => assert(r)) + } } "username is different and access key is different" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - - doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObjectVerification.cred.accessKey) - .map(r => assert(!r)) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + doesUsernameNotExistAndAccessKeyExist(testObjectVerification.userName, testObjectVerification.cred.accessKey) + .map(r => assert(!r)) + } } "username is same and access key is different" in { val testObject = new TestObject val testObjectVerification = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - - doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObjectVerification.cred.accessKey) - .map(r => assert(!r)) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObjectVerification.cred.accessKey) + .map(r => assert(!r)) + } } "username is same and access key is same" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObject.cred.accessKey) + .map(r => assert(!r)) + } - doesUsernameNotExistAndAccessKeyExist(testObject.userName, testObject.cred.accessKey) - .map(r => assert(!r)) } } "verify groups" that { "user has two groups then one and then zero" in { val testObject = new TestObject - insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false) - insertUserGroups(testObject.userName, testObject.userGroups) - getUserSecretWithExtInfo(testObject.cred.accessKey) - .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), testObject.userGroups)))) - insertUserGroups(testObject.userName, Set(testObject.userGroups.head)) - getUserSecretWithExtInfo(testObject.cred.accessKey) - .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set(testObject.userGroups.head))))) - insertUserGroups(testObject.userName, Set.empty[UserGroup]) - getUserSecretWithExtInfo(testObject.cred.accessKey) - .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => + insertUserGroups(testObject.userName, testObject.userGroups).flatMap { _ => + { + getUserSecretWithExtInfo(testObject.cred.accessKey) + .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), testObject.userGroups)))) + + insertUserGroups(testObject.userName, Set(testObject.userGroups.head)).flatMap { _ => + getUserSecretWithExtInfo(testObject.cred.accessKey) + .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set(testObject.userGroups.head))))) + + insertUserGroups(testObject.userName, Set.empty[UserGroup]).flatMap { _ => + getUserSecretWithExtInfo(testObject.cred.accessKey) + .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + } + } + } + } + } } } @@ -189,16 +199,18 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit val testObject = new TestObject val newUser = testObject.userName val newCred = testObject.cred - insertAwsCredentials(newUser, newCred, isNPA = false).map(r => assert(r)) - - setAccountStatus(newUser, false) - getAwsCredentialAndStatus(newUser).map { case (_, AccountStatus(isEnabled)) => assert(!isEnabled) } - getUserSecretWithExtInfo(newCred.accessKey).map(c => assert(c.contains((newUser, newCred.secretKey, NPA(false), AccountStatus(false), Set.empty[UserGroup])))) + insertAwsCredentials(newUser, newCred, isNPA = false).map(r => assert(r)).flatMap { _ => - setAccountStatus(newUser, true) - getAwsCredentialAndStatus(newUser).map { case (_, AccountStatus(isEnabled)) => assert(isEnabled) } - getUserSecretWithExtInfo(newCred.accessKey).map(c => assert(c.contains((newUser, newCred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + setAccountStatus(newUser, false).flatMap { _ => + getAwsCredentialAndStatus(newUser).map { case (_, AccountStatus(isEnabled)) => assert(!isEnabled) } + getUserSecretWithExtInfo(newCred.accessKey).map(c => assert(c.contains((newUser, newCred.secretKey, NPA(false), AccountStatus(false), Set.empty[UserGroup])))) + setAccountStatus(newUser, true).flatMap { _ => + getAwsCredentialAndStatus(newUser).map { case (_, AccountStatus(isEnabled)) => assert(isEnabled) } + getUserSecretWithExtInfo(newCred.accessKey).map(c => assert(c.contains((newUser, newCred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) + } + } + } } } @@ -207,8 +219,9 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec with STSUserAndGroupDAO wit val testObject = new TestObject val newUser = testObject.userName val newCred = testObject.cred - insertAwsCredentials(newUser, newCred, isNPA = true) - getAllNPAAccounts.map(l=> assert(l.data.head.accountName == newUser.value)) + insertAwsCredentials(newUser, newCred, isNPA = true).flatMap { _ => + getAllNPAAccounts.map(l => assert(l.data.head.accountName == newUser.value)) + } } } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala index 1f6be51..8aa400c 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala @@ -13,7 +13,7 @@ trait Redis extends LazyLogging { protected[this] implicit def system: ActorSystem - protected[this] def redisSettings: RedisSettings + protected def redisSettings: RedisSettings protected[this] implicit lazy val dbExecutionContext: ExecutionContext = Try { @@ -27,9 +27,11 @@ trait Redis extends LazyLogging { system.dispatcher } - private val DUPLICATE_KEY_EXCEPTION_MESSAGE = "Index already exists" + private val DuplicateKeyExceptionMsg = "Index already exists" - protected[this] lazy val redisConnectionPool: JedisPooled = new JedisPooled( + protected val UsersIndex = "users-idx" + + protected lazy val redisConnectionPool: JedisPooled = new JedisPooled( redisSettings.host, redisSettings.port, redisSettings.username, @@ -41,8 +43,6 @@ trait Redis extends LazyLogging { * connection errors on startup instead of when the first call is made. */ protected[this] def forceInitRedisConnectionPool(): Unit = { - val client = redisConnectionPool - val usersIndex = "users-idx" val schema = new Schema() .addTagField("accessKey") .addTagField("isNPA") @@ -52,20 +52,20 @@ trait Redis extends LazyLogging { // @TODO Check return value try { - client.ftCreate( - "users-index", + redisConnectionPool.ftCreate( + UsersIndex, IndexOptions.defaultOptions().setDefinition(prefixDefinition), schema) - logger.info(s"Created index ${usersIndex}") + logger.info(s"Created index ${UsersIndex}") } catch { case exc: JedisDataException => exc.getMessage() match { - case DUPLICATE_KEY_EXCEPTION_MESSAGE => - logger.info(s"Index ${usersIndex} already exists. Continuing...") + case DuplicateKeyExceptionMsg => + logger.info(s"Index ${UsersIndex} already exists. Continuing...") case _ => - logger.error(s"Unable to create index $usersIndex. Error: ${exc.getMessage()}") + logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") } case exc: Exception => - logger.error(s"Unable to create index $usersIndex. Error: ${exc.getMessage()}") + logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") } } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala index 8bbc12f..914a5cd 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala @@ -4,19 +4,18 @@ import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } import com.ing.wbaa.rokku.sts.data.aws.{ AwsSessionToken, AwsSessionTokenExpiration } import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging -import redis.clients.jedis.{ JedisPooled, Jedis } +import redis.clients.jedis.{ Jedis } +import com.ing.wbaa.rokku.sts.service.db.Redis import java.time.Instant import scala.jdk.CollectionConverters._ import scala.concurrent.{ ExecutionContext, Future } -trait STSTokenDAO extends LazyLogging with Encryption { +trait STSTokenDAO extends LazyLogging with Encryption with Redis { protected[this] implicit def dbExecutionContext: ExecutionContext - protected[this] def withRedisPool[T](f: JedisPooled => Future[T]): Future[T] - - private[this] val TOKENS_PREFIX = "sessionTokens:" + private val SessionTokensKeyPrefix = "sessionTokens:" /** * Get Token from database against the token session identifier @@ -31,7 +30,7 @@ trait STSTokenDAO extends LazyLogging with Encryption { { Future { val values = client - .hgetAll(s"${TOKENS_PREFIX}${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}") + .hgetAll(s"$SessionTokensKeyPrefix${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}") if (values.size() > 0) { val assumeRole = getAssumeRole(values.get("assumeRole")) @@ -69,8 +68,7 @@ trait STSTokenDAO extends LazyLogging with Encryption { { Future { val connection = client.getPool().getResource() - val key = s"${TOKENS_PREFIX}${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}" - + val key = s"$SessionTokensKeyPrefix${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}" if (!client.exists(key)) { val trx = new Jedis(connection).multi() trx.hset(key, Map( @@ -79,48 +77,15 @@ trait STSTokenDAO extends LazyLogging with Encryption { "expirationTime" -> expirationDate.value.toString(), ).asJava) - trx.expireAt(key, expirationDate.value.toEpochMilli()) - // @TODO check what exec returns + trx.expireAt(key, expirationDate.value.getEpochSecond()) trx.exec() connection.close() true - } else { - //A SQL Exception could be thrown as a result of the column sessiontoken containing a duplicate value - //return a successful future with a false result indicating it did not insert and needs to be retried with a new sessiontoken - false - } + } else false } } } - // /** - // * Remove all expired tokens (after the expirationDate) - // * - move to archive table - // * - delete from original table - // * @param expirationDate after the date the tokens are removed - // * @return how many tokens have been removed - // */ - // def cleanExpiredTokens(expirationDate: AwsSessionTokenExpiration): Future[Int] = { - // withMariaDbConnection[Int] { - // connection => - // val archiveTokensQuery = s"insert into $TOKENS_ARCH_TABLE select *, now() from $TOKENS_TABLE where expirationtime < ?" - // val deleteOldTokenQuery = s"delete from $TOKENS_TABLE where expirationtime < ?" - // Future { - // val preparedStmArch = connection.prepareStatement(archiveTokensQuery) - // preparedStmArch.setTimestamp(1, Timestamp.from(expirationDate.value)) - // val archRecords = preparedStmArch.executeUpdate() - // preparedStmArch.close() - - // val preparedStmDel = connection.prepareStatement(deleteOldTokenQuery) - // preparedStmDel.setTimestamp(1, Timestamp.from(expirationDate.value)) - // val delRecords = preparedStmDel.executeUpdate() - // preparedStmDel.close() - // logger.info(s"archived {} tokens and deleted {}", archRecords, delRecords) - // delRecords - // } - // } - // } - def getAssumeRole(role: String): UserAssumeRole = { if (role == null || role.equals("null")) UserAssumeRole("") else UserAssumeRole(role) } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala index 382336c..88a43b6 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala @@ -3,24 +3,20 @@ package com.ing.wbaa.rokku.sts.service.db.dao import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential, AwsSecretKey } import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, NPAAccount, NPAAccountList, UserGroup, UserName } import com.ing.wbaa.rokku.sts.service.db.security.Encryption +import com.ing.wbaa.rokku.sts.service.db.Redis import com.typesafe.scalalogging.LazyLogging import redis.clients.jedis.search.{ Query } import scala.jdk.CollectionConverters._ import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success, Try } -import redis.clients.jedis.JedisPooled -trait STSUserAndGroupDAO extends LazyLogging with Encryption { +trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { protected[this] implicit def dbExecutionContext: ExecutionContext - protected[this] def withRedisPool[T](f: JedisPooled => Future[T]): Future[T] - - // private[this] val MYSQL_DUPLICATE__KEY_ERROR_CODE = 1062 - private[this] val USERS_PREFIX = "users:" - private[this] val GROUPNAME_SEPARATOR = "," - // private[this] val USER_GROUP_TABLE = "user_groups" + private val UsersKeyPrefix = "users:" + private val GroupnameSeparator = "," /** * Retrieves AWS user credentials based on the username @@ -34,7 +30,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { Future { Try { val values = client - .hgetAll(s"${USERS_PREFIX}${username.value}") + .hgetAll(s"$UsersKeyPrefix${username.value}") if (values.size() > 0) { val accessKey = AwsAccessKey(values.get("accessKey")) @@ -65,18 +61,17 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { { Future { val query = new Query(s"@accessKey:{${awsAccessKey.value}}") - println(s" GET @accessKey:{${awsAccessKey.value}}") - val results = client.ftSearch("users-index", query); + val results = client.ftSearch(UsersIndex, query); if (results.getDocuments().size == 1) { val document = results.getDocuments().get(0) - val username = UserName(document.getId()) + val username = UserName(document.getId().replace(UsersKeyPrefix, "")) val secretKey = AwsSecretKey(decryptSecret(document.getString("secretKey").trim(), username.value.trim())) val isEnabled = Try(document.getString("isEnabled").toBoolean).getOrElse(false) val isNPA = Try(document.getString("isNPA").toBoolean).getOrElse(false) - val groups = Option(document.getString("groups") - .split(GROUPNAME_SEPARATOR) - .map(g => UserGroup(g.trim)).toSet) - .getOrElse(Set.empty[UserGroup]) + val groups = document.getString("groups") + .split(GroupnameSeparator) + .filter(_.trim.nonEmpty) + .map(g => UserGroup(g.trim)).toSet[UserGroup] Some((username, secretKey, NPA(isNPA), AccountStatus(isEnabled), groups)) } else None @@ -96,27 +91,19 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { withRedisPool[Boolean] { client => { - Future { - // @TODO check return value and how to handle it - println(s"Inserting : ${awsCredential.accessKey.value} ${username.value}") - client.hset(s"users:${username.value}", Map( - "accessKey" -> awsCredential.accessKey.value, - "secretKey" -> encryptSecret(awsCredential.secretKey.value.trim(), username.value.trim()), - "isNPA" -> isNPA.toString(), - "isEnabled" -> "true", - "groups" -> "", - ).asJava) - true - } + doesUsernameNotExistAndAccessKeyNotExist(username, awsCredential.accessKey).map { + case true => + client.hset(s"$UsersKeyPrefix${username.value}", Map( + "accessKey" -> awsCredential.accessKey.value, + "secretKey" -> encryptSecret(awsCredential.secretKey.value.trim(), username.value.trim()), + "isNPA" -> isNPA.toString(), + "isEnabled" -> "true", + "groups" -> "", + ).asJava) + true + case false => false - // recoverWith { - // //A SQL Exception could be thrown as a result of the column accesskey containing a duplicate value - // //return a successful future with a false result indicating it did not insert and needs to be retried with a new accesskey - // case sqlEx: SQLException if (sqlEx.isInstanceOf[SQLIntegrityConstraintViolationException] - // && sqlEx.getErrorCode.equals(MYSQL_DUPLICATE__KEY_ERROR_CODE)) => - // logger.error(sqlEx.getMessage, sqlEx) - // Future.successful(false) - // } + } } } @@ -131,8 +118,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { client => { Future { - // @TODO handle errors - client.hset(s"users:${username.value}", "groups", userGroups.mkString(GROUPNAME_SEPARATOR)) + client.hset(s"users:${username.value}", "groups", userGroups.mkString(GroupnameSeparator)) true } } @@ -182,29 +168,36 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { withRedisPool { client => Future { - val query = new Query(s"@isNPA:{true}") + val query = new Query("@isNPA:{true}") // @TODO HANDLE ERRORS - val results = client.ftSearch("users-index", query) + val results = client.ftSearch(UsersIndex, query) val npaAccounts = results.getDocuments().asScala - .map(doc => NPAAccount( - doc.getString("username"), - Try(doc.getString("isEnabled").toBoolean).getOrElse(false) - )) + .map(doc => { + NPAAccount( + doc.getId().replace("users:", ""), + Try(doc.getString("isEnabled").toBoolean).getOrElse(false) + ) + }) - println(npaAccounts) NPAAccountList(npaAccounts.toList) } } } + private[this] def doesUsernameNotExistAndAccessKeyNotExist(username: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = { + Future.sequence(List(doesUsernameExist(username), doesAccessKeyExist(awsAccessKey))).map { + case List(false, false) => true + case _ => false + } + } + private[this] def doesUsernameExist(username: UserName): Future[Boolean] = withRedisPool { client => { Future { - // @TODO handle errors - client.exists(s"users:$username") + client.exists(s"users:${username.value}") } } } @@ -215,7 +208,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption { Future { val query = new Query(s"@accessKey:{${awsAccessKey.value}}") // @TODO HANDLE ERRORS - val results = client.ftSearch("users-index", query) + val results = client.ftSearch(UsersIndex, query) val accessKeyExists = results.getTotalResults() != 0 accessKeyExists } From ab5197f422e86c8872f857b8070c38ca2686938d Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 29 Aug 2022 10:55:01 +0200 Subject: [PATCH 03/18] Removed MariaDB references --- docker-compose.yml | 7 -- src/main/resources/application.conf | 6 -- src/main/resources/reference.conf | 7 -- .../rokku/sts/config/MariaDBSettings.scala | 16 ----- .../wbaa/rokku/sts/service/db/MariaDb.scala | 68 ------------------- 5 files changed, 104 deletions(-) delete mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/config/MariaDBSettings.scala delete mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/service/db/MariaDb.scala diff --git a/docker-compose.yml b/docker-compose.yml index 7a7a15e..c5c3055 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,13 +10,6 @@ services: ports: - 8080:8080 - mariadb: - image: wbaa/rokku-dev-mariadb:0.0.8 - environment: - - MYSQL_ROOT_PASSWORD=admin - ports: - - 3307:3306 - redis: image: redislabs/redisearch environment: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 6216f57..e0d9481 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -44,12 +44,6 @@ rokku { } } -mariadb { - url = ${?MARIADB_URL} - username = ${?MARIADB_USERNAME} - password = ${?MARIADB_PASSWORD} -} - redis { host = ${?REDIS_HOST} port = ${?REDIS_PORT} diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 2a368e9..b8580d6 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -29,13 +29,6 @@ rokku { } } - -mariadb { - url = "jdbc:mysql:loadbalance://localhost:3307,localhost:3307/rokku" - username = "root" - password = "admin" -} - redis { host = "localhost" port = 6379 diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/config/MariaDBSettings.scala b/src/main/scala/com/ing/wbaa/rokku/sts/config/MariaDBSettings.scala deleted file mode 100644 index 4190dd1..0000000 --- a/src/main/scala/com/ing/wbaa/rokku/sts/config/MariaDBSettings.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.ing.wbaa.rokku.sts.config - -import akka.actor.{ ExtendedActorSystem, Extension, ExtensionId, ExtensionIdProvider } -import com.typesafe.config.Config - -class MariaDBSettings(config: Config) extends Extension { - val url: String = config.getString("mariadb.url") - val username: String = config.getString("mariadb.username") - val password: String = config.getString("mariadb.password") -} - -object MariaDBSettings extends ExtensionId[MariaDBSettings] with ExtensionIdProvider { - override def createExtension(system: ExtendedActorSystem): MariaDBSettings = new MariaDBSettings(system.settings.config) - - override def lookup: ExtensionId[MariaDBSettings] = MariaDBSettings -} diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/MariaDb.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/MariaDb.scala deleted file mode 100644 index b25223d..0000000 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/MariaDb.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.ing.wbaa.rokku.sts.service.db - -import java.sql.Connection - -import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.MariaDBSettings -import com.typesafe.scalalogging.LazyLogging -import org.mariadb.jdbc.MariaDbPoolDataSource - -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Success, Try } - -trait MariaDb extends LazyLogging { - - protected[this] implicit def system: ActorSystem - - protected[this] def mariaDBSettings: MariaDBSettings - - protected[this] implicit lazy val dbExecutionContext: ExecutionContext = - Try { - system.dispatchers.lookup("db-dispatcher") - } match { - case Success(dispatcher) => dispatcher - case Failure(ex) => - logger.error("Failed to configure dedicated db dispatcher, using default one, " + ex.getMessage) - system.dispatcher - } - - protected[this] lazy val mariaDbConnectionPool: MariaDbPoolDataSource = { - val pool = new MariaDbPoolDataSource(mariaDBSettings.url) - pool.setUser(mariaDBSettings.username) - pool.setPassword(mariaDBSettings.password) - pool - } - - /** - * Force initialization of the MariaDB client plugin. - * This ensures we get connection errors on startup instead of when the first call is made. - */ - protected[this] def forceInitMariaDbConnectionPool(): Unit = mariaDbConnectionPool - - protected[this] def withMariaDbConnection[T](databaseOperation: Connection => Future[T]): Future[T] = { - Try(mariaDbConnectionPool.getConnection()) match { - case Success(connection) => - val result = databaseOperation(connection) - connection.close() - result - case Failure(exc) => - logger.error("Error when getting a connection from the pool", exc) - Future.failed(exc) - } - } - - private[this] def selectOne(connection: Connection): Future[Unit] = Future { - val statement = connection.prepareStatement("SELECT 1") - val results = statement.executeQuery() - - assert(results.first()) - } - - /** - * Performs a simple query to check the connectivity with the database/ - * - * @return A future that is completed when the query returns or the failure - * otherwise. - */ - protected[this] final def checkDbConnection(): Future[Unit] = withMariaDbConnection(selectOne) -} From e3737b8b977170d96d6b0a703811d97c1c22c7a1 Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 29 Aug 2022 11:21:59 +0200 Subject: [PATCH 04/18] Refactored/renamed code related to Redis connection --- .../ing/wbaa/rokku/sts/StsServiceItTest.scala | 12 ++--- .../sts/keycloak/KeycloakClientItTest.scala | 6 +-- .../keycloak/KeycloakTokenVerifierTest.scala | 6 +-- .../rokku/sts/service/db/RedisItTest.scala | 4 +- .../service/db/dao/STSTokenDAOItTest.scala | 18 +++---- ...DAOItTest.scala => STSUserDAOItTest.scala} | 16 +++---- .../scala/com/ing/wbaa/rokku/sts/Server.scala | 6 +-- .../com/ing/wbaa/rokku/sts/api/AdminApi.scala | 22 ++++----- .../com/ing/wbaa/rokku/sts/api/STSApi.scala | 4 +- .../sts/data/AuthenticationUserInfo.scala | 2 +- .../ing/wbaa/rokku/sts/data/STSUserInfo.scala | 4 +- .../rokku/sts/keycloak/KeycloakClient.scala | 4 +- .../sts/keycloak/KeycloakTokenVerifier.scala | 2 +- .../sts/service/UserTokenDbService.scala | 32 ++++++------- .../ing/wbaa/rokku/sts/service/db/Redis.scala | 42 +++++----------- .../sts/service/db/dao/STSTokenDAO.scala | 10 ++-- ...UserAndGroupDAO.scala => STSUserDAO.scala} | 24 +++++----- .../wbaa/rokku/sts/vault/VaultService.scala | 10 ++-- .../ing/wbaa/rokku/sts/api/AdminApiTest.scala | 12 ++--- .../ing/wbaa/rokku/sts/api/STSApiTest.scala | 6 +-- .../ing/wbaa/rokku/sts/api/UserApiTest.scala | 4 +- .../rokku/sts/data/aws/AwsRoleArnTest.scala | 6 +-- .../sts/service/UserTokenDbServiceTest.scala | 48 +++++++++---------- 23 files changed, 141 insertions(+), 159 deletions(-) rename src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/{STSUserAndGroupDAOItTest.scala => STSUserDAOItTest.scala} (96%) rename src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/{STSUserAndGroupDAO.scala => STSUserDAO.scala} (89%) diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala index 91ece6f..300fe4e 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala @@ -8,12 +8,12 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenService import com.amazonaws.services.securitytoken.model.{ AWSSecurityTokenServiceException, AssumeRoleRequest, GetSessionTokenRequest } import com.ing.wbaa.rokku.sts.config.{ HttpSettings, KeycloakSettings, RedisSettings, StsSettings, VaultSettings } import com.ing.wbaa.rokku.sts.data.aws._ -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } +import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } import com.ing.wbaa.rokku.sts.helper.{ KeycloackToken, OAuth2TokenRequest } import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } import com.ing.wbaa.rokku.sts.service.UserTokenDbService import com.ing.wbaa.rokku.sts.service.db.Redis -import com.ing.wbaa.rokku.sts.service.db.dao.STSUserAndGroupDAO +import com.ing.wbaa.rokku.sts.service.db.dao.STSUserDAO import com.ing.wbaa.rokku.sts.vault.VaultService import org.scalatest.Assertion import org.scalatest.diagrams.Diagrams @@ -51,7 +51,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams // Fixture for starting and stopping a test proxy that tests can interact with. def withTestStsService(testCode: Authority => Future[Assertion]): Future[Assertion] = { - val sts = new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO with Redis with VaultService with KeycloakClient { + val sts = new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with Redis with VaultService with KeycloakClient { override implicit def system: ActorSystem = testSystem override protected[this] def httpSettings: HttpSettings = rokkuHttpSettings @@ -62,13 +62,13 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams override protected[this] def redisSettings: RedisSettings = new RedisSettings(testSystem.settings.config) - override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) - override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) - override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = + override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = Future.successful(None) override def generateAwsSession(duration: Option[Duration]): AwsSession = AwsSession( diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClientItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClientItTest.scala index 28f3647..f267a3d 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClientItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClientItTest.scala @@ -3,7 +3,7 @@ package com.ing.wbaa.rokku.sts.keycloak import akka.Done import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.KeycloakSettings -import com.ing.wbaa.rokku.sts.data.UserName +import com.ing.wbaa.rokku.sts.data.Username import com.ing.wbaa.rokku.sts.helper.OAuth2TokenRequest import org.scalatest.diagrams.Diagrams import org.scalatest.wordspec.AsyncWordSpec @@ -22,14 +22,14 @@ class KeycloakClientItTest extends AsyncWordSpec with Diagrams with OAuth2TokenR var createdUserId = KeycloakUserId("") "add a user" in { - insertUserToKeycloak(UserName(username)).map(addedUserId => { + insertUserToKeycloak(Username(username)).map(addedUserId => { createdUserId = addedUserId assert(addedUserId.id.nonEmpty) }) } "thrown error when adding existing user" in { - recoverToSucceededIf[javax.ws.rs.WebApplicationException](insertUserToKeycloak(UserName(username))) + recoverToSucceededIf[javax.ws.rs.WebApplicationException](insertUserToKeycloak(Username(username))) } "delete the created user" in { diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala index 770a466..978a118 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifierTest.scala @@ -2,7 +2,7 @@ package com.ing.wbaa.rokku.sts.keycloak import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.KeycloakSettings -import com.ing.wbaa.rokku.sts.data.{ BearerToken, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.{ BearerToken, UserGroup, Username } import com.ing.wbaa.rokku.sts.helper.{ KeycloackToken, OAuth2TokenRequest } import org.keycloak.common.VerificationException import org.keycloak.representations.JsonWebToken @@ -41,13 +41,13 @@ class KeycloakTokenVerifierTest extends AsyncWordSpec with Diagrams with OAuth2T "Keycloak verifier" should { "return verified token for user 1" in withOAuth2TokenRequest(validCredentialsUser1) { keycloakToken => val token = verifyAuthenticationToken(BearerToken(keycloakToken.access_token)) - assert(token.map(_.userName).contains(UserName("userone"))) + assert(token.map(_.userName).contains(Username("userone"))) assert(token.exists(_.userGroups.isEmpty)) } "return verified token for user 2" in withOAuth2TokenRequest(validCredentialsUser2) { keycloakToken => val token = verifyAuthenticationToken(BearerToken(keycloakToken.access_token)) - assert(token.map(_.userName).contains(UserName("testuser"))) + assert(token.map(_.userName).contains(Username("testuser"))) assert(token.exists(g => g.userGroups(UserGroup("testgroup")) && g.userGroups(UserGroup("group3")))) } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala index a7775ce..2eba005 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala @@ -25,8 +25,8 @@ class RedisItTest extends AsyncWordSpec with Redis { } "create index upon forceInitRedisConnectionPool call" in { - forceInitRedisConnectionPool() - val info = redisConnectionPool.ftInfo(UsersIndex) + createSecondaryIndex() + val info = redisPooledConnection.ftInfo(UsersIndex) assert(info.containsValue(UsersIndex)) } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala index 6280244..0188573 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala @@ -4,7 +4,7 @@ import java.time.Instant import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } +import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } import com.ing.wbaa.rokku.sts.data.aws.{ AwsCredential, AwsSessionToken, AwsSessionTokenExpiration } import com.ing.wbaa.rokku.sts.service.TokenGeneration import com.ing.wbaa.rokku.sts.service.db.Redis @@ -15,7 +15,7 @@ import scala.concurrent.Future import scala.util.Random import scala.jdk.CollectionConverters._ -class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGroupDAO with Redis with TokenGeneration with BeforeAndAfterAll { +class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserDAO with Redis with TokenGeneration with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") @@ -26,21 +26,21 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr override lazy val dbExecutionContext = executionContext override protected def beforeAll(): Unit = { - forceInitRedisConnectionPool() + createSecondaryIndex() } override protected def afterAll(): Unit = { List("users:*", "sessionTokens:*").foreach(pattern => { - val keys = redisConnectionPool.keys(pattern) + val keys = redisPooledConnection.keys(pattern) keys.asScala.foreach(key => { - redisConnectionPool.del(key) + redisPooledConnection.del(key) }) }) } private class TestObject { val testAwsSessionToken: AwsSessionToken = AwsSessionToken(Random.alphanumeric.take(32).mkString) - val username: UserName = UserName(Random.alphanumeric.take(32).mkString) + val username: Username = Username(Random.alphanumeric.take(32).mkString) val testExpirationDate: AwsSessionTokenExpiration = AwsSessionTokenExpiration(Instant.now().plusSeconds(120)) val cred: AwsCredential = generateAwsCredential val testAwsSessionTokenValid1 = AwsSessionToken(Random.alphanumeric.take(32).mkString) @@ -48,7 +48,7 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr val assumeRole = UserAssumeRole("testRole") } - private def withInsertedUser(testCode: UserName => Future[Assertion]): Future[Assertion] = { + private def withInsertedUser(testCode: Username => Future[Assertion]): Future[Assertion] = { val testObject = new TestObject insertAwsCredentials(testObject.username, testObject.cred, isNPA = false).flatMap { _ => testCode(testObject.username) @@ -80,10 +80,10 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserAndGr "insert Token" that { "that expires after 1 millisecond" in { val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, UserName("boom"), testObject.assumeRole, + insertToken(testObject.testAwsSessionToken, Username("boom"), testObject.assumeRole, AwsSessionTokenExpiration(Instant.now().plusMillis(1))).flatMap { inserted => assert(inserted) - getToken(testObject.testAwsSessionToken, UserName("boom")).map { o => + getToken(testObject.testAwsSessionToken, Username("boom")).map { o => assert(o.isEmpty) } } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala similarity index 96% rename from src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala rename to src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala index 16ec8c8..f465148 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala @@ -2,7 +2,7 @@ package com.ing.wbaa.rokku.sts.service.db.dao import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } -import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, UserGroup, Username } import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential } import com.ing.wbaa.rokku.sts.service.TokenGeneration import org.scalatest.wordspec.AsyncWordSpec @@ -11,8 +11,8 @@ import scala.jdk.CollectionConverters._ import scala.util.Random import com.ing.wbaa.rokku.sts.service.db.Redis -class STSUserAndGroupDAOItTest extends AsyncWordSpec - with STSUserAndGroupDAO +class STSUserDAOItTest extends AsyncWordSpec + with STSUserDAO with Redis with TokenGeneration with BeforeAndAfterAll { @@ -25,19 +25,19 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec override lazy val dbExecutionContext = executionContext override protected def beforeAll(): Unit = { - forceInitRedisConnectionPool() + createSecondaryIndex() } override protected def afterAll(): Unit = { - val keys = redisConnectionPool.keys("users:*") + val keys = redisPooledConnection.keys("users:*") keys.asScala.foreach(key => { - redisConnectionPool.del(key) + redisPooledConnection.del(key) }) } private class TestObject { val cred: AwsCredential = generateAwsCredential - val userName: UserName = UserName(Random.alphanumeric.take(32).mkString) + val userName: Username = Username(Random.alphanumeric.take(32).mkString) val userGroups: Set[UserGroup] = Set(UserGroup(Random.alphanumeric.take(10).mkString), UserGroup(Random.alphanumeric.take(10).mkString)) } @@ -126,7 +126,7 @@ class STSUserAndGroupDAOItTest extends AsyncWordSpec } "doesn't exist" in { - getAwsCredentialAndStatus(UserName("DOESNTEXIST")).map { case (o, _) => + getAwsCredentialAndStatus(Username("DOESNTEXIST")).map { case (o, _) => assert(o.isEmpty) } } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala index 73b4b01..5894fdc 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala @@ -5,11 +5,11 @@ import com.ing.wbaa.rokku.sts.config._ import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } import com.ing.wbaa.rokku.sts.service.{ UserTokenDbService } import com.ing.wbaa.rokku.sts.service.db.Redis -import com.ing.wbaa.rokku.sts.service.db.dao.{ STSTokenDAO, STSUserAndGroupDAO } +import com.ing.wbaa.rokku.sts.service.db.dao.{ STSTokenDAO, STSUserDAO } import com.ing.wbaa.rokku.sts.vault.VaultService object Server extends App { - new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserAndGroupDAO with STSTokenDAO with Redis with VaultService with KeycloakClient { + new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with STSTokenDAO with Redis with VaultService with KeycloakClient { override implicit lazy val system: ActorSystem = ActorSystem.create("rokku-sts") override protected[this] def httpSettings: HttpSettings = HttpSettings(system) @@ -23,6 +23,6 @@ object Server extends App { override protected[this] def redisSettings: RedisSettings = RedisSettings(system) //Connects to Redis on startup and initializes indeces - forceInitRedisConnectionPool() + createSecondaryIndex() }.startup } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/api/AdminApi.scala b/src/main/scala/com/ing/wbaa/rokku/sts/api/AdminApi.scala index 366b7c0..1f3664b 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/api/AdminApi.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/api/AdminApi.scala @@ -34,15 +34,15 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { // Keycloak protected[this] def verifyAuthenticationToken(token: BearerToken): Option[AuthenticationUserInfo] - protected[this] def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] + protected[this] def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] - protected[this] def insertNpaCredentialsToVault(username: UserName, safeName: String, awsCredential: AwsCredential): Future[Boolean] + protected[this] def insertNpaCredentialsToVault(username: Username, safeName: String, awsCredential: AwsCredential): Future[Boolean] - protected[this] def setAccountStatus(username: UserName, enabled: Boolean): Future[Boolean] + protected[this] def setAccountStatus(username: Username, enabled: Boolean): Future[Boolean] protected[this] def getAllNPAAccounts: Future[NPAAccountList] - protected[this] def insertUserToKeycloak(username: UserName): Future[KeycloakUserId] + protected[this] def insertUserToKeycloak(username: Username): Future[KeycloakUserId] implicit val requestId = RequestId("") @@ -57,9 +57,9 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { authorizeToken(verifyAuthenticationToken) { keycloakUserInfo => if (userInAdminGroups(keycloakUserInfo.userGroups)) { val awsCredentials = AwsCredential(AwsAccessKey(awsAccessKey), AwsSecretKey(awsSecretKey)) - onComplete(insertAwsCredentials(UserName(npaAccount), awsCredentials, isNpa = true)) { + onComplete(insertAwsCredentials(Username(npaAccount), awsCredentials, isNpa = true)) { case Success(true) => - insertNpaCredentialsToVault(UserName(npaAccount), safeName, awsCredentials) + insertNpaCredentialsToVault(Username(npaAccount), safeName, awsCredentials) logger.info(s"NPA: $npaAccount successfully created by ${keycloakUserInfo.userName}") complete(ResponseMessage("NPA Created", s"NPA: $npaAccount successfully created by ${keycloakUserInfo.userName}", "NPA add")) case Success(false) => @@ -85,9 +85,9 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { headerValueByName("Authorization") { bearerToken => if (verifyInternalToken(bearerToken)) { val awsCredentials = AwsCredential(AwsAccessKey(awsAccessKey), AwsSecretKey(awsSecretKey)) - onComplete(insertAwsCredentials(UserName(npaAccount), awsCredentials, isNpa = true)) { + onComplete(insertAwsCredentials(Username(npaAccount), awsCredentials, isNpa = true)) { case Success(true) => - insertNpaCredentialsToVault(UserName(npaAccount), safeName, awsCredentials) + insertNpaCredentialsToVault(Username(npaAccount), safeName, awsCredentials) logger.info(s"NPA: $npaAccount successfully created") complete(ResponseMessage("NPA Created", s"NPA: $npaAccount successfully created", "NPA add")) case Success(false) => @@ -132,7 +132,7 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { case "enable" => true case "disable" => false } - onComplete(setAccountStatus(UserName(uid), action)) { + onComplete(setAccountStatus(Username(uid), action)) { case Success(_) => complete(ResponseMessage(s"Account action", s"User account $uid enabled: $action", "user account")) case Failure(ex) => complete(ResponseMessage("Account disable failed", ex.getMessage, "user account")) } @@ -151,7 +151,7 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { authorizeToken(verifyAuthenticationToken) { keycloakUserInfo => extractUri { _ => if (userInAdminGroups(keycloakUserInfo.userGroups)) { - onComplete(insertUserToKeycloak(UserName(username))) { + onComplete(insertUserToKeycloak(Username(username))) { case Success(_) => complete(ResponseMessage(s"Add user ok", s"$username added", "keycloak")) case Failure(ex) => complete(ResponseMessage(s"Add user error", ex.getMessage, "keycloak")) } @@ -171,7 +171,7 @@ trait AdminApi extends LazyLogging with Encryption with JwtToken { formFields((Symbol("username"))) { username => headerValueByName("Authorization") { bearerToken => if (verifyInternalToken(bearerToken)) { - onComplete(insertUserToKeycloak(UserName(username))) { + onComplete(insertUserToKeycloak(Username(username))) { case Success(_) => complete(ResponseMessage(s"Add user ok", s"$username added", "keycloak")) case Failure(ex) => complete(ResponseMessage(s"Add user error", ex.getMessage, "keycloak")) } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/api/STSApi.scala b/src/main/scala/com/ing/wbaa/rokku/sts/api/STSApi.scala index 49567e2..a857042 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/api/STSApi.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/api/STSApi.scala @@ -38,8 +38,8 @@ trait STSApi extends LazyLogging with TokenXML { ) } - protected[this] def getAwsCredentialWithToken(userName: UserName, userGroups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] - protected[this] def getAwsCredentialWithToken(userName: UserName, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] + protected[this] def getAwsCredentialWithToken(userName: Username, userGroups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] + protected[this] def getAwsCredentialWithToken(userName: Username, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] // Keycloak protected[this] def verifyAuthenticationToken(token: BearerToken): Option[AuthenticationUserInfo] diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/data/AuthenticationUserInfo.scala b/src/main/scala/com/ing/wbaa/rokku/sts/data/AuthenticationUserInfo.scala index e7e708f..8add21a 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/data/AuthenticationUserInfo.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/data/AuthenticationUserInfo.scala @@ -5,7 +5,7 @@ case class AuthenticationTokenId(value: String) extends AnyVal case class UserGroup(value: String) extends AnyVal case class AuthenticationUserInfo( - userName: UserName, + userName: Username, userGroups: Set[UserGroup], keycloakTokenId: AuthenticationTokenId, userRoles: Set[UserAssumeRole]) diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/data/STSUserInfo.scala b/src/main/scala/com/ing/wbaa/rokku/sts/data/STSUserInfo.scala index d1d56b1..ca61641 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/data/STSUserInfo.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/data/STSUserInfo.scala @@ -2,12 +2,12 @@ package com.ing.wbaa.rokku.sts.data import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsSecretKey } -case class UserName(value: String) extends AnyVal +case class Username(value: String) extends AnyVal case class UserAssumeRole(value: String) extends AnyVal case class STSUserInfo( - userName: UserName, + userName: Username, userGroup: Set[UserGroup], awsAccessKey: AwsAccessKey, awsSecretKey: AwsSecretKey, diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClient.scala b/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClient.scala index 953ece9..20ebc9e 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClient.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakClient.scala @@ -2,7 +2,7 @@ package com.ing.wbaa.rokku.sts.keycloak import akka.Done import com.ing.wbaa.rokku.sts.config.KeycloakSettings -import com.ing.wbaa.rokku.sts.data.UserName +import com.ing.wbaa.rokku.sts.data.Username import com.typesafe.scalalogging.LazyLogging import org.keycloak.OAuth2Constants import org.keycloak.admin.client.{ CreatedResponseUtil, KeycloakBuilder } @@ -34,7 +34,7 @@ trait KeycloakClient extends LazyLogging { * @param username npa username * @return the created user keycloak id */ - def insertUserToKeycloak(username: UserName): Future[KeycloakUserId] = { + def insertUserToKeycloak(username: Username): Future[KeycloakUserId] = { val user = new UserRepresentation() user.setEnabled(false) diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifier.scala b/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifier.scala index 3140c8a..d5e44bc 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifier.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/keycloak/KeycloakTokenVerifier.scala @@ -34,7 +34,7 @@ trait KeycloakTokenVerifier extends LazyLogging { logger.info("Token successfully validated with Keycloak user = {}", keycloakToken.getPreferredUsername) Some( AuthenticationUserInfo( - UserName(keycloakToken.getPreferredUsername), + Username(keycloakToken.getPreferredUsername), keycloakToken.getOtherClaims .getOrDefault("user-groups", new util.ArrayList[String]()) .asInstanceOf[util.ArrayList[String]].asScala.toSet.map(UserGroup), diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala index db360bb..0a3ccd9 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala @@ -3,7 +3,7 @@ package com.ing.wbaa.rokku.sts.service import java.time.Instant import com.ing.wbaa.rokku.sts.data.aws._ -import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, STSUserInfo, TokenActive, TokenActiveForRole, TokenNotActive, TokenStatus, UserAssumeRole, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, STSUserInfo, TokenActive, TokenActiveForRole, TokenNotActive, TokenStatus, UserAssumeRole, UserGroup, Username } import com.typesafe.scalalogging.LazyLogging import scala.concurrent.duration.Duration @@ -13,21 +13,21 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { implicit protected[this] def executionContext: ExecutionContext - protected[this] def getAwsCredentialAndStatus(userName: UserName): Future[(Option[AwsCredential], AccountStatus)] + protected[this] def getAwsCredentialAndStatus(userName: Username): Future[(Option[AwsCredential], AccountStatus)] - protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] + protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] - protected[this] def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] + protected[this] def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] - protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] + protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] - protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] + protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, expirationDate: AwsSessionTokenExpiration): Future[Boolean] - protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] + protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] - protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] + protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: Username, awsAccessKey: AwsAccessKey): Future[Boolean] - protected[this] def insertUserGroups(userName: UserName, userGroups: Set[UserGroup]): Future[Boolean] + protected[this] def insertUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] /** * Retrieve or generate Credentials and generate a new Session @@ -37,7 +37,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { * @param duration optional: the duration of the session, if duration is not given then it defaults to the application application default * @return */ - def getAwsCredentialWithToken(userName: UserName, userGroups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] = + def getAwsCredentialWithToken(userName: Username, userGroups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] = for { (awsCredential, AccountStatus(isEnabled)) <- getOrGenerateAwsCredentialWithStatus(userName) awsSession <- getNewAwsSession(userName, duration) @@ -57,7 +57,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { * @param duration optional: the duration of the session, if duration is not given then it defaults to the application application default * @return */ - def getAwsCredentialWithToken(userName: UserName, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] = + def getAwsCredentialWithToken(userName: Username, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] = for { (awsCredential, AccountStatus(isEnabled)) <- getOrGenerateAwsCredentialWithStatus(userName) awsSession <- getNewAwsSessionWithToken(userName, role, duration) @@ -114,7 +114,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { * @param generationTriesLeft Number of times to retry token generation, in case it collides * @return */ - private[this] def getNewAwsSession(userName: UserName, duration: Option[Duration], generationTriesLeft: Int = 3): Future[AwsSession] = { + private[this] def getNewAwsSession(userName: Username, duration: Option[Duration], generationTriesLeft: Int = 3): Future[AwsSession] = { val newAwsSession = generateAwsSession(duration) insertToken(newAwsSession.sessionToken, userName, newAwsSession.expiration) .flatMap { @@ -137,7 +137,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { * @param generationTriesLeft Number of times to retry token generation, in case it collides * @return */ - private[this] def getNewAwsSessionWithToken(userName: UserName, role: UserAssumeRole, duration: Option[Duration], generationTriesLeft: Int = 3): Future[AwsSession] = { + private[this] def getNewAwsSessionWithToken(userName: Username, role: UserAssumeRole, duration: Option[Duration], generationTriesLeft: Int = 3): Future[AwsSession] = { val newAwsSession = generateAwsSession(duration) insertToken(newAwsSession.sessionToken, userName, role, newAwsSession.expiration) .flatMap { @@ -155,7 +155,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { * Adds a user to the DB with aws credentials generated for it. * In case the user already exists, it returns the already existing credentials. */ - private[this] def getOrGenerateAwsCredentialWithStatus(userName: UserName): Future[(AwsCredential, AccountStatus)] = + private[this] def getOrGenerateAwsCredentialWithStatus(userName: Username): Future[(AwsCredential, AccountStatus)] = getAwsCredentialAndStatus(userName) .flatMap { case (Some(awsCredential), AccountStatus(isEnabled)) => @@ -168,7 +168,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { case (None, _) => getNewAwsCredential(userName).map(c => (c, AccountStatus(true))) } - private[this] def getNewAwsCredential(userName: UserName): Future[AwsCredential] = { + private[this] def getNewAwsCredential(userName: Username): Future[AwsCredential] = { val newAwsCredential = generateAwsCredential insertAwsCredentials(userName, newAwsCredential, isNpa = false) .flatMap { @@ -185,7 +185,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { } } - private[this] def isTokenActive(awsSessionToken: AwsSessionToken, userName: UserName): Future[TokenStatus] = { + private[this] def isTokenActive(awsSessionToken: AwsSessionToken, userName: Username): Future[TokenStatus] = { getToken(awsSessionToken, userName).map { case Some((_, _, tokenExpiration)) if isTokenExpired(tokenExpiration) => logger.warn(s"Provided sessionToken has expired at: {} for token: {}", tokenExpiration.value, awsSessionToken.value) diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala index 8aa400c..4a6ac8e 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala @@ -3,7 +3,7 @@ package com.ing.wbaa.rokku.sts.service.db import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.RedisSettings import com.typesafe.scalalogging.LazyLogging -import redis.clients.jedis.{ JedisPooled, Connection, Jedis } +import redis.clients.jedis.{ JedisPooled, Jedis } import redis.clients.jedis.exceptions.JedisDataException import redis.clients.jedis.search.{ Schema, IndexDefinition, IndexOptions } import scala.concurrent.{ ExecutionContext, Future } @@ -31,7 +31,7 @@ trait Redis extends LazyLogging { protected val UsersIndex = "users-idx" - protected lazy val redisConnectionPool: JedisPooled = new JedisPooled( + protected lazy val redisPooledConnection: JedisPooled = new JedisPooled( redisSettings.host, redisSettings.port, redisSettings.username, @@ -39,10 +39,9 @@ trait Redis extends LazyLogging { ) /** - * Force initialization of the Redis client. This ensures we get - * connection errors on startup instead of when the first call is made. + * Create secondary search index for users fields */ - protected[this] def forceInitRedisConnectionPool(): Unit = { + protected[this] def createSecondaryIndex(): Unit = { val schema = new Schema() .addTagField("accessKey") .addTagField("isNPA") @@ -50,9 +49,8 @@ trait Redis extends LazyLogging { val prefixDefinition = new IndexDefinition() .setPrefixes("users:") - // @TODO Check return value try { - redisConnectionPool.ftCreate( + redisPooledConnection.ftCreate( UsersIndex, IndexOptions.defaultOptions().setDefinition(prefixDefinition), schema) logger.info(s"Created index ${UsersIndex}") @@ -69,25 +67,11 @@ trait Redis extends LazyLogging { } } - protected[this] def withRedisConnection[T]( - databaseOperation: Connection => Future[T] - ): Future[T] = { - Try(redisConnectionPool.getPool().getResource()) match { - case Success(connection) => - val result = databaseOperation(connection) - connection.close() - result - case Failure(exc) => - logger.error("Error when getting a connection from the pool", exc) - Future.failed(exc) - } - } - protected[this] def withRedisPool[T]( databaseOperation: JedisPooled => Future[T] ): Future[T] = { try { - val result = databaseOperation(redisConnectionPool) + val result = databaseOperation(redisPooledConnection) result } catch { case exc: Exception => @@ -96,12 +80,6 @@ trait Redis extends LazyLogging { } } - private[this] def ping(connection: Connection): Future[Unit] = Future { - val response = new Jedis(connection).ping() - - assert(response.toLowerCase().equals("pong")) - } - /** * Performs a simple query to check the connectivity with the database/ * @@ -109,6 +87,10 @@ trait Redis extends LazyLogging { * A future that is completed when the query returns or the failure * otherwise. */ - protected[this] final def checkDbConnection(): Future[Unit] = - withRedisConnection(ping) + protected[this] final def checkDbConnection(): Future[Unit] = { + Future { + val response = new Jedis(redisPooledConnection.getPool().getResource()).ping() + assert(response.toLowerCase().equals("pong")) + } + } } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala index 914a5cd..398811a 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala @@ -1,6 +1,6 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, UserName } +import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } import com.ing.wbaa.rokku.sts.data.aws.{ AwsSessionToken, AwsSessionTokenExpiration } import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging @@ -24,8 +24,8 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis { * @param userName * @return */ - def getToken(awsSessionToken: AwsSessionToken, username: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - withRedisPool[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] { + def getToken(awsSessionToken: AwsSessionToken, username: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = + withRedisPool[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] { client => { Future { @@ -50,7 +50,7 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis { * @param expirationDate * @return */ - def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + def insertToken(awsSessionToken: AwsSessionToken, username: Username, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = insertToken(awsSessionToken, username, UserAssumeRole(""), expirationDate) /** @@ -62,7 +62,7 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis { * @param expirationDate * @return */ - def insertToken(awsSessionToken: AwsSessionToken, username: UserName, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + def insertToken(awsSessionToken: AwsSessionToken, username: Username, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = withRedisPool[Boolean] { client => { diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala similarity index 89% rename from src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala rename to src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala index 88a43b6..ebad128 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserAndGroupDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala @@ -1,7 +1,7 @@ package com.ing.wbaa.rokku.sts.service.db.dao import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential, AwsSecretKey } -import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, NPAAccount, NPAAccountList, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, NPAAccount, NPAAccountList, UserGroup, Username } import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.ing.wbaa.rokku.sts.service.db.Redis import com.typesafe.scalalogging.LazyLogging @@ -11,7 +11,7 @@ import scala.jdk.CollectionConverters._ import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success, Try } -trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { +trait STSUserDAO extends LazyLogging with Encryption with Redis { protected[this] implicit def dbExecutionContext: ExecutionContext @@ -23,7 +23,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * * @param userName The username to search an entry against */ - def getAwsCredentialAndStatus(username: UserName): Future[(Option[AwsCredential], AccountStatus)] = + def getAwsCredentialAndStatus(username: Username): Future[(Option[AwsCredential], AccountStatus)] = withRedisPool[(Option[AwsCredential], AccountStatus)] { client => { @@ -55,8 +55,8 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * @param awsAccessKey * @return */ - def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = - withRedisPool[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] { + def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = + withRedisPool[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] { client => { Future { @@ -64,7 +64,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { val results = client.ftSearch(UsersIndex, query); if (results.getDocuments().size == 1) { val document = results.getDocuments().get(0) - val username = UserName(document.getId().replace(UsersKeyPrefix, "")) + val username = Username(document.getId().replace(UsersKeyPrefix, "")) val secretKey = AwsSecretKey(decryptSecret(document.getString("secretKey").trim(), username.value.trim())) val isEnabled = Try(document.getString("isEnabled").toBoolean).getOrElse(false) val isNPA = Try(document.getString("isNPA").toBoolean).getOrElse(false) @@ -87,7 +87,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * @param isNpa * @return A future with a boolean if the operation was successful or not */ - def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNPA: Boolean): Future[Boolean] = + def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNPA: Boolean): Future[Boolean] = withRedisPool[Boolean] { client => { @@ -113,7 +113,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * @param userGroups * @return true if succeeded */ - def insertUserGroups(username: UserName, userGroups: Set[UserGroup]): Future[Boolean] = + def insertUserGroups(username: Username, userGroups: Set[UserGroup]): Future[Boolean] = withRedisPool[Boolean] { client => { @@ -131,7 +131,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * @param enabled * @return */ - def setAccountStatus(username: UserName, enabled: Boolean): Future[Boolean] = + def setAccountStatus(username: Username, enabled: Boolean): Future[Boolean] = withRedisPool[Boolean] { client => { @@ -157,7 +157,7 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { * @param awsAccessKey * @return */ - def doesUsernameNotExistAndAccessKeyExist(username: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = { + def doesUsernameNotExistAndAccessKeyExist(username: Username, awsAccessKey: AwsAccessKey): Future[Boolean] = { Future.sequence(List(doesUsernameExist(username), doesAccessKeyExist(awsAccessKey))).map { case List(false, true) => true case _ => false @@ -185,14 +185,14 @@ trait STSUserAndGroupDAO extends LazyLogging with Encryption with Redis { } } - private[this] def doesUsernameNotExistAndAccessKeyNotExist(username: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = { + private[this] def doesUsernameNotExistAndAccessKeyNotExist(username: Username, awsAccessKey: AwsAccessKey): Future[Boolean] = { Future.sequence(List(doesUsernameExist(username), doesAccessKeyExist(awsAccessKey))).map { case List(false, false) => true case _ => false } } - private[this] def doesUsernameExist(username: UserName): Future[Boolean] = + private[this] def doesUsernameExist(username: Username): Future[Boolean] = withRedisPool { client => { diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/vault/VaultService.scala b/src/main/scala/com/ing/wbaa/rokku/sts/vault/VaultService.scala index b345fa5..9d471e9 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/vault/VaultService.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/vault/VaultService.scala @@ -6,7 +6,7 @@ import akka.actor.ActorSystem import com.bettercloud.vault.response.VaultResponse import com.bettercloud.vault.{ Vault, VaultConfig } import com.ing.wbaa.rokku.sts.config.VaultSettings -import com.ing.wbaa.rokku.sts.data.UserName +import com.ing.wbaa.rokku.sts.data.Username import com.ing.wbaa.rokku.sts.data.aws.AwsCredential import com.typesafe.scalalogging.LazyLogging @@ -22,7 +22,7 @@ trait VaultService extends LazyLogging { implicit protected[this] def executionContext: ExecutionContext - def insertNpaCredentialsToVault(username: UserName, safeName: String, awsCredential: AwsCredential): Future[Boolean] = Future { + def insertNpaCredentialsToVault(username: Username, safeName: String, awsCredential: AwsCredential): Future[Boolean] = Future { if (safeName.equalsIgnoreCase(vaultSettings.vaultPath)) { writeSingleVaultEntry(username, safeName, awsCredential) @@ -33,7 +33,7 @@ trait VaultService extends LazyLogging { }(executionContext) - private def writeSingleVaultEntry(username: UserName, safeName: String, awsCredential: AwsCredential) = { + private def writeSingleVaultEntry(username: Username, safeName: String, awsCredential: AwsCredential) = { val vault = getVaultInstance() val secretsToSave: Map[String, AnyRef] = Map("accessKey" -> awsCredential.accessKey.value, "secretKey" -> awsCredential.secretKey.value) logger.info(s"Performing vault write operation to ${vaultSettings.vaultPath} for ${username.value}") @@ -70,7 +70,7 @@ trait VaultService extends LazyLogging { vault } - private def reportOnOperationOutcome(s: VaultResponse, name: UserName): Boolean = { + private def reportOnOperationOutcome(s: VaultResponse, name: Username): Boolean = { val status = s.getRestResponse.getStatus val retries = s.getRetries val body = new String(s.getRestResponse.getBody, StandardCharsets.UTF_8) @@ -89,7 +89,7 @@ trait VaultService extends LazyLogging { } } - private def reportOnOperationOutcome(e: Throwable, name: UserName): Boolean = { + private def reportOnOperationOutcome(e: Throwable, name: Username): Boolean = { logger.error(s"Couldn't write credentials for ${name.value} to vault: \n" + e.getMessage) false } diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/api/AdminApiTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/api/AdminApiTest.scala index 5ba92ef..64709f8 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/api/AdminApiTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/api/AdminApiTest.scala @@ -32,19 +32,19 @@ class AdminApiTest extends AnyWordSpec protected[this] def verifyAuthenticationToken(token: BearerToken): Option[AuthenticationUserInfo] = token.value match { - case "valid" => Some(AuthenticationUserInfo(UserName("username"), Set(UserGroup("admins"), UserGroup("group2")), AuthenticationTokenId("tokenOk"), Set.empty)) - case "notAdmin" => Some(AuthenticationUserInfo(UserName("username"), Set(UserGroup("group1"), UserGroup("group2")), AuthenticationTokenId("tokenOk"), Set.empty)) + case "valid" => Some(AuthenticationUserInfo(Username("username"), Set(UserGroup("admins"), UserGroup("group2")), AuthenticationTokenId("tokenOk"), Set.empty)) + case "notAdmin" => Some(AuthenticationUserInfo(Username("username"), Set(UserGroup("group1"), UserGroup("group2")), AuthenticationTokenId("tokenOk"), Set.empty)) case _ => None } - override protected[this] def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = Future(true) + override protected[this] def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = Future(true) - override protected[this] def setAccountStatus(username: UserName, enabled: Boolean): Future[Boolean] = Future.successful(true) + override protected[this] def setAccountStatus(username: Username, enabled: Boolean): Future[Boolean] = Future.successful(true) override protected[this] def getAllNPAAccounts: Future[NPAAccountList] = Future(NPAAccountList(List(NPAAccount("testNPA", true)))) - override protected[this] def insertNpaCredentialsToVault(username: UserName, safeName: String, awsCredential: AwsCredential): Future[Boolean] = Future(true) + override protected[this] def insertNpaCredentialsToVault(username: Username, safeName: String, awsCredential: AwsCredential): Future[Boolean] = Future(true) - protected[this] def insertUserToKeycloak(username: UserName): Future[KeycloakUserId] = username.value match { + protected[this] def insertUserToKeycloak(username: Username): Future[KeycloakUserId] = username.value match { case "duplicate" => Future.failed(new RuntimeException("duplicate")) case _ => Future.successful(KeycloakUserId("1)")) } diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/api/STSApiTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/api/STSApiTest.scala index fe40a81..c7c494d 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/api/STSApiTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/api/STSApiTest.scala @@ -32,11 +32,11 @@ class STSApiTest extends AnyWordSpec with Diagrams with ScalatestRouteTest { override def verifyAuthenticationToken(token: BearerToken): Option[AuthenticationUserInfo] = token.value match { - case "valid" => Some(data.AuthenticationUserInfo(UserName("name"), Set(UserGroup("testgroup")), AuthenticationTokenId("token"), Set(UserAssumeRole("testrole")))) + case "valid" => Some(data.AuthenticationUserInfo(Username("name"), Set(UserGroup("testgroup")), AuthenticationTokenId("token"), Set(UserAssumeRole("testrole")))) case _ => None } - override protected[this] def getAwsCredentialWithToken(userName: UserName, groups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] = { + override protected[this] def getAwsCredentialWithToken(userName: Username, groups: Set[UserGroup], duration: Option[Duration]): Future[AwsCredentialWithToken] = { Future.successful(AwsCredentialWithToken( AwsCredential( AwsAccessKey("accesskey"), @@ -49,7 +49,7 @@ class STSApiTest extends AnyWordSpec with Diagrams with ScalatestRouteTest { )) } - override protected[this] def getAwsCredentialWithToken(userName: UserName, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] = { + override protected[this] def getAwsCredentialWithToken(userName: Username, userGroups: Set[UserGroup], role: UserAssumeRole, duration: Option[Duration]): Future[AwsCredentialWithToken] = { Future.successful(AwsCredentialWithToken( AwsCredential( AwsAccessKey("accesskey"), diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/api/UserApiTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/api/UserApiTest.scala index 8bccce1..cd5a8b5 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/api/UserApiTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/api/UserApiTest.scala @@ -9,7 +9,7 @@ import com.auth0.jwt.JWT import com.auth0.jwt.algorithms.Algorithm import com.ing.wbaa.rokku.sts.config.StsSettings import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsSecretKey, AwsSessionToken } -import com.ing.wbaa.rokku.sts.data.{ STSUserInfo, UserGroup, UserName } +import com.ing.wbaa.rokku.sts.data.{ STSUserInfo, UserGroup, Username } import org.scalatest.BeforeAndAfterAll import org.scalatest.diagrams.Diagrams import org.scalatest.wordspec.AnyWordSpec @@ -23,7 +23,7 @@ class UserApiTest extends AnyWordSpec trait testUserApi extends UserApi { override def isCredentialActive(awsAccessKey: AwsAccessKey, awsSessionToken: Option[AwsSessionToken]): Future[Option[STSUserInfo]] = - Future.successful(Some(STSUserInfo(UserName("username"), Set(UserGroup("group1"), UserGroup("group2")), AwsAccessKey("a"), AwsSecretKey("s"), None))) + Future.successful(Some(STSUserInfo(Username("username"), Set(UserGroup("group1"), UserGroup("group2")), AwsAccessKey("a"), AwsSecretKey("s"), None))) } val testSystem: ActorSystem = ActorSystem.create("test-system") diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/data/aws/AwsRoleArnTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/data/aws/AwsRoleArnTest.scala index 7f97c6f..400ace4 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/data/aws/AwsRoleArnTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/data/aws/AwsRoleArnTest.scala @@ -13,7 +13,7 @@ class AwsRoleArnTest extends AnyWordSpec { val result = AwsRoleArn(s"arn:aws:iam::123456789012:role/$testRoleName") .getRoleUserCanAssume( - AuthenticationUserInfo(UserName(""), Set.empty, AuthenticationTokenId(""), Set(UserAssumeRole(testRoleName))) + AuthenticationUserInfo(Username(""), Set.empty, AuthenticationTokenId(""), Set(UserAssumeRole(testRoleName))) ) assert(result.contains(UserAssumeRole(testRoleName))) } @@ -23,7 +23,7 @@ class AwsRoleArnTest extends AnyWordSpec { val result = AwsRoleArn(s"arn:aws:iam:invalid:123456789012:role/$testRoleName") .getRoleUserCanAssume( - AuthenticationUserInfo(UserName(""), Set.empty, AuthenticationTokenId(""), Set(UserAssumeRole(testRoleName))) + AuthenticationUserInfo(Username(""), Set.empty, AuthenticationTokenId(""), Set(UserAssumeRole(testRoleName))) ) assert(result.isEmpty) } @@ -33,7 +33,7 @@ class AwsRoleArnTest extends AnyWordSpec { val result = AwsRoleArn(s"arn:aws:iam::123456789012:role/$testRoleName") .getRoleUserCanAssume( - AuthenticationUserInfo(UserName(""), Set.empty, AuthenticationTokenId(""), Set.empty) + AuthenticationUserInfo(Username(""), Set.empty, AuthenticationTokenId(""), Set.empty) ) assert(result.isEmpty) } diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala index a9db97b..33ee0d6 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala @@ -21,33 +21,33 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { override implicit def executionContext: ExecutionContext = testSystem.dispatcher override protected[this] def stsSettings: StsSettings = StsSettings(testSystem) - override protected[this] def getAwsCredentialAndStatus(userName: UserName): Future[(Option[AwsCredential], AccountStatus)] = + override protected[this] def getAwsCredentialAndStatus(userName: Username): Future[(Option[AwsCredential], AccountStatus)] = Future.successful((Some(AwsCredential(AwsAccessKey("a"), AwsSecretKey("s"))), AccountStatus(true))) - override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = - Future.successful(Some((UserName("u"), AwsSecretKey("s"), NPA(false), AccountStatus(true), Set(UserGroup("testGroup"))))) + override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = + Future.successful(Some((Username("u"), AwsSecretKey("s"), NPA(false), AccountStatus(true), Set(UserGroup("testGroup"))))) - override protected[this] def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = + override protected[this] def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = Future.successful(true) - override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - Future.successful(Some((UserName("u"), UserAssumeRole(""), AwsSessionTokenExpiration(Instant.now().plusSeconds(20))))) + override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = + Future.successful(Some((Username("u"), UserAssumeRole(""), AwsSessionTokenExpiration(Instant.now().plusSeconds(20))))) - override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) - override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: UserName, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = + override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) - override protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: UserName, awsAccessKey: AwsAccessKey): Future[Boolean] = + override protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: Username, awsAccessKey: AwsAccessKey): Future[Boolean] = Future.successful(false) - override protected[this] def insertUserGroups(userName: UserName, userGroups: Set[UserGroup]): Future[Boolean] = + override protected[this] def insertUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] = Future.successful(true) } private class TestObject { - val userName: UserName = UserName(Random.alphanumeric.take(32).mkString) + val userName: Username = Username(Random.alphanumeric.take(32).mkString) val duration: Duration = Duration(2, TimeUnit.HOURS) } @@ -68,7 +68,7 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { "are new credentials" in { val testObject = new TestObject new UserTokenDbServiceTest { - override protected[this] def getAwsCredentialAndStatus(userName: UserName): Future[(Option[AwsCredential], AccountStatus)] = Future.successful((None, AccountStatus(false))) + override protected[this] def getAwsCredentialAndStatus(userName: Username): Future[(Option[AwsCredential], AccountStatus)] = Future.successful((None, AccountStatus(false))) }.getAwsCredentialWithToken(testObject.userName, Set.empty[UserGroup], Some(testObject.duration)).map { c => assertExpirationValid(c.session.expiration, testObject.duration) } @@ -78,9 +78,9 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { val testObject = new TestObject val utdst = new UserTokenDbServiceTest { - override protected[this] def getAwsCredentialAndStatus(userName: UserName): Future[(Option[AwsCredential], AccountStatus)] = Future.successful((None, AccountStatus(false))) + override protected[this] def getAwsCredentialAndStatus(userName: Username): Future[(Option[AwsCredential], AccountStatus)] = Future.successful((None, AccountStatus(false))) - override protected[this] def insertAwsCredentials(username: UserName, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = + override protected[this] def insertAwsCredentials(username: Username, awsCredential: AwsCredential, isNpa: Boolean): Future[Boolean] = Future.successful(false) } @@ -110,7 +110,7 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { val utds = new UserTokenDbServiceTest {} utds.getAwsCredentialWithToken(t.userName, Set.empty[UserGroup], Some(t.duration)).flatMap { awsCredWithToken => utds.isCredentialActive(awsCredWithToken.awsCredential.accessKey, Some(awsCredWithToken.session.sessionToken)).map { u => - assert(u.map(_.userName).contains(UserName("u"))) + assert(u.map(_.userName).contains(Username("u"))) assert(u.map(_.awsAccessKey).contains(AwsAccessKey("a"))) assert(u.map(_.awsSecretKey).contains(AwsSecretKey("s"))) assert(u.map(_.userGroup).contains(Set(UserGroup("testGroup")))) @@ -122,8 +122,8 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { "has valid accesskey and sessiontoken is inactive" in { val t = new TestObject val utds = new UserTokenDbServiceTest { - override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - Future.successful(Some((UserName("u"), UserAssumeRole("testGroup"), AwsSessionTokenExpiration(Instant.now().minusSeconds(20))))) + override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = + Future.successful(Some((Username("u"), UserAssumeRole("testGroup"), AwsSessionTokenExpiration(Instant.now().minusSeconds(20))))) } utds.getAwsCredentialWithToken(t.userName, Set.empty[UserGroup], Some(Duration(-1, TimeUnit.HOURS))) .flatMap { awsCredWithToken => @@ -135,13 +135,13 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { "has valid accesskey and sessiontoken is active for a role" in { val t = new TestObject val utds = new UserTokenDbServiceTest { - override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: UserName): Future[Option[(UserName, UserAssumeRole, AwsSessionTokenExpiration)]] = - Future.successful(Some((UserName("u"), UserAssumeRole("testRole"), AwsSessionTokenExpiration(Instant.now().plusSeconds(20))))) + override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = + Future.successful(Some((Username("u"), UserAssumeRole("testRole"), AwsSessionTokenExpiration(Instant.now().plusSeconds(20))))) } utds.getAwsCredentialWithToken(t.userName, Set.empty[UserGroup], Some(t.duration)) .flatMap { awsCredWithToken => utds.isCredentialActive(awsCredWithToken.awsCredential.accessKey, Some(awsCredWithToken.session.sessionToken)).map { u => - assert(u.map(_.userName).contains(UserName("u"))) + assert(u.map(_.userName).contains(Username("u"))) assert(u.map(_.awsAccessKey).contains(AwsAccessKey("a"))) assert(u.map(_.awsSecretKey).contains(AwsSecretKey("s"))) assert(u.map(_.userGroup).contains(Set.empty)) @@ -166,8 +166,8 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { "has valid accesskey, no sessiontoken and is an NPA" in { val t = new TestObject val utds = new UserTokenDbServiceTest { - override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = - Future.successful(Some((UserName("u"), AwsSecretKey("s"), NPA(true), AccountStatus(true), Set.empty[UserGroup]))) + override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = + Future.successful(Some((Username("u"), AwsSecretKey("s"), NPA(true), AccountStatus(true), Set.empty[UserGroup]))) } utds.getAwsCredentialWithToken(t.userName, Set.empty[UserGroup], Some(t.duration)).flatMap { awsCredWithToken => utds.isCredentialActive(awsCredWithToken.awsCredential.accessKey, None).map(a => assert(a.isDefined)) @@ -177,8 +177,8 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { "not provide user credentials with account disabled" in { val t = new TestObject val utds = new UserTokenDbServiceTest { - override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(UserName, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = - Future.successful(Some((UserName("u"), AwsSecretKey("s"), NPA(false), AccountStatus(false), Set.empty[UserGroup]))) + override protected[this] def getUserSecretWithExtInfo(awsAccessKey: AwsAccessKey): Future[Option[(Username, AwsSecretKey, NPA, AccountStatus, Set[UserGroup])]] = + Future.successful(Some((Username("u"), AwsSecretKey("s"), NPA(false), AccountStatus(false), Set.empty[UserGroup]))) } utds.getAwsCredentialWithToken(t.userName, Set.empty[UserGroup], Some(t.duration)).flatMap { awsCredWithToken => utds.isCredentialActive(awsCredWithToken.awsCredential.accessKey, Some(awsCredWithToken.session.sessionToken)).map(a => assert(!a.isDefined)) From f97388c586e89a3179f6fd3b1044fc0c9b81d654 Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 29 Aug 2022 13:48:53 +0200 Subject: [PATCH 05/18] Introduced RedisModel --- .gitignore | 2 +- .../com/ing/wbaa/rokku/sts/AWSSTSClient.scala | 6 +- .../ing/wbaa/rokku/sts/StsServiceItTest.scala | 33 ++++--- .../rokku/sts/service/db/RedisItTest.scala | 10 +- .../service/db/dao/STSTokenDAOItTest.scala | 30 ++++-- .../sts/service/db/dao/STSUserDAOItTest.scala | 22 +++-- .../scala/com/ing/wbaa/rokku/sts/Server.scala | 6 +- .../ing/wbaa/rokku/sts/service/db/Redis.scala | 46 ++------- .../rokku/sts/service/db/RedisModel.scala | 99 +++++++++++++++++++ .../sts/service/db/dao/STSTokenDAO.scala | 42 ++++---- .../rokku/sts/service/db/dao/STSUserDAO.scala | 82 +++++++-------- 11 files changed, 243 insertions(+), 135 deletions(-) create mode 100644 src/main/scala/com/ing/wbaa/rokku/sts/service/db/RedisModel.scala diff --git a/.gitignore b/.gitignore index cba27d1..8448682 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ snapshots .scalafmt.conf project/.bloop/ project/metals.sbt - +.vscode diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala b/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala index d1341ce..1de1b77 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/AWSSTSClient.scala @@ -1,10 +1,12 @@ package com.ing.wbaa.rokku.sts import akka.http.scaladsl.model.Uri.Authority -import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials } +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.auth.BasicAWSCredentials import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.regions.Regions -import com.amazonaws.services.securitytoken.{ AWSSecurityTokenService, AWSSecurityTokenServiceClientBuilder } +import com.amazonaws.services.securitytoken.AWSSecurityTokenService +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder trait AWSSTSClient { diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala index 300fe4e..66a07d5 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/StsServiceItTest.scala @@ -1,26 +1,37 @@ package com.ing.wbaa.rokku.sts -import java.time.Instant - import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri.{ Authority, Host } +import akka.http.scaladsl.model.Uri.Authority +import akka.http.scaladsl.model.Uri.Host import com.amazonaws.services.securitytoken.AWSSecurityTokenService -import com.amazonaws.services.securitytoken.model.{ AWSSecurityTokenServiceException, AssumeRoleRequest, GetSessionTokenRequest } -import com.ing.wbaa.rokku.sts.config.{ HttpSettings, KeycloakSettings, RedisSettings, StsSettings, VaultSettings } +import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException +import com.amazonaws.services.securitytoken.model.AssumeRoleRequest +import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest +import com.ing.wbaa.rokku.sts.config.HttpSettings +import com.ing.wbaa.rokku.sts.config.KeycloakSettings +import com.ing.wbaa.rokku.sts.config.RedisSettings +import com.ing.wbaa.rokku.sts.config.StsSettings +import com.ing.wbaa.rokku.sts.config.VaultSettings +import com.ing.wbaa.rokku.sts.data.UserAssumeRole +import com.ing.wbaa.rokku.sts.data.Username import com.ing.wbaa.rokku.sts.data.aws._ -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } -import com.ing.wbaa.rokku.sts.helper.{ KeycloackToken, OAuth2TokenRequest } -import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } +import com.ing.wbaa.rokku.sts.helper.KeycloackToken +import com.ing.wbaa.rokku.sts.helper.OAuth2TokenRequest +import com.ing.wbaa.rokku.sts.keycloak.KeycloakClient +import com.ing.wbaa.rokku.sts.keycloak.KeycloakTokenVerifier import com.ing.wbaa.rokku.sts.service.UserTokenDbService import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.RedisModel import com.ing.wbaa.rokku.sts.service.db.dao.STSUserDAO import com.ing.wbaa.rokku.sts.vault.VaultService import org.scalatest.Assertion import org.scalatest.diagrams.Diagrams import org.scalatest.wordspec.AsyncWordSpec +import java.time.Instant +import scala.concurrent.ExecutionContextExecutor +import scala.concurrent.Future import scala.concurrent.duration.Duration -import scala.concurrent.{ ExecutionContextExecutor, Future } import scala.util.Random class StsServiceItTest extends AsyncWordSpec with Diagrams @@ -51,7 +62,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams // Fixture for starting and stopping a test proxy that tests can interact with. def withTestStsService(testCode: Authority => Future[Assertion]): Future[Assertion] = { - val sts = new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with Redis with VaultService with KeycloakClient { + val sts = new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with Redis with RedisModel with VaultService with KeycloakClient { override implicit def system: ActorSystem = testSystem override protected[this] def httpSettings: HttpSettings = rokkuHttpSettings @@ -68,7 +79,7 @@ class StsServiceItTest extends AsyncWordSpec with Diagrams override protected[this] def insertToken(awsSessionToken: AwsSessionToken, username: Username, role: UserAssumeRole, expirationDate: AwsSessionTokenExpiration): Future[Boolean] = Future.successful(true) - override protected[this] def getToken(awsSessionToken: AwsSessionToken, userName: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = + override protected[this] def getToken(awsSessionToken: AwsSessionToken, username: Username): Future[Option[(Username, UserAssumeRole, AwsSessionTokenExpiration)]] = Future.successful(None) override def generateAwsSession(duration: Option[Duration]): AwsSession = AwsSession( diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala index 2eba005..6fdeb63 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/RedisItTest.scala @@ -1,12 +1,14 @@ package com.ing.wbaa.rokku.sts.service.db import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } +import com.ing.wbaa.rokku.sts.config.RedisSettings +import com.ing.wbaa.rokku.sts.config.StsSettings import org.scalatest.wordspec.AsyncWordSpec -import scala.util.{ Failure, Success } +import scala.util.Failure +import scala.util.Success -class RedisItTest extends AsyncWordSpec with Redis { +class RedisItTest extends AsyncWordSpec with Redis with RedisModel { val system: ActorSystem = ActorSystem.create("test-system") protected[this] def redisSettings: RedisSettings = RedisSettings(system) @@ -25,7 +27,7 @@ class RedisItTest extends AsyncWordSpec with Redis { } "create index upon forceInitRedisConnectionPool call" in { - createSecondaryIndex() + initializeUserSearchIndex(redisPooledConnection) val info = redisPooledConnection.ftInfo(UsersIndex) assert(info.containsValue(UsersIndex)) } diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala index 0188573..ad89454 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala @@ -1,21 +1,31 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import java.time.Instant - import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } -import com.ing.wbaa.rokku.sts.data.aws.{ AwsCredential, AwsSessionToken, AwsSessionTokenExpiration } +import com.ing.wbaa.rokku.sts.config.RedisSettings +import com.ing.wbaa.rokku.sts.config.StsSettings +import com.ing.wbaa.rokku.sts.data.UserAssumeRole +import com.ing.wbaa.rokku.sts.data.Username +import com.ing.wbaa.rokku.sts.data.aws.AwsCredential +import com.ing.wbaa.rokku.sts.data.aws.AwsSessionToken +import com.ing.wbaa.rokku.sts.data.aws.AwsSessionTokenExpiration import com.ing.wbaa.rokku.sts.service.TokenGeneration import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.RedisModel import org.scalatest.Assertion +import org.scalatest.BeforeAndAfterAll import org.scalatest.wordspec.AsyncWordSpec -import org.scalatest.{ BeforeAndAfterAll } + +import java.time.Instant import scala.concurrent.Future -import scala.util.Random import scala.jdk.CollectionConverters._ +import scala.util.Random -class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserDAO with Redis with TokenGeneration with BeforeAndAfterAll { +class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO + with STSUserDAO + with Redis + with RedisModel + with TokenGeneration + with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") @@ -26,11 +36,11 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO with STSUserDAO w override lazy val dbExecutionContext = executionContext override protected def beforeAll(): Unit = { - createSecondaryIndex() + initializeUserSearchIndex(redisPooledConnection) } override protected def afterAll(): Unit = { - List("users:*", "sessionTokens:*").foreach(pattern => { + List(s"${UserKeyPrefix}*", s"${SessionTokenKeyPrefix}*").foreach(pattern => { val keys = redisPooledConnection.keys(pattern) keys.asScala.foreach(key => { redisPooledConnection.del(key) diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala index f465148..5b2e70e 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala @@ -1,19 +1,27 @@ package com.ing.wbaa.rokku.sts.service.db.dao import akka.actor.ActorSystem -import com.ing.wbaa.rokku.sts.config.{ RedisSettings, StsSettings } -import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, UserGroup, Username } -import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential } +import com.ing.wbaa.rokku.sts.config.RedisSettings +import com.ing.wbaa.rokku.sts.config.StsSettings +import com.ing.wbaa.rokku.sts.data.AccountStatus +import com.ing.wbaa.rokku.sts.data.NPA +import com.ing.wbaa.rokku.sts.data.UserGroup +import com.ing.wbaa.rokku.sts.data.Username +import com.ing.wbaa.rokku.sts.data.aws.AwsAccessKey +import com.ing.wbaa.rokku.sts.data.aws.AwsCredential import com.ing.wbaa.rokku.sts.service.TokenGeneration +import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.RedisModel +import org.scalatest.BeforeAndAfterAll import org.scalatest.wordspec.AsyncWordSpec -import org.scalatest.{ BeforeAndAfterAll } + import scala.jdk.CollectionConverters._ import scala.util.Random -import com.ing.wbaa.rokku.sts.service.db.Redis class STSUserDAOItTest extends AsyncWordSpec with STSUserDAO with Redis + with RedisModel with TokenGeneration with BeforeAndAfterAll { val system: ActorSystem = ActorSystem.create("test-system") @@ -25,11 +33,11 @@ class STSUserDAOItTest extends AsyncWordSpec override lazy val dbExecutionContext = executionContext override protected def beforeAll(): Unit = { - createSecondaryIndex() + initializeUserSearchIndex(redisPooledConnection) } override protected def afterAll(): Unit = { - val keys = redisPooledConnection.keys("users:*") + val keys = redisPooledConnection.keys(s"${UserKeyPrefix}*") keys.asScala.foreach(key => { redisPooledConnection.del(key) }) diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala index 5894fdc..7269668 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/Server.scala @@ -4,12 +4,12 @@ import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config._ import com.ing.wbaa.rokku.sts.keycloak.{ KeycloakClient, KeycloakTokenVerifier } import com.ing.wbaa.rokku.sts.service.{ UserTokenDbService } -import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.{ Redis, RedisModel } import com.ing.wbaa.rokku.sts.service.db.dao.{ STSTokenDAO, STSUserDAO } import com.ing.wbaa.rokku.sts.vault.VaultService object Server extends App { - new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with STSTokenDAO with Redis with VaultService with KeycloakClient { + new RokkuStsService with KeycloakTokenVerifier with UserTokenDbService with STSUserDAO with STSTokenDAO with Redis with RedisModel with VaultService with KeycloakClient { override implicit lazy val system: ActorSystem = ActorSystem.create("rokku-sts") override protected[this] def httpSettings: HttpSettings = HttpSettings(system) @@ -23,6 +23,6 @@ object Server extends App { override protected[this] def redisSettings: RedisSettings = RedisSettings(system) //Connects to Redis on startup and initializes indeces - createSecondaryIndex() + initializeUserSearchIndex(redisPooledConnection) }.startup } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala index 4a6ac8e..fec65c3 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/Redis.scala @@ -3,11 +3,14 @@ package com.ing.wbaa.rokku.sts.service.db import akka.actor.ActorSystem import com.ing.wbaa.rokku.sts.config.RedisSettings import com.typesafe.scalalogging.LazyLogging -import redis.clients.jedis.{ JedisPooled, Jedis } -import redis.clients.jedis.exceptions.JedisDataException -import redis.clients.jedis.search.{ Schema, IndexDefinition, IndexOptions } -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Success, Try } +import redis.clients.jedis.Jedis +import redis.clients.jedis.JedisPooled + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.util.Failure +import scala.util.Success +import scala.util.Try trait Redis extends LazyLogging { @@ -27,10 +30,6 @@ trait Redis extends LazyLogging { system.dispatcher } - private val DuplicateKeyExceptionMsg = "Index already exists" - - protected val UsersIndex = "users-idx" - protected lazy val redisPooledConnection: JedisPooled = new JedisPooled( redisSettings.host, redisSettings.port, @@ -38,35 +37,6 @@ trait Redis extends LazyLogging { redisSettings.password, ) - /** - * Create secondary search index for users fields - */ - protected[this] def createSecondaryIndex(): Unit = { - val schema = new Schema() - .addTagField("accessKey") - .addTagField("isNPA") - - val prefixDefinition = new IndexDefinition() - .setPrefixes("users:") - - try { - redisPooledConnection.ftCreate( - UsersIndex, - IndexOptions.defaultOptions().setDefinition(prefixDefinition), schema) - logger.info(s"Created index ${UsersIndex}") - } catch { - case exc: JedisDataException => - exc.getMessage() match { - case DuplicateKeyExceptionMsg => - logger.info(s"Index ${UsersIndex} already exists. Continuing...") - case _ => - logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") - } - case exc: Exception => - logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") - } - } - protected[this] def withRedisPool[T]( databaseOperation: JedisPooled => Future[T] ): Future[T] = { diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/RedisModel.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/RedisModel.scala new file mode 100644 index 0000000..41b4628 --- /dev/null +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/RedisModel.scala @@ -0,0 +1,99 @@ +package com.ing.wbaa.rokku.sts.service.db + +import com.ing.wbaa.rokku.sts.data.UserGroup +import com.ing.wbaa.rokku.sts.data.Username +import com.ing.wbaa.rokku.sts.data.aws.AwsSessionToken +import com.ing.wbaa.rokku.sts.service.db.security.Encryption +import redis.clients.jedis.JedisPooled +import redis.clients.jedis.exceptions.JedisDataException +import redis.clients.jedis.search.IndexDefinition +import redis.clients.jedis.search.IndexOptions +import redis.clients.jedis.search.Schema + +trait RedisModel extends Encryption { + protected val UsersIndex = "users-idx" + + protected val SessionTokenKeyPrefix = "sessionTokens:" + + protected val UserKeyPrefix = "users:" + + private val GroupnameSeparator = "," + + private val DuplicateIndexExceptionMsg = "Index already exists" + + object UserKey { + def encode(username: Username): String = { + s"$UserKeyPrefix${username.value}" + } + + def decode(key: String): Username = { + Username(key.replace(UserKeyPrefix, "")) + } + } + + object SessionTokenKey { + def apply(sessionToken: AwsSessionToken, username: Username): String = { + s"$SessionTokenKeyPrefix${encryptSecret(sessionToken.value, username.value)}" + } + } + + object UserGroups { + def encode(groups: Set[UserGroup]): String = { + groups.mkString(GroupnameSeparator) + } + + def decode(groupsAsString: String): Set[UserGroup] = { + groupsAsString.split(GroupnameSeparator) + .filter(_.trim.nonEmpty) + .map(g => UserGroup(g.trim)).toSet[UserGroup] + } + } + + object UserFields extends Enumeration { + type UserFields = String + + val accessKey = "accessKey" + val secretKey = "secretKey" + val isNPA = "isNPA" + val isEnabled = "isEnabled" + val groups = "groups" + } + + object SessionTokenFields extends Enumeration { + type SessionTokenFields = Value + + val username = "username" + val assumeRole = "assumeRole" + val expirationTime = "expirationTime" + } + + /** + * Create secondary search index for users fields + */ + protected[this] def initializeUserSearchIndex(redisPooledConnection: JedisPooled): Unit = { + val schema = new Schema() + .addTagField(UserFields.accessKey) + .addTagField(UserFields.isNPA) + + val prefixDefinition = new IndexDefinition() + .setPrefixes(UserKeyPrefix) + + try { + redisPooledConnection.ftCreate( + UsersIndex, + IndexOptions.defaultOptions().setDefinition(prefixDefinition), schema) + logger.info(s"Created index ${UsersIndex}") + } catch { + case exc: JedisDataException => + exc.getMessage() match { + case DuplicateIndexExceptionMsg => + logger.info(s"Index ${UsersIndex} already exists. Continuing...") + case _ => + logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") + } + case exc: Exception => + logger.error(s"Unable to create index $UsersIndex. Error: ${exc.getMessage()}") + } + } + +} diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala index 398811a..c7bd5ab 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala @@ -1,22 +1,24 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import com.ing.wbaa.rokku.sts.data.{ UserAssumeRole, Username } -import com.ing.wbaa.rokku.sts.data.aws.{ AwsSessionToken, AwsSessionTokenExpiration } +import com.ing.wbaa.rokku.sts.data.UserAssumeRole +import com.ing.wbaa.rokku.sts.data.Username +import com.ing.wbaa.rokku.sts.data.aws.AwsSessionToken +import com.ing.wbaa.rokku.sts.data.aws.AwsSessionTokenExpiration +import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.RedisModel import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging -import redis.clients.jedis.{ Jedis } -import com.ing.wbaa.rokku.sts.service.db.Redis +import redis.clients.jedis.Jedis + import java.time.Instant +import scala.concurrent.ExecutionContext +import scala.concurrent.Future import scala.jdk.CollectionConverters._ -import scala.concurrent.{ ExecutionContext, Future } - -trait STSTokenDAO extends LazyLogging with Encryption with Redis { +trait STSTokenDAO extends LazyLogging with Encryption with Redis with RedisModel { protected[this] implicit def dbExecutionContext: ExecutionContext - private val SessionTokensKeyPrefix = "sessionTokens:" - /** * Get Token from database against the token session identifier * @@ -30,11 +32,11 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis { { Future { val values = client - .hgetAll(s"$SessionTokensKeyPrefix${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}") + .hgetAll(SessionTokenKey(awsSessionToken, username)) if (values.size() > 0) { - val assumeRole = getAssumeRole(values.get("assumeRole")) - val expirationDate = AwsSessionTokenExpiration(Instant.parse(values.get("expirationTime"))) + val assumeRole = getAssumeRole(values.get(SessionTokenFields.assumeRole)) + val expirationDate = AwsSessionTokenExpiration(Instant.parse(values.get(SessionTokenFields.expirationTime))) logger.debug("getToken {} expire {}", awsSessionToken, expirationDate) Some((username, assumeRole, expirationDate)) } else None @@ -68,17 +70,17 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis { { Future { val connection = client.getPool().getResource() - val key = s"$SessionTokensKeyPrefix${encryptSecret(awsSessionToken.value.trim(), username.value.trim())}" + val key = SessionTokenKey(awsSessionToken, username) if (!client.exists(key)) { - val trx = new Jedis(connection).multi() - trx.hset(key, Map( - "username" -> username.value, - "assumeRole" -> role.value, - "expirationTime" -> expirationDate.value.toString(), + val tx = new Jedis(connection).multi() + tx.hset(key, Map( + SessionTokenFields.username -> username.value, + SessionTokenFields.assumeRole -> role.value, + SessionTokenFields.expirationTime -> expirationDate.value.toString(), ).asJava) - trx.expireAt(key, expirationDate.value.getEpochSecond()) - trx.exec() + tx.expireAt(key, expirationDate.value.getEpochSecond()) + tx.exec() connection.close() true } else false diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala index ebad128..f77097d 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala @@ -1,23 +1,31 @@ package com.ing.wbaa.rokku.sts.service.db.dao -import com.ing.wbaa.rokku.sts.data.aws.{ AwsAccessKey, AwsCredential, AwsSecretKey } -import com.ing.wbaa.rokku.sts.data.{ AccountStatus, NPA, NPAAccount, NPAAccountList, UserGroup, Username } -import com.ing.wbaa.rokku.sts.service.db.security.Encryption +import com.ing.wbaa.rokku.sts.data.AccountStatus +import com.ing.wbaa.rokku.sts.data.NPA +import com.ing.wbaa.rokku.sts.data.NPAAccount +import com.ing.wbaa.rokku.sts.data.NPAAccountList +import com.ing.wbaa.rokku.sts.data.UserGroup +import com.ing.wbaa.rokku.sts.data.Username +import com.ing.wbaa.rokku.sts.data.aws.AwsAccessKey +import com.ing.wbaa.rokku.sts.data.aws.AwsCredential +import com.ing.wbaa.rokku.sts.data.aws.AwsSecretKey import com.ing.wbaa.rokku.sts.service.db.Redis +import com.ing.wbaa.rokku.sts.service.db.RedisModel +import com.ing.wbaa.rokku.sts.service.db.security.Encryption import com.typesafe.scalalogging.LazyLogging -import redis.clients.jedis.search.{ Query } -import scala.jdk.CollectionConverters._ +import redis.clients.jedis.search.Query -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Success, Try } +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.jdk.CollectionConverters._ +import scala.util.Failure +import scala.util.Success +import scala.util.Try -trait STSUserDAO extends LazyLogging with Encryption with Redis { +trait STSUserDAO extends LazyLogging with Encryption with Redis with RedisModel { protected[this] implicit def dbExecutionContext: ExecutionContext - private val UsersKeyPrefix = "users:" - private val GroupnameSeparator = "," - /** * Retrieves AWS user credentials based on the username * @@ -30,12 +38,12 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { Future { Try { val values = client - .hgetAll(s"$UsersKeyPrefix${username.value}") + .hgetAll(UserKey.encode(username)) if (values.size() > 0) { - val accessKey = AwsAccessKey(values.get("accessKey")) - val secretKey = AwsSecretKey(decryptSecret(values.get("secretKey").trim(), username.value.trim())) - val isEnabled = values.get("isEnabled").toBooleanOption.getOrElse(false) + val accessKey = AwsAccessKey(values.get(UserFields.accessKey)) + val secretKey = AwsSecretKey(decryptSecret(values.get(UserFields.secretKey).trim(), username.value.trim())) + val isEnabled = values.get(UserFields.isEnabled).toBooleanOption.getOrElse(false) (Some(AwsCredential(accessKey, secretKey)), AccountStatus(isEnabled)) } else (None, AccountStatus(false)) @@ -60,18 +68,15 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { client => { Future { - val query = new Query(s"@accessKey:{${awsAccessKey.value}}") + val query = new Query(s"@${UserFields.accessKey}:{${awsAccessKey.value}}") val results = client.ftSearch(UsersIndex, query); if (results.getDocuments().size == 1) { val document = results.getDocuments().get(0) - val username = Username(document.getId().replace(UsersKeyPrefix, "")) - val secretKey = AwsSecretKey(decryptSecret(document.getString("secretKey").trim(), username.value.trim())) - val isEnabled = Try(document.getString("isEnabled").toBoolean).getOrElse(false) - val isNPA = Try(document.getString("isNPA").toBoolean).getOrElse(false) - val groups = document.getString("groups") - .split(GroupnameSeparator) - .filter(_.trim.nonEmpty) - .map(g => UserGroup(g.trim)).toSet[UserGroup] + val username = UserKey.decode(document.getId()) + val secretKey = AwsSecretKey(decryptSecret(document.getString(UserFields.secretKey).trim(), username.value.trim())) + val isEnabled = Try(document.getString(UserFields.isEnabled).toBoolean).getOrElse(false) + val isNPA = Try(document.getString(UserFields.isNPA).toBoolean).getOrElse(false) + val groups = UserGroups.decode(document.getString(UserFields.groups)) Some((username, secretKey, NPA(isNPA), AccountStatus(isEnabled), groups)) } else None @@ -93,12 +98,12 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { { doesUsernameNotExistAndAccessKeyNotExist(username, awsCredential.accessKey).map { case true => - client.hset(s"$UsersKeyPrefix${username.value}", Map( - "accessKey" -> awsCredential.accessKey.value, - "secretKey" -> encryptSecret(awsCredential.secretKey.value.trim(), username.value.trim()), - "isNPA" -> isNPA.toString(), - "isEnabled" -> "true", - "groups" -> "", + client.hset(UserKey.encode(username), Map( + UserFields.accessKey -> awsCredential.accessKey.value, + UserFields.secretKey -> encryptSecret(awsCredential.secretKey.value.trim(), username.value.trim()), + UserFields.isNPA -> isNPA.toString(), + UserFields.isEnabled -> "true", + UserFields.groups -> "", ).asJava) true case false => false @@ -118,7 +123,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { client => { Future { - client.hset(s"users:${username.value}", "groups", userGroups.mkString(GroupnameSeparator)) + client.hset(UserKey.encode(username), UserFields.groups, UserGroups.encode(userGroups)) true } } @@ -137,7 +142,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { { Future { Try { - client.hset(s"users:${username.value}", "isEnabled", enabled.toString()) + client.hset(UserKey.encode(username), UserFields.isEnabled, enabled.toString()) true } match { case Success(r) => r @@ -168,15 +173,14 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { withRedisPool { client => Future { - val query = new Query("@isNPA:{true}") - // @TODO HANDLE ERRORS + // val query = new Query(s"@isNpa:{true}") + val query = new Query(s"@${UserFields.isNPA}:{true}") val results = client.ftSearch(UsersIndex, query) - val npaAccounts = results.getDocuments().asScala .map(doc => { NPAAccount( - doc.getId().replace("users:", ""), - Try(doc.getString("isEnabled").toBoolean).getOrElse(false) + UserKey.decode(doc.getId()).value, + Try(doc.getString(UserFields.isEnabled).toBoolean).getOrElse(false) ) }) @@ -197,7 +201,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { client => { Future { - client.exists(s"users:${username.value}") + client.exists(UserKey.encode(username)) } } } @@ -206,7 +210,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis { withRedisPool { client => { Future { - val query = new Query(s"@accessKey:{${awsAccessKey.value}}") + val query = new Query(s"@${UserFields.accessKey}:{${awsAccessKey.value}}") // @TODO HANDLE ERRORS val results = client.ftSearch(UsersIndex, query) val accessKeyExists = results.getTotalResults() != 0 From b298a05b9fccf1f4865efa294ce29f13a25d4f6f Mon Sep 17 00:00:00 2001 From: yannis Date: Mon, 29 Aug 2022 14:11:01 +0200 Subject: [PATCH 06/18] Impoved some log messages --- build.sbt | 1 - scripts/rokku-assume-role.sh | 0 scripts/rokku-get-session-token.sh | 0 .../wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala | 8 ++++---- .../ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala | 4 ++-- .../ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala | 5 ++--- 6 files changed, 8 insertions(+), 10 deletions(-) mode change 100644 => 100755 scripts/rokku-assume-role.sh mode change 100644 => 100755 scripts/rokku-get-session-token.sh diff --git a/build.sbt b/build.sbt index 688a731..9eb4af5 100644 --- a/build.sbt +++ b/build.sbt @@ -42,7 +42,6 @@ libraryDependencies ++= Seq( "org.keycloak" % "keycloak-admin-client" % keycloakVersion, "org.jboss.logging" % "jboss-logging" % "3.5.0.Final", "org.apache.httpcomponents" % "httpclient" % "4.5.13", - "org.mariadb.jdbc" % "mariadb-java-client" % "2.3.0", "ch.qos.logback.contrib" % "logback-json-classic" % logbackJson, "ch.qos.logback.contrib" % "logback-jackson" % logbackJson, "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.3", diff --git a/scripts/rokku-assume-role.sh b/scripts/rokku-assume-role.sh old mode 100644 new mode 100755 diff --git a/scripts/rokku-get-session-token.sh b/scripts/rokku-get-session-token.sh old mode 100644 new mode 100755 diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala index ad89454..b5fba0e 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAOItTest.scala @@ -88,12 +88,12 @@ class STSTokenDAOItTest extends AsyncWordSpec with STSTokenDAO } "insert Token" that { - "that expires after 1 millisecond" in { + "that expires immediately" in { val testObject = new TestObject - insertToken(testObject.testAwsSessionToken, Username("boom"), testObject.assumeRole, - AwsSessionTokenExpiration(Instant.now().plusMillis(1))).flatMap { inserted => + insertToken(testObject.testAwsSessionToken, testObject.username, testObject.assumeRole, + AwsSessionTokenExpiration(Instant.now().plusMillis(0))).flatMap { inserted => assert(inserted) - getToken(testObject.testAwsSessionToken, Username("boom")).map { o => + getToken(testObject.testAwsSessionToken, testObject.username).map { o => assert(o.isEmpty) } } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala index c7bd5ab..1773ff1 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSTokenDAO.scala @@ -34,10 +34,10 @@ trait STSTokenDAO extends LazyLogging with Encryption with Redis with RedisModel val values = client .hgetAll(SessionTokenKey(awsSessionToken, username)) - if (values.size() > 0) { + if (values.size() > 1) { val assumeRole = getAssumeRole(values.get(SessionTokenFields.assumeRole)) val expirationDate = AwsSessionTokenExpiration(Instant.parse(values.get(SessionTokenFields.expirationTime))) - logger.debug("getToken {} expire {}", awsSessionToken, expirationDate) + logger.debug(s"getToken(${awsSessionToken.value}, ${username.value}) returned fields assumeRole:$assumeRole, expirationDate: $expirationDate") Some((username, assumeRole, expirationDate)) } else None } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala index f77097d..a480e1c 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala @@ -50,7 +50,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis with RedisModel } match { case Success(r) => r case Failure(ex) => - logger.error("Cannot find user credentials for ({}), {} ", username, ex.getMessage) + logger.error(s"getAwsCredentialAndStatus(${username.value} failed: ${ex.getMessage}") throw ex } } @@ -147,7 +147,7 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis with RedisModel } match { case Success(r) => r case Failure(ex) => - logger.error("Cannot enable or disable user account {} reason {}", username.value, ex) + logger.error(s"setAccountStatus(${username.value}) failed: ${ex.getMessage}") throw ex } } @@ -211,7 +211,6 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis with RedisModel { Future { val query = new Query(s"@${UserFields.accessKey}:{${awsAccessKey.value}}") - // @TODO HANDLE ERRORS val results = client.ftSearch(UsersIndex, query) val accessKeyExists = results.getTotalResults() != 0 accessKeyExists From fee064ce056efb052e3bc8bd02655bd5509a163f Mon Sep 17 00:00:00 2001 From: yannis Date: Tue, 30 Aug 2022 02:33:39 +0200 Subject: [PATCH 07/18] Renamed insertUserGroups func to setUserGroups --- .../wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala | 6 +++--- .../com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala | 6 +++--- .../com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala | 4 ++-- .../ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala index 5b2e70e..e608c48 100644 --- a/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala +++ b/src/it/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAOItTest.scala @@ -182,16 +182,16 @@ class STSUserDAOItTest extends AsyncWordSpec "user has two groups then one and then zero" in { val testObject = new TestObject insertAwsCredentials(testObject.userName, testObject.cred, isNPA = false).flatMap { _ => - insertUserGroups(testObject.userName, testObject.userGroups).flatMap { _ => + setUserGroups(testObject.userName, testObject.userGroups).flatMap { _ => { getUserSecretWithExtInfo(testObject.cred.accessKey) .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), testObject.userGroups)))) - insertUserGroups(testObject.userName, Set(testObject.userGroups.head)).flatMap { _ => + setUserGroups(testObject.userName, Set(testObject.userGroups.head)).flatMap { _ => getUserSecretWithExtInfo(testObject.cred.accessKey) .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set(testObject.userGroups.head))))) - insertUserGroups(testObject.userName, Set.empty[UserGroup]).flatMap { _ => + setUserGroups(testObject.userName, Set.empty[UserGroup]).flatMap { _ => getUserSecretWithExtInfo(testObject.cred.accessKey) .map(c => assert(c.contains((testObject.userName, testObject.cred.secretKey, NPA(false), AccountStatus(true), Set.empty[UserGroup])))) } diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala index 0a3ccd9..b5ff652 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbService.scala @@ -27,7 +27,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: Username, awsAccessKey: AwsAccessKey): Future[Boolean] - protected[this] def insertUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] + protected[this] def setUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] /** * Retrieve or generate Credentials and generate a new Session @@ -41,7 +41,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { for { (awsCredential, AccountStatus(isEnabled)) <- getOrGenerateAwsCredentialWithStatus(userName) awsSession <- getNewAwsSession(userName, duration) - _ <- insertUserGroups(userName, userGroups) + _ <- setUserGroups(userName, userGroups) if isEnabled } yield AwsCredentialWithToken( awsCredential, @@ -61,7 +61,7 @@ trait UserTokenDbService extends LazyLogging with TokenGeneration { for { (awsCredential, AccountStatus(isEnabled)) <- getOrGenerateAwsCredentialWithStatus(userName) awsSession <- getNewAwsSessionWithToken(userName, role, duration) - _ <- insertUserGroups(userName, userGroups) + _ <- setUserGroups(userName, userGroups) if isEnabled } yield AwsCredentialWithToken( awsCredential, diff --git a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala index a480e1c..ff19d0c 100644 --- a/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala +++ b/src/main/scala/com/ing/wbaa/rokku/sts/service/db/dao/STSUserDAO.scala @@ -113,12 +113,12 @@ trait STSUserDAO extends LazyLogging with Encryption with Redis with RedisModel } /** - * Removes all user groups and inserts the new on from userGroup + * Sets the user groups for the target user * @param userName * @param userGroups * @return true if succeeded */ - def insertUserGroups(username: Username, userGroups: Set[UserGroup]): Future[Boolean] = + def setUserGroups(username: Username, userGroups: Set[UserGroup]): Future[Boolean] = withRedisPool[Boolean] { client => { diff --git a/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala b/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala index 33ee0d6..be21f61 100644 --- a/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala +++ b/src/test/scala/com/ing/wbaa/rokku/sts/service/UserTokenDbServiceTest.scala @@ -42,7 +42,7 @@ class UserTokenDbServiceTest extends AsyncWordSpec with Diagrams { override protected[this] def doesUsernameNotExistAndAccessKeyExist(userName: Username, awsAccessKey: AwsAccessKey): Future[Boolean] = Future.successful(false) - override protected[this] def insertUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] = + override protected[this] def setUserGroups(userName: Username, userGroups: Set[UserGroup]): Future[Boolean] = Future.successful(true) } From 4b3017042fba3e7f09c5c7c9bd55ac59edbe6331 Mon Sep 17 00:00:00 2001 From: yannis Date: Tue, 30 Aug 2022 02:42:19 +0200 Subject: [PATCH 08/18] Updated README.md --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index d63983c..44bd5ac 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,7 @@ To get a quickstart on running the Rokku STS, you'll need the following: The STS service is dependant on two services: * [Keycloak](https://www.keycloak.org/) for MFA authentication of users. -* A persistence store to maintain the user and session tokens issued, in the current infrastructure that is [MariaDB](https://mariadb.org). - -For the persistence, Rokku STS does not autogenerate the tables required. So if you launch your own MariaDB database, -you will need to create the tables as well. You can find the script to create the database, and the related tables -[here](https://github.com/ing-bank/rokku-dev-mariadb/blob/master/database/rokkudb.sql). +* [Redis] A persistence store to maintain the user and session tokens issued ## Test (mock version) From 3ee414f20aeb010964922671055cd3a2eff41fda Mon Sep 17 00:00:00 2001 From: yannis Date: Wed, 31 Aug 2022 12:36:41 +0200 Subject: [PATCH 09/18] Added github test workflow --- .github/workflows/test.yaml | 21 +++++++++++++++++++ .../waitForContainerSetup.sh | 14 +++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test.yaml rename waitForContainerSetup.sh => scripts/waitForContainerSetup.sh (61%) mode change 100644 => 100755 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..b3d3a2a --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,21 @@ +name: Rokku-STS tests + +on: [ push ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'sbt' + - name: Run docker-compose + run: docker-compose up -d && ./scripts/waitForContainerSetup.sh + - name: Run tests + run: sbt clean coverage test it:test coverageReport \ No newline at end of file diff --git a/waitForContainerSetup.sh b/scripts/waitForContainerSetup.sh old mode 100644 new mode 100755 similarity index 61% rename from waitForContainerSetup.sh rename to scripts/waitForContainerSetup.sh index 2da80fc..27b2fbb --- a/waitForContainerSetup.sh +++ b/scripts/waitForContainerSetup.sh @@ -6,10 +6,18 @@ set -e # Max query attempts before consider setup failed MAX_TRIES=90 -function rokkuKeycloak() { +function keycloak() { docker-compose logs keycloak | grep "Admin console listening" } +function redis() { + docker-compose logs redis | grep "Ready to accept connections" +} + +function vault() { + docker-compose logs vault | grep "upgrading keys finished" +} + function waitUntilServiceIsReady() { attempt=1 while [ $attempt -le $MAX_TRIES ]; do @@ -27,4 +35,6 @@ function waitUntilServiceIsReady() { fi } -waitUntilServiceIsReady rokkuKeycloak "Keycloack ready" +waitUntilServiceIsReady redis "Redis is ready" +waitUntilServiceIsReady vault "Vault is ready" +waitUntilServiceIsReady keycloak "Keycloack is ready" From 655e6fd1cb3a5594d470ccaa32ac83df027eef27 Mon Sep 17 00:00:00 2001 From: yannis Date: Wed, 31 Aug 2022 13:09:53 +0200 Subject: [PATCH 10/18] Disabled 'byname-implicit' linting rule --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9eb4af5..d5bc301 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ scalacOptions := Seq( "-encoding", "utf-8", "-target:11", "-feature", - "-Xlint", + "-Xlint:-byname-implicit", "-Xfatal-warnings", ) From 5cf95b230c20a88c4f7790904e555663908c1878 Mon Sep 17 00:00:00 2001 From: yannis Date: Wed, 31 Aug 2022 15:08:09 +0200 Subject: [PATCH 11/18] Added docker build step to workflow --- .github/workflows/build.yaml | 43 ++++++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 21 ------------------ build.sbt | 2 +- 3 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..85fd4ec --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,43 @@ +name: Rokku-STS build + +on: [ push ] + +env: + DOCKER_REPO: wbaa/rokku-sts + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'sbt' + - name: Set version + run: | + ROKKU_STS_VERSION="${GITHUB_REF##*/}" + echo "ROKKU_STS_VERSION=${ROKKU_STS_VERSION}" >> $GITHUB_ENV + + - name: Run docker-compose + run: docker-compose up -d && ./scripts/waitForContainerSetup.sh + + - name: Run tests + run: sbt clean coverage test it:test coverageReport + + - name: Build and publish docker image + run: | + if [ "$ROKKU_STS_VERSION" = "master" ]; then + echo $(git describe --tags --abbrev=0) + fi + + # Login to docker + echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + # Build docker image + echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; + sbt clean docker:publish; + diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index b3d3a2a..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Rokku-STS tests - -on: [ push ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: 'sbt' - - name: Run docker-compose - run: docker-compose up -d && ./scripts/waitForContainerSetup.sh - - name: Run tests - run: sbt clean coverage test it:test coverageReport \ No newline at end of file diff --git a/build.sbt b/build.sbt index d5bc301..bbfac27 100644 --- a/build.sbt +++ b/build.sbt @@ -68,7 +68,7 @@ fork := true dockerExposedPorts := Seq(12345) dockerCommands += ExecCmd("ENV", "PROXY_HOST", "0.0.0.0") -dockerBaseImage := "openjdk:8u171-jre-slim-buster" +dockerBaseImage := "openjdk:11-slim-buster" dockerAlias := docker.DockerAlias(Some("docker.io"), Some("wbaa"), "rokku-sts", Some(rokkuStsVersion)) scalariformPreferences := scalariformPreferences.value From 0c9776418feb824d8fffd327dca8b60666a430b6 Mon Sep 17 00:00:00 2001 From: yannis Date: Thu, 1 Sep 2022 15:30:03 +0200 Subject: [PATCH 12/18] Added release workflow --- .../{build.yaml => build-feature.yaml} | 32 +++++------ .github/workflows/build-master.yaml | 57 +++++++++++++++++++ .github/workflows/release.yaml | 17 ++++++ 3 files changed, 89 insertions(+), 17 deletions(-) rename .github/workflows/{build.yaml => build-feature.yaml} (69%) create mode 100644 .github/workflows/build-master.yaml create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build-feature.yaml similarity index 69% rename from .github/workflows/build.yaml rename to .github/workflows/build-feature.yaml index 85fd4ec..20ee7c8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build-feature.yaml @@ -1,15 +1,16 @@ -name: Rokku-STS build +name: Rokku-STS build and publish -on: [ push ] +on: + push: + branches-ignore: + - master env: DOCKER_REPO: wbaa/rokku-sts jobs: - build: - + test: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 - name: Set up JDK 11 @@ -18,26 +19,23 @@ jobs: java-version: '11' distribution: 'temurin' cache: 'sbt' - - name: Set version - run: | - ROKKU_STS_VERSION="${GITHUB_REF##*/}" - echo "ROKKU_STS_VERSION=${ROKKU_STS_VERSION}" >> $GITHUB_ENV - - name: Run docker-compose run: docker-compose up -d && ./scripts/waitForContainerSetup.sh - - name: Run tests run: sbt clean coverage test it:test coverageReport - + + upload-image: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' - name: Build and publish docker image run: | - if [ "$ROKKU_STS_VERSION" = "master" ]; then - echo $(git describe --tags --abbrev=0) - fi - # Login to docker echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin # Build docker image + ROKKU_STS_VERSION=${GITHUB_REF##*/} echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; sbt clean docker:publish; - diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml new file mode 100644 index 0000000..bdca1b4 --- /dev/null +++ b/.github/workflows/build-master.yaml @@ -0,0 +1,57 @@ +name: Rokku-STS build and publish + +on: + push: + branches: + - master + +env: + DOCKER_REPO: wbaa/rokku-sts + +jobs: + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'sbt' + - name: Run docker-compose + run: docker-compose up -d && ./scripts/waitForContainerSetup.sh + - name: Run tests + run: sbt clean coverage test it:test coverageReport + + tag: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' + - name: Bump version and push tag + uses: anothrNick/github-tag-action@1.36.0 + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + WITH_V: true + VERBOSE: true + DEFAULT_BUMP: patch + + upload-image: + runs-on: ubuntu-latest + needs: tag + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' + - name: Build and publish docker image + run: | + # Login to docker + echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + # Build docker image + ROKKU_STS_VERSION=$(git describe --tags --abbrev=0) + echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; + sbt clean docker:publish; diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..af4bbc7 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,17 @@ +name: Release + +on: + push: + tags: + - "*" + +jobs: + tagged-release: + name: "Tagged Release" + runs-on: "ubuntu-latest" + + steps: + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.PAT }}" + prerelease: false From 336fa2d6d5cf85e053fca738e01d28868ad7f56a Mon Sep 17 00:00:00 2001 From: yannis Date: Thu, 1 Sep 2022 16:47:01 +0200 Subject: [PATCH 13/18] Cleaned up workflow files --- .github/workflows/build-feature.yaml | 1 + .github/workflows/build-master.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build-feature.yaml b/.github/workflows/build-feature.yaml index 20ee7c8..a351c4f 100644 --- a/.github/workflows/build-feature.yaml +++ b/.github/workflows/build-feature.yaml @@ -35,6 +35,7 @@ jobs: run: | # Login to docker echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + # Build docker image ROKKU_STS_VERSION=${GITHUB_REF##*/} echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index bdca1b4..ba321c2 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -51,6 +51,7 @@ jobs: run: | # Login to docker echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + # Build docker image ROKKU_STS_VERSION=$(git describe --tags --abbrev=0) echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; From f3808ec16e7384205f1dccdbd9bfd634b87dafec Mon Sep 17 00:00:00 2001 From: yannis Date: Thu, 1 Sep 2022 16:57:51 +0200 Subject: [PATCH 14/18] Added 'latest' tag to docker images that are build from master branch --- .github/workflows/build-master.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index ba321c2..a41fd43 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -56,3 +56,6 @@ jobs: ROKKU_STS_VERSION=$(git describe --tags --abbrev=0) echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; sbt clean docker:publish; + + docker tag $DOCKER_REPO:$ROKKU_STS_VERSION $DOCKER_REPO:latest; + docker push $DOCKER_REPO:latest From 81a105dc266548da0dac594f688ad46469947f8c Mon Sep 17 00:00:00 2001 From: yannis Date: Thu, 1 Sep 2022 16:59:12 +0200 Subject: [PATCH 15/18] Adjusted workflow titles --- .github/workflows/build-feature.yaml | 2 +- .github/workflows/build-master.yaml | 3 ++- .github/workflows/release.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-feature.yaml b/.github/workflows/build-feature.yaml index a351c4f..a6e850a 100644 --- a/.github/workflows/build-feature.yaml +++ b/.github/workflows/build-feature.yaml @@ -1,4 +1,4 @@ -name: Rokku-STS build and publish +name: Rokku-STS feature branch build on: push: diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index a41fd43..d958096 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -1,4 +1,4 @@ -name: Rokku-STS build and publish +name: Rokku-STS master branch build on: push: @@ -57,5 +57,6 @@ jobs: echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; sbt clean docker:publish; + # Add latest tag docker tag $DOCKER_REPO:$ROKKU_STS_VERSION $DOCKER_REPO:latest; docker push $DOCKER_REPO:latest diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index af4bbc7..bfdf40a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -name: Release +name: Rokku-STS release on: push: From 4128127cdaaf54e6ccb0904602293a6dbdcb5830 Mon Sep 17 00:00:00 2001 From: yannis Date: Tue, 6 Sep 2022 11:59:53 +0200 Subject: [PATCH 16/18] Updated keycloak client version and docker base image --- .github/workflows/build-feature.yaml | 7 +++---- .github/workflows/build-master.yaml | 7 +++---- build.sbt | 5 +++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-feature.yaml b/.github/workflows/build-feature.yaml index a6e850a..aa6dd16 100644 --- a/.github/workflows/build-feature.yaml +++ b/.github/workflows/build-feature.yaml @@ -35,8 +35,7 @@ jobs: run: | # Login to docker echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - + # Build docker image - ROKKU_STS_VERSION=${GITHUB_REF##*/} - echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; - sbt clean docker:publish; + echo "Build image $DOCKER_REPO:${GITHUB_REF##*/}"; + ROKKU_STS_VERSION=${GITHUB_REF##*/} sbt clean docker:publish; diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index d958096..44a897f 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -53,10 +53,9 @@ jobs: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin # Build docker image - ROKKU_STS_VERSION=$(git describe --tags --abbrev=0) - echo "Build image for with name $DOCKER_REPO:$ROKKU_STS_VERSION"; - sbt clean docker:publish; + ROKKU_STS_VERSION=$(git describe --tags --abbrev=0) sbt clean docker:publish; + echo "Built image $DOCKER_REPO:$(git describe --tags --abbrev=0)"; # Add latest tag - docker tag $DOCKER_REPO:$ROKKU_STS_VERSION $DOCKER_REPO:latest; + docker tag $DOCKER_REPO:$(git describe --tags --abbrev=0) $DOCKER_REPO:latest; docker push $DOCKER_REPO:latest diff --git a/build.sbt b/build.sbt index bbfac27..47716cb 100644 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,7 @@ assemblyJarName in assembly := "rokku-sts.jar" val akkaVersion = "2.6.19" val akkaHttpVersion = "10.2.9" -val keycloakVersion = "16.1.1" +val keycloakVersion = "19.0.0" val logbackJson = "0.1.5" libraryDependencies ++= Seq( @@ -68,7 +68,8 @@ fork := true dockerExposedPorts := Seq(12345) dockerCommands += ExecCmd("ENV", "PROXY_HOST", "0.0.0.0") -dockerBaseImage := "openjdk:11-slim-buster" +dockerCommands += ExecCmd("RUN", "apt-get update && apt-get upgrade") +dockerBaseImage := "openjdk:11-slim-bullseye" dockerAlias := docker.DockerAlias(Some("docker.io"), Some("wbaa"), "rokku-sts", Some(rokkuStsVersion)) scalariformPreferences := scalariformPreferences.value From a051c3c94bdfb780539c9d312e747339418bd1cd Mon Sep 17 00:00:00 2001 From: yannis Date: Tue, 6 Sep 2022 14:49:26 +0200 Subject: [PATCH 17/18] Added extra docker command sto upgrade packages --- build.sbt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 47716cb..3288ef1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,5 @@ import com.typesafe.sbt.packager.docker -import com.typesafe.sbt.packager.docker.ExecCmd +import com.typesafe.sbt.packager.docker.Cmd import scalariform.formatter.preferences._ val rokkuStsVersion = scala.sys.env.getOrElse("ROKKU_STS_VERSION", "SNAPSHOT") @@ -67,8 +67,15 @@ enablePlugins(JavaAppPackaging) fork := true dockerExposedPorts := Seq(12345) -dockerCommands += ExecCmd("ENV", "PROXY_HOST", "0.0.0.0") -dockerCommands += ExecCmd("RUN", "apt-get update && apt-get upgrade") + +dockerCommands ++= Seq( + Cmd("ENV", "PROXY_HOST", "0.0.0.0"), + Cmd("USER", "root"), + Cmd("RUN", "apt-get update && apt-get upgrade -y"), + Cmd("USER", "1001"), +) + + dockerBaseImage := "openjdk:11-slim-bullseye" dockerAlias := docker.DockerAlias(Some("docker.io"), Some("wbaa"), "rokku-sts", Some(rokkuStsVersion)) From 6d9403b02ffe7e95a3795621dd1046e3fd21c1f3 Mon Sep 17 00:00:00 2001 From: yannis Date: Wed, 14 Sep 2022 15:27:00 +0200 Subject: [PATCH 18/18] Now pushes both latest and version docker image tag --- .github/workflows/build-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index 44a897f..1e83b5c 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -58,4 +58,5 @@ jobs: # Add latest tag docker tag $DOCKER_REPO:$(git describe --tags --abbrev=0) $DOCKER_REPO:latest; + docker push $DOCKER_REPO:$(git describe --tags --abbrev=0) docker push $DOCKER_REPO:latest