From aeb69745b6fe02773179ac0c1724709a6c61445e Mon Sep 17 00:00:00 2001 From: venkatvghub Date: Tue, 30 Mar 2021 20:11:53 +0530 Subject: [PATCH] [config] - Provides ability to read configs and secrets from environment variables --- .../datalayer/util/DaoProviderUtil.java | 8 +++-- .../pinot/PinotThirdEyeDataSourceConfig.java | 15 ++++---- .../sql/SqlResponseCacheLoader.java | 35 ++++++++++--------- .../thirdeye/util/CustomConfigReader.java | 26 ++++++++++++++ 4 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java diff --git a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java index 841b61bec6..c047650943 100644 --- a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java +++ b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java @@ -33,6 +33,7 @@ import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AbstractManagerImpl; import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO; import org.apache.pinot.thirdeye.datalayer.util.PersistenceConfig.DatabaseConfiguration; +import org.apache.pinot.thirdeye.util.CustomConfigReader; import org.apache.tomcat.jdbc.pool.DataSource; import org.h2.store.fs.FileUtils; import org.slf4j.Logger; @@ -81,12 +82,13 @@ private static void createSchemaIfReqd( private static DataSource createDataSource(final PersistenceConfig configuration) { final DataSource dataSource = new DataSource(); + CustomConfigReader ccr = new CustomConfigReader(); dataSource.setInitialSize(10); dataSource.setDefaultAutoCommit(false); dataSource.setMaxActive(100); - dataSource.setUsername(configuration.getDatabaseConfiguration().getUser()); - dataSource.setPassword(configuration.getDatabaseConfiguration().getPassword()); - dataSource.setUrl(configuration.getDatabaseConfiguration().getUrl()); + dataSource.setUsername(ccr.readEnv(configuration.getDatabaseConfiguration().getUser())); + dataSource.setPassword(ccr.readEnv(configuration.getDatabaseConfiguration().getPassword())); + dataSource.setUrl(ccr.readEnv(configuration.getDatabaseConfiguration().getUrl())); dataSource.setDriverClassName(configuration.getDatabaseConfiguration().getDriver()); dataSource.setValidationQuery("select 1"); diff --git a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java index 4c01d8cd93..ea07331027 100644 --- a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java +++ b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource; import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig; +import org.apache.pinot.thirdeye.util.CustomConfigReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -212,13 +213,15 @@ public PinotThirdEyeDataSourceConfig build() { "{} accepts only 'http' or 'https' connection schemes", className); PinotThirdEyeDataSourceConfig config = new PinotThirdEyeDataSourceConfig(); - config.setControllerHost(controllerHost); + // Read from the custom config reader for reading from environment variables + CustomConfigReader ccr = new CustomConfigReader(); + config.setControllerHost(ccr.readEnv(controllerHost)); config.setControllerPort(controllerPort); - config.setZookeeperUrl(zookeeperUrl); - config.setClusterName(clusterName); - config.setBrokerUrl(brokerUrl); - config.setTag(tag); - config.setControllerConnectionScheme(controllerConnectionScheme); + config.setZookeeperUrl(ccr.readEnv(zookeeperUrl)); + config.setClusterName(ccr.readEnv(clusterName)); + config.setBrokerUrl(ccr.readEnv(brokerUrl)); + config.setTag(ccr.readEnv(tag)); + config.setControllerConnectionScheme(ccr.readEnv(controllerConnectionScheme)); config.setName(name); return config; } diff --git a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java index 852e948600..6d95fe07d2 100644 --- a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java +++ b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java @@ -39,6 +39,7 @@ import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup; import org.apache.pinot.thirdeye.detection.ConfigUtils; import org.apache.pinot.thirdeye.util.ThirdEyeUtils; +import org.apache.pinot.thirdeye.util.CustomConfigReader; import org.apache.tomcat.jdbc.pool.DataSource; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -85,14 +86,14 @@ public class SqlResponseCacheLoader extends CacheLoader properties) throws Exception { - + CustomConfigReader ccr = new CustomConfigReader(); // Init Presto datasources if (properties.containsKey(PRESTO)) { List> prestoMapList = ConfigUtils.getList(properties.get(PRESTO)); for (Map objMap: prestoMapList) { Map dbNameToURLMap = (Map)objMap.get(DB); - String prestoUser = (String)objMap.get(USER); - String prestoPassword = getPassword(objMap); + String prestoUser = ccr.readEnv((String)objMap.get(USER)); + String prestoPassword = ccr.readEnv(getPassword(objMap)); for (Map.Entry entry: dbNameToURLMap.entrySet()) { DataSource dataSource = new DataSource(); @@ -100,7 +101,7 @@ public SqlResponseCacheLoader(Map properties) throws Exception { dataSource.setMaxActive(MAX_CONNECTIONS); dataSource.setUsername(prestoUser); dataSource.setPassword(prestoPassword); - dataSource.setUrl(entry.getValue()); + dataSource.setUrl(ccr.readEnv(entry.getValue())); // Timeout before an abandoned(in use) connection can be removed. dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT); @@ -117,8 +118,8 @@ public SqlResponseCacheLoader(Map properties) throws Exception { List> mysqlMapList = ConfigUtils.getList(properties.get(MYSQL)); for (Map objMap: mysqlMapList) { Map dbNameToURLMap = (Map)objMap.get(DB); - String mysqlUser = (String)objMap.get(USER); - String mysqlPassword = getPassword(objMap); + String mysqlUser = ccr.readEnv((String)objMap.get(USER)); + String mysqlPassword = ccr.readEnv(getPassword(objMap)); for (Map.Entry entry: dbNameToURLMap.entrySet()) { DataSource dataSource = new DataSource(); @@ -126,7 +127,7 @@ public SqlResponseCacheLoader(Map properties) throws Exception { dataSource.setMaxActive(MAX_CONNECTIONS); dataSource.setUsername(mysqlUser); dataSource.setPassword(mysqlPassword); - dataSource.setUrl(entry.getValue()); + dataSource.setUrl(ccr.readEnv(entry.getValue())); // Timeout before an abandoned(in use) connection can be removed. dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT); @@ -143,9 +144,9 @@ public SqlResponseCacheLoader(Map properties) throws Exception { List> verticaMapList = ConfigUtils.getList(properties.get(VERTICA)); for (Map objMap: verticaMapList) { Map dbNameToURLMap = (Map)objMap.get(DB); - String verticaUser = (String)objMap.get(USER); - String verticaPassword = getPassword(objMap); - String verticaDriver = (String)objMap.get(DRIVER); + String verticaUser = ccr.readEnv((String)objMap.get(USER)); + String verticaPassword = ccr.readEnv(getPassword(objMap)); + String verticaDriver = ccr.readEnv((String)objMap.get(DRIVER)); for (Map.Entry entry: dbNameToURLMap.entrySet()) { DataSource dataSource = new DataSource(); @@ -154,7 +155,7 @@ public SqlResponseCacheLoader(Map properties) throws Exception { dataSource.setUsername(verticaUser); dataSource.setPassword(verticaPassword); dataSource.setDriverClassName(verticaDriver); - dataSource.setUrl(entry.getValue()); + dataSource.setUrl(ccr.readEnv(entry.getValue())); // Timeout before an abandoned(in use) connection can be removed. dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT); @@ -170,16 +171,16 @@ public SqlResponseCacheLoader(Map properties) throws Exception { if (properties.containsKey(BIGQUERY)) { List> bigQueryMapList = ConfigUtils.getList(properties.get(BIGQUERY)); for (Map objMap: bigQueryMapList) { - System.out.println(bigQueryMapList.toString()); + // System.out.println(bigQueryMapList.toString()); Looks like this might leak credentials. Removing this Map dbNameToURLMap = (Map)objMap.get(DB); - String bigQueryDriver = (String)objMap.get(DRIVER); + String bigQueryDriver = ccr.readEnv((String)objMap.get(DRIVER)); for (Map.Entry entry: dbNameToURLMap.entrySet()) { DataSource dataSource = new DataSource(); dataSource.setInitialSize(INIT_CONNECTIONS); dataSource.setMaxActive(MAX_CONNECTIONS); dataSource.setDriverClassName(bigQueryDriver); - dataSource.setUrl(entry.getValue()); + dataSource.setUrl(ccr.readEnv(entry.getValue())); // Timeout before an abandoned(in use) connection can be removed. dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT); @@ -198,9 +199,9 @@ public SqlResponseCacheLoader(Map properties) throws Exception { h2DataSource.setInitialSize(INIT_CONNECTIONS); h2DataSource.setMaxActive(MAX_CONNECTIONS); - String h2User = (String) objMap.get(USER); - String h2Password = getPassword(objMap); - h2Url = (String) objMap.get(DB); + String h2User = ccr.readEnv((String) objMap.get(USER)); + String h2Password = ccr.readEnv(getPassword(objMap)); + h2Url = ccr.readEnv((String) objMap.get(DB)); h2DataSource.setUsername(h2User); h2DataSource.setPassword(h2Password); h2DataSource.setUrl(h2Url); diff --git a/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java new file mode 100644 index 0000000000..a689ae0b08 --- /dev/null +++ b/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java @@ -0,0 +1,26 @@ +package org.apache.pinot.thirdeye.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CustomConfigReader { + public String readEnv(String inputStr) { + final String regex = "\\?\\$[A-Z0-9_]*"; + final Pattern pattern = Pattern.compile(regex); + final Matcher matcher = pattern.matcher(inputStr); + + while (matcher.find()) { + String matchStr = matcher.group(0); + String env = matchStr.substring(2); + String value = System.getenv(env); + if (value != null) { + // replace original string with configured env variables + inputStr = inputStr.replace(matchStr, value); + } else { + System.out.format("%s is" + " not assigned.%n", env); + } + } + + return inputStr; + } +}