Custom Config reader for reading configs and secrets from environment variables

thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java

@@ -33,6 +33,7 @@
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AbstractManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
 import org.apache.pinot.thirdeye.datalayer.util.PersistenceConfig.DatabaseConfiguration;
+import org.apache.pinot.thirdeye.util.CustomConfigReader;
 import org.apache.tomcat.jdbc.pool.DataSource;
 import org.h2.store.fs.FileUtils;
 import org.slf4j.Logger;
@@ -81,12 +82,13 @@ private static void createSchemaIfReqd(
 
   private static DataSource createDataSource(final PersistenceConfig configuration) {
     final DataSource dataSource = new DataSource();
+    CustomConfigReader ccr = new CustomConfigReader();
     dataSource.setInitialSize(10);
     dataSource.setDefaultAutoCommit(false);
     dataSource.setMaxActive(100);
-    dataSource.setUsername(configuration.getDatabaseConfiguration().getUser());
-    dataSource.setPassword(configuration.getDatabaseConfiguration().getPassword());
-    dataSource.setUrl(configuration.getDatabaseConfiguration().getUrl());
+    dataSource.setUsername(ccr.readEnv(configuration.getDatabaseConfiguration().getUser()));
+    dataSource.setPassword(ccr.readEnv(configuration.getDatabaseConfiguration().getPassword()));
+    dataSource.setUrl(ccr.readEnv(configuration.getDatabaseConfiguration().getUrl()));
     dataSource.setDriverClassName(configuration.getDatabaseConfiguration().getDriver());
 
     dataSource.setValidationQuery("select 1");

thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java

@@ -27,6 +27,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
 import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
+import org.apache.pinot.thirdeye.util.CustomConfigReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -212,13 +213,15 @@ public PinotThirdEyeDataSourceConfig build() {
           "{} accepts only 'http' or 'https' connection schemes", className);
 
       PinotThirdEyeDataSourceConfig config = new PinotThirdEyeDataSourceConfig();
-      config.setControllerHost(controllerHost);
+      // Read from the custom config reader for reading from environment variables
+      CustomConfigReader ccr = new CustomConfigReader();
+      config.setControllerHost(ccr.readEnv(controllerHost));
       config.setControllerPort(controllerPort);
-      config.setZookeeperUrl(zookeeperUrl);
-      config.setClusterName(clusterName);
-      config.setBrokerUrl(brokerUrl);
-      config.setTag(tag);
-      config.setControllerConnectionScheme(controllerConnectionScheme);
+      config.setZookeeperUrl(ccr.readEnv(zookeeperUrl));
+      config.setClusterName(ccr.readEnv(clusterName));
+      config.setBrokerUrl(ccr.readEnv(brokerUrl));
+      config.setTag(ccr.readEnv(tag));
+      config.setControllerConnectionScheme(ccr.readEnv(controllerConnectionScheme));
       config.setName(name);
       return config;
     }

thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java

@@ -39,6 +39,7 @@
 import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup;
 import org.apache.pinot.thirdeye.detection.ConfigUtils;
 import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
+import org.apache.pinot.thirdeye.util.CustomConfigReader;
 import org.apache.tomcat.jdbc.pool.DataSource;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -85,22 +86,22 @@ public class SqlResponseCacheLoader extends CacheLoader<SqlQuery, ThirdEyeResult
   DataSource h2DataSource;
 
   public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
-
+    CustomConfigReader ccr = new CustomConfigReader();
     // Init Presto datasources
     if (properties.containsKey(PRESTO)) {
       List<Map<String, Object>> prestoMapList = ConfigUtils.getList(properties.get(PRESTO));
       for (Map<String, Object> objMap: prestoMapList) {
         Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
-        String prestoUser = (String)objMap.get(USER);
-        String prestoPassword = getPassword(objMap);
+        String prestoUser = ccr.readEnv((String)objMap.get(USER));
+        String prestoPassword = ccr.readEnv(getPassword(objMap));
 
         for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
           DataSource dataSource = new DataSource();
           dataSource.setInitialSize(INIT_CONNECTIONS);
           dataSource.setMaxActive(MAX_CONNECTIONS);
           dataSource.setUsername(prestoUser);
           dataSource.setPassword(prestoPassword);
-          dataSource.setUrl(entry.getValue());
+          dataSource.setUrl(ccr.readEnv(entry.getValue()));
 
           // Timeout before an abandoned(in use) connection can be removed.
           dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -117,16 +118,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
       List<Map<String, Object>> mysqlMapList = ConfigUtils.getList(properties.get(MYSQL));
       for (Map<String, Object> objMap: mysqlMapList) {
         Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
-        String mysqlUser = (String)objMap.get(USER);
-        String mysqlPassword = getPassword(objMap);
+        String mysqlUser = ccr.readEnv((String)objMap.get(USER));
+        String mysqlPassword = ccr.readEnv(getPassword(objMap));
 
         for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
           DataSource dataSource = new DataSource();
           dataSource.setInitialSize(INIT_CONNECTIONS);
           dataSource.setMaxActive(MAX_CONNECTIONS);
           dataSource.setUsername(mysqlUser);
           dataSource.setPassword(mysqlPassword);
-          dataSource.setUrl(entry.getValue());
+          dataSource.setUrl(ccr.readEnv(entry.getValue()));
 
           // Timeout before an abandoned(in use) connection can be removed.
           dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -143,9 +144,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
       List<Map<String, Object>> verticaMapList = ConfigUtils.getList(properties.get(VERTICA));
       for (Map<String, Object> objMap: verticaMapList) {
         Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
-        String verticaUser = (String)objMap.get(USER);
-        String verticaPassword = getPassword(objMap);
-        String verticaDriver = (String)objMap.get(DRIVER);
+        String verticaUser = ccr.readEnv((String)objMap.get(USER));
+        String verticaPassword = ccr.readEnv(getPassword(objMap));
+        String verticaDriver = ccr.readEnv((String)objMap.get(DRIVER));
 
         for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
           DataSource dataSource = new DataSource();
@@ -154,7 +155,7 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
           dataSource.setUsername(verticaUser);
           dataSource.setPassword(verticaPassword);
           dataSource.setDriverClassName(verticaDriver);
-          dataSource.setUrl(entry.getValue());
+          dataSource.setUrl(ccr.readEnv(entry.getValue()));
 
           // Timeout before an abandoned(in use) connection can be removed.
           dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -170,16 +171,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
     if (properties.containsKey(BIGQUERY)) {
       List<Map<String, Object>> bigQueryMapList = ConfigUtils.getList(properties.get(BIGQUERY));
       for (Map<String, Object> objMap: bigQueryMapList) {
-        System.out.println(bigQueryMapList.toString());
+       // System.out.println(bigQueryMapList.toString()); Looks like this might leak credentials. Removing this
         Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
-        String bigQueryDriver = (String)objMap.get(DRIVER);
+        String bigQueryDriver = ccr.readEnv((String)objMap.get(DRIVER));
 
         for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
           DataSource dataSource = new DataSource();
           dataSource.setInitialSize(INIT_CONNECTIONS);
           dataSource.setMaxActive(MAX_CONNECTIONS);
           dataSource.setDriverClassName(bigQueryDriver);
-          dataSource.setUrl(entry.getValue());
+          dataSource.setUrl(ccr.readEnv(entry.getValue()));
 
           // Timeout before an abandoned(in use) connection can be removed.
           dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -198,9 +199,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
 
       h2DataSource.setInitialSize(INIT_CONNECTIONS);
       h2DataSource.setMaxActive(MAX_CONNECTIONS);
-      String h2User = (String) objMap.get(USER);
-      String h2Password = getPassword(objMap);
-      h2Url = (String) objMap.get(DB);
+      String h2User = ccr.readEnv((String) objMap.get(USER));
+      String h2Password = ccr.readEnv(getPassword(objMap));
+      h2Url = ccr.readEnv((String) objMap.get(DB));
       h2DataSource.setUsername(h2User);
       h2DataSource.setPassword(h2Password);
       h2DataSource.setUrl(h2Url);

thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java

@@ -0,0 +1,26 @@
+package org.apache.pinot.thirdeye.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CustomConfigReader {
+	public String readEnv(String inputStr) {
+		final String regex = "\\?\\$[A-Z0-9_]*";
+		final Pattern pattern = Pattern.compile(regex);
+		final Matcher matcher = pattern.matcher(inputStr);
+
+		while (matcher.find()) {
+			String matchStr = matcher.group(0);
+			String env = matchStr.substring(2);
+			String value = System.getenv(env);
+			if (value != null) {
+				// replace original string with configured env variables
+				inputStr = inputStr.replace(matchStr, value);
+			} else {
+				System.out.format("%s is" + " not assigned.%n", env);
+			}
+		}
+
+		return inputStr;
+	}
+}