Skip to content
This repository has been archived by the owner on Oct 18, 2022. It is now read-only.

Commit

Permalink
[config] - Provides ability to read configs and secrets from environm…
Browse files Browse the repository at this point in the history 
…ent variables
  • Loading branch information
venkatvghub committed Mar 30, 2021
1 parent 6cee427 commit 90b30be
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 31 deletions.
19 changes: 12 additions & 7 deletions thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AbstractManagerImpl;
import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
import org.apache.pinot.thirdeye.datalayer.util.PersistenceConfig.DatabaseConfiguration;
import org.apache.pinot.thirdeye.util.CustomConfigReader;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.h2.store.fs.FileUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -84,11 +85,18 @@ private static DataSource createDataSource(final PersistenceConfig configuration
dataSource.setInitialSize(10);
dataSource.setDefaultAutoCommit(false);
dataSource.setMaxActive(100);
dataSource.setUsername(configuration.getDatabaseConfiguration().getUser());
dataSource.setPassword(configuration.getDatabaseConfiguration().getPassword());
dataSource.setUrl(configuration.getDatabaseConfiguration().getUrl());

/* Read the database credentials and details from environment variables */
CustomConfigReader ccr = new CustomConfigReader();
String userName = ccr.readEnv(configuration.getDatabaseConfiguration().getUser());
dataSource.setUsername(userName);
String password = ccr.readEnv(configuration.getDatabaseConfiguration().getPassword());
dataSource.setPassword(password);
String url = ccr.readEnv(configuration.getDatabaseConfiguration().getUrl());
dataSource.setUrl(url);
dataSource.setDriverClassName(configuration.getDatabaseConfiguration().getDriver());

// Additional DB healthchecks and pool configuration. Can be removed
dataSource.setValidationQuery("select 1");
dataSource.setTestWhileIdle(true);
dataSource.setTestOnBorrow(true);
Expand All @@ -102,10 +110,7 @@ private static DataSource createDataSource(final PersistenceConfig configuration
return dataSource;
}

public static void init(DataSource dataSource) {
injector = Guice.createInjector(new ThirdEyePersistenceModule(dataSource));
}


public static PersistenceConfig readPersistenceConfig(File configFile) {
YamlConfigurationFactory<PersistenceConfig> factory = new YamlConfigurationFactory<>(
PersistenceConfig.class,
Expand Down
17 changes: 10 additions & 7 deletions ...c/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
import org.apache.pinot.thirdeye.util.CustomConfigReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -212,14 +213,16 @@ public PinotThirdEyeDataSourceConfig build() {
"{} accepts only 'http' or 'https' connection schemes", className);

PinotThirdEyeDataSourceConfig config = new PinotThirdEyeDataSourceConfig();
config.setControllerHost(controllerHost);
config.setControllerPort(controllerPort);
config.setZookeeperUrl(zookeeperUrl);
config.setClusterName(clusterName);
config.setBrokerUrl(brokerUrl);
config.setTag(tag);
// Read from the custom config reader for reading from environment variables
CustomConfigReader ccr = new CustomConfigReader();
config.setControllerHost(ccr.readEnv(controllerHost));
config.setControllerPort(ccr.readEnv(controllerPort));
config.setZookeeperUrl(ccr.readEnv(zookeeperUrl));
config.setClusterName(ccr.readEnv(clusterName));
config.setBrokerUrl(ccr.readEnv(brokerUrl));
config.setTag(ccr.readEnv(tag));
config.setControllerConnectionScheme(controllerConnectionScheme);
config.setName(name);
config.setName(ccr.readEnv(name));
return config;
}
}
Expand Down
35 changes: 18 additions & 17 deletions ...-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
import org.apache.pinot.thirdeye.util.CustomConfigReader;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
Expand Down Expand Up @@ -85,22 +86,22 @@ public class SqlResponseCacheLoader extends CacheLoader<SqlQuery, ThirdEyeResult
DataSource h2DataSource;

public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {

CustomConfigReader ccr = new CustomConfigReader();
// Init Presto datasources
if (properties.containsKey(PRESTO)) {
List<Map<String, Object>> prestoMapList = ConfigUtils.getList(properties.get(PRESTO));
for (Map<String, Object> objMap: prestoMapList) {
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
String prestoUser = (String)objMap.get(USER);
String prestoPassword = getPassword(objMap);
String prestoUser = ccr.readEnv((String)objMap.get(USER));
String prestoPassword = ccr.readEnv(getPassword(objMap));

for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
DataSource dataSource = new DataSource();
dataSource.setInitialSize(INIT_CONNECTIONS);
dataSource.setMaxActive(MAX_CONNECTIONS);
dataSource.setUsername(prestoUser);
dataSource.setPassword(prestoPassword);
dataSource.setUrl(entry.getValue());
dataSource.setUrl(ccr.readEnv(entry.getValue()));

// Timeout before an abandoned(in use) connection can be removed.
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
Expand All @@ -117,16 +118,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
List<Map<String, Object>> mysqlMapList = ConfigUtils.getList(properties.get(MYSQL));
for (Map<String, Object> objMap: mysqlMapList) {
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
String mysqlUser = (String)objMap.get(USER);
String mysqlPassword = getPassword(objMap);
String mysqlUser = ccr.readEnv((String)objMap.get(USER));
String mysqlPassword = ccr.readEnv(getPassword(objMap));

for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
DataSource dataSource = new DataSource();
dataSource.setInitialSize(INIT_CONNECTIONS);
dataSource.setMaxActive(MAX_CONNECTIONS);
dataSource.setUsername(mysqlUser);
dataSource.setPassword(mysqlPassword);
dataSource.setUrl(entry.getValue());
dataSource.setUrl(ccr.readEnv(entry.getValue()));

// Timeout before an abandoned(in use) connection can be removed.
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
Expand All @@ -143,9 +144,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
List<Map<String, Object>> verticaMapList = ConfigUtils.getList(properties.get(VERTICA));
for (Map<String, Object> objMap: verticaMapList) {
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
String verticaUser = (String)objMap.get(USER);
String verticaPassword = getPassword(objMap);
String verticaDriver = (String)objMap.get(DRIVER);
String verticaUser = ccr.readEnv((String)objMap.get(USER));
String verticaPassword = ccr.readEnv(getPassword(objMap));
String verticaDriver = ccr.readEnv((String)objMap.get(DRIVER));

for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
DataSource dataSource = new DataSource();
Expand All @@ -154,7 +155,7 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
dataSource.setUsername(verticaUser);
dataSource.setPassword(verticaPassword);
dataSource.setDriverClassName(verticaDriver);
dataSource.setUrl(entry.getValue());
dataSource.setUrl(ccr.readEnv(entry.getValue()));

// Timeout before an abandoned(in use) connection can be removed.
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
Expand All @@ -170,16 +171,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
if (properties.containsKey(BIGQUERY)) {
List<Map<String, Object>> bigQueryMapList = ConfigUtils.getList(properties.get(BIGQUERY));
for (Map<String, Object> objMap: bigQueryMapList) {
System.out.println(bigQueryMapList.toString());
// System.out.println(bigQueryMapList.toString()); Looks like this might leak credentials. Removing this
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
String bigQueryDriver = (String)objMap.get(DRIVER);
String bigQueryDriver = ccr.readEnv((String)objMap.get(DRIVER));

for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
DataSource dataSource = new DataSource();
dataSource.setInitialSize(INIT_CONNECTIONS);
dataSource.setMaxActive(MAX_CONNECTIONS);
dataSource.setDriverClassName(bigQueryDriver);
dataSource.setUrl(entry.getValue());
dataSource.setUrl(ccr.readEnv(entry.getValue()));

// Timeout before an abandoned(in use) connection can be removed.
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
Expand All @@ -198,9 +199,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {

h2DataSource.setInitialSize(INIT_CONNECTIONS);
h2DataSource.setMaxActive(MAX_CONNECTIONS);
String h2User = (String) objMap.get(USER);
String h2Password = getPassword(objMap);
h2Url = (String) objMap.get(DB);
String h2User = ccr.readEnv((String) objMap.get(USER));
String h2Password = ccr.readEnv(getPassword(objMap));
h2Url = ccr.readEnv((String) objMap.get(DB));
h2DataSource.setUsername(h2User);
h2DataSource.setPassword(h2Password);
h2DataSource.setUrl(h2Url);
Expand Down
26 changes: 26 additions & 0 deletions thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package org.apache.pinot.thirdeye.util;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CustomConfigReader {
public String readEnv(String inputStr) {
final String regex = "\\?\\$[A-Z0-9_]*";
final Pattern pattern = Pattern.compile(regex);
final Matcher matcher = pattern.matcher(inputStr);

while (matcher.find()) {
String matchStr = matcher.group(0);
String env = matchStr.substring(2);
String value = System.getenv(env);
if (value != null) {
// replace original string with configured env variables
inputStr = inputStr.replace(matchStr, value);
} else {
System.out.format("%s is" + " not assigned.%n", env);
}
}

return inputStr;
}
}

0 comments on commit 90b30be

Please sign in to comment.