From f4d0dd808ff60e66b6cab653a09b96e05337bf62 Mon Sep 17 00:00:00 2001 From: siddhant2001 Date: Fri, 25 Oct 2024 17:38:27 +0530 Subject: [PATCH] ENG-50193: Document store using credentials (#100) --- .gitignore | 3 +- config-bootstrapper/build.gradle.kts | 2 +- .../ConfigBootstrapperIntegrationTest.java | 9 ++- .../config-bootstrapper-test/application.conf | 19 ++++-- .../core/bootstrapper/ConfigBootstrapper.java | 13 +++- .../dao/ConfigBootstrapStatusDao.java | 60 +++++++++++-------- .../config-bootstrapper/application.conf | 23 +++++-- .../templates/config-bootstrapper-config.yaml | 50 +++++++--------- helm/templates/job.yaml | 14 +++++ helm/values.yaml | 18 +----- 10 files changed, 126 insertions(+), 85 deletions(-) diff --git a/.gitignore b/.gitignore index b7a3b05..1698321 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,6 @@ test-output .idea/dictionaries/ .idea/codeStyles/ .idea/.name +.idea/ # Local config to handle using Java 8 vs java 11. -.java-version \ No newline at end of file +.java-version diff --git a/config-bootstrapper/build.gradle.kts b/config-bootstrapper/build.gradle.kts index e2965da..3c5fda8 100644 --- a/config-bootstrapper/build.gradle.kts +++ b/config-bootstrapper/build.gradle.kts @@ -128,7 +128,7 @@ tasks.test { dependencies { implementation("org.hypertrace.entity.service:entity-service-client:0.6.4") implementation("org.hypertrace.entity.service:entity-service-api:0.6.4") - implementation("org.hypertrace.core.documentstore:document-store:0.7.23") + implementation("org.hypertrace.core.documentstore:document-store:0.8.4") implementation("org.hypertrace.core.attribute.service:attribute-service-client:0.14.5") implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.11.2") implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.11.2") diff --git a/config-bootstrapper/src/integrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java b/config-bootstrapper/src/integrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java index 9ddb8bb..5a31df8 100644 --- a/config-bootstrapper/src/integrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java +++ b/config-bootstrapper/src/integrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java @@ -19,6 +19,8 @@ import org.hypertrace.core.documentstore.Datastore; import org.hypertrace.core.documentstore.DatastoreProvider; import org.hypertrace.core.documentstore.DocumentStoreConfig; +import org.hypertrace.core.documentstore.model.config.DatastoreConfig; +import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor; import org.hypertrace.core.grpcutils.context.RequestContext; import org.hypertrace.core.grpcutils.context.RequestContextConstants; import org.hypertrace.entity.service.client.config.EntityServiceClientConfig; @@ -30,6 +32,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.hypertrace.core.bootstrapper.ConfigBootstrapper.DATASTORE_TYPE_CONFIG_KEY; import static org.hypertrace.core.bootstrapper.dao.ConfigBootstrapStatusDao.CONFIG_BOOTSTRAPPER_COLLECTION; public class ConfigBootstrapperIntegrationTest { @@ -62,9 +65,11 @@ public static void setUp() { new TenantIdClientInterceptor(TENANT_ID)); attributesServiceClient = new AttributeServiceClient(channel); - String dataStoreType = config.getString(DocumentStoreConfig.DATASTORE_TYPE_CONFIG_KEY); + Config documentStoreConfig = config.getConfig("document.store"); + DatastoreConfig datastoreConfig = TypesafeConfigDatastoreConfigExtractor.from(documentStoreConfig, DATASTORE_TYPE_CONFIG_KEY) + .extract(); Datastore datastore = - DatastoreProvider.getDatastore(dataStoreType, config.getConfig(dataStoreType)); + DatastoreProvider.getDatastore(datastoreConfig); datastore.getCollection(CONFIG_BOOTSTRAPPER_COLLECTION).drop(); } diff --git a/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf b/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf index cf9dc70..40c81b6 100644 --- a/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf +++ b/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf @@ -6,8 +6,19 @@ entity.service.config = { host=localhost port=50061 } -dataStoreType = mongo -mongo = { - host=localhost - port=37017 + +document.store { + dataStoreType = "mongo" + appName = "config-bootstrapper-integration-test" + maxPoolSize = 10 + mongo = { + endpoints = [ + { + host: "localhost" + port: 37017 + } + ] + database = default_db + authDb = admin + } } diff --git a/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java b/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java index 2a24a94..78c4b90 100644 --- a/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java +++ b/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java @@ -10,12 +10,14 @@ import org.hypertrace.core.bootstrapper.dao.ConfigBootstrapStatusDao; import org.hypertrace.core.documentstore.Datastore; import org.hypertrace.core.documentstore.DatastoreProvider; -import org.hypertrace.core.documentstore.DocumentStoreConfig; +import org.hypertrace.core.documentstore.model.config.DatastoreConfig; +import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ConfigBootstrapper { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigBootstrapper.class); + public static final String DATASTORE_TYPE_CONFIG_KEY = "dataStoreType"; public static void main(String[] args) { updateRuntime(); @@ -29,9 +31,14 @@ public static void main(String[] args) { * */ public static BootstrapRunner bootstrapper(BootstrapArgs bootstrapArgs) { Config config = ConfigFactory.parseFile(new File(bootstrapArgs.getConfigFile())).resolve(); - String dataStoreType = config.getString(DocumentStoreConfig.DATASTORE_TYPE_CONFIG_KEY); + Config documentStoreConfig = config.getConfig("document.store"); + DatastoreConfig datastoreConfig = TypesafeConfigDatastoreConfigExtractor.from(documentStoreConfig, DATASTORE_TYPE_CONFIG_KEY) + .extract(); Datastore datastore = - DatastoreProvider.getDatastore(dataStoreType, config.getConfig(dataStoreType)); + DatastoreProvider.getDatastore(datastoreConfig); + + Runtime.getRuntime().addShutdownHook(new Thread(datastore::close)); + return new BootstrapRunner(new ConfigBootstrapStatusDao(datastore)); } diff --git a/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java b/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java index a21a714..adf5541 100644 --- a/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java +++ b/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java @@ -9,6 +9,7 @@ import java.util.stream.StreamSupport; import org.hypertrace.core.bootstrapper.model.ConfigBootstrapStatus; import org.hypertrace.core.bootstrapper.model.ConfigBootstrapStatusKey; +import org.hypertrace.core.documentstore.CloseableIterator; import org.hypertrace.core.documentstore.Datastore; import org.hypertrace.core.documentstore.Document; import org.hypertrace.core.documentstore.Filter; @@ -33,15 +34,20 @@ public ConfigBootstrapStatusDao(Datastore datastore) { public ConfigBootstrapStatus getConfigBootstrapStatus(ConfigBootstrapStatusKey key) { Query query = new Query(); query.setFilter(new Filter(Filter.Op.EQ, ID, key.toString())); - Iterator docs = configBootstrapStatusCollection.search(query); - if (docs.hasNext()) { - Document doc = docs.next(); - try { - return ConfigBootstrapStatus.fromJson(doc.toJson()); - } catch (IOException ex) { - LOGGER.error(String.format("Error creating ConfigBootstrapStatus from doc:%s", doc), ex); - return null; + try (final CloseableIterator docsIterator = + configBootstrapStatusCollection.search(query)){ + if (docsIterator.hasNext()) { + Document doc = docsIterator.next(); + try { + return ConfigBootstrapStatus.fromJson(doc.toJson()); + } catch (IOException ex) { + LOGGER.error(String.format("Error creating ConfigBootstrapStatus from doc:%s", doc), ex); + return null; + } } + } catch (IOException e) { + LOGGER.error("Error in query: {}", query); + return null; } return null; } @@ -65,22 +71,26 @@ public boolean deleteConfigBootstrapStatus(ConfigBootstrapStatusKey key) { public List getConfigBootstrapStatusGreaterThan(int version) { Query query = new Query(); query.setFilter(new Filter(Filter.Op.GT, VERSION, version)); - Iterator docs = configBootstrapStatusCollection.search(query); - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(docs, 0), false) - .map( - document -> { - try { - return ConfigBootstrapStatus.fromJson(document.toJson()); - } catch (IOException ex) { - LOGGER.error("Error creating ConfigBootstrapStatus from doc:{}", document); - return null; - } - }) - .filter(Objects::nonNull) - // sort from higher version to lower version - .sorted( - (c1, c2) -> - ConfigBootstrapStatusKey.from(c2).compareTo(ConfigBootstrapStatusKey.from(c1))) - .collect(Collectors.toList()); + try(final CloseableIterator docs = configBootstrapStatusCollection.search(query)){ + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(docs, 0), false) + .map( + document -> { + try { + return ConfigBootstrapStatus.fromJson(document.toJson()); + } catch (IOException ex) { + LOGGER.error("Error creating ConfigBootstrapStatus from doc:{}", document); + return null; + } + }) + .filter(Objects::nonNull) + // sort from higher version to lower version + .sorted( + (c1, c2) -> + ConfigBootstrapStatusKey.from(c2).compareTo(ConfigBootstrapStatusKey.from(c1))) + .collect(Collectors.toList()); + } catch (IOException e) { + LOGGER.error("Error in query: {}", query); + return null; + } } } diff --git a/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf b/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf index 9b8b2e4..309cc29 100644 --- a/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf +++ b/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf @@ -10,9 +10,22 @@ entity.service.config = { port=50061 port=${?ENTITY_SERVICE_PORT_CONFIG} } -dataStoreType = mongo -mongo = { - host=localhost - host=${?MONGO_HOST} - port=27017 + +document.store { + dataStoreType = mongo + appName = config-bootstrapper-local + maxPoolSize = 10 + mongo { + database = default_db + user = ${?MONGO_SERVICE_USERNAME} + password = ${?MONGO_SERVICE_USER_PASSWORD} + endpoints = [ + { + host = localhost + host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable + port = 27017 + port = ${?MONGO_PORT} + } + ] + } } \ No newline at end of file diff --git a/helm/templates/config-bootstrapper-config.yaml b/helm/templates/config-bootstrapper-config.yaml index fc80eeb..2f77ed2 100644 --- a/helm/templates/config-bootstrapper-config.yaml +++ b/helm/templates/config-bootstrapper-config.yaml @@ -14,32 +14,28 @@ data: host={{ .Values.entityServiceConfig.data.host }} port={{ .Values.entityServiceConfig.data.port }} } - dataStoreType = {{ .Values.dataStoreType }} - {{- if eq .Values.dataStoreType "mongo" }} - mongo = { - {{- if .Values.mongoConfig.data.url }} - url = {{ .Values.mongoConfig.data.url | quote }} + + document.store { + {{- if .Values.database }} + {{- $dst := .Values.database.type }} + dataStoreType = {{ $dst }} + appName = {{ .Values.name }} + {{- range $key, $value := (index .Values "database") }} + {{- if ne $key $dst }} + {{ $key }} = {{- toJson $value }} + {{- end }} + {{- end }} + + {{ $dst }} { + {{- range $key, $value := (index .Values "database" (printf "%s" $dst)) }} + {{- if $value }} + {{- if hasPrefix "${?" (printf "%s" $value) }} + {{ $key }} = {{ $value }} {{- else }} - host={{ .Values.mongoConfig.data.host }} - port={{ .Values.mongoConfig.data.port }} + {{ $key }} = {{- toJson $value }} {{- end }} - } - {{ else if eq .Values.dataStoreType "postgres" }} - postgres = { - {{- if .Values.postgresConfig.data.url }} - url={{ .Values.postgresConfig.data.url | quote }} - {{- else }} - host={{ .Values.postgresConfig.data.host }} - port={{ .Values.postgresConfig.data.port }} - {{- end }} - {{- if .Values.postgresConfig.data.database }} - database= {{ .Values.postgresConfig.data.database }} - {{- end }} - {{- if .Values.postgresConfig.data.user }} - user={{ .Values.postgresConfig.data.user }} - {{- end }} - {{- if .Values.postgresConfig.data.password }} - password={{ .Values.postgresConfig.data.password }} - {{- end }} - } - {{- end }} + {{- end }} + {{- end }} + } + {{- end }} + } \ No newline at end of file diff --git a/helm/templates/job.yaml b/helm/templates/job.yaml index 1565423..17144ea 100644 --- a/helm/templates/job.yaml +++ b/helm/templates/job.yaml @@ -103,6 +103,20 @@ spec: value: "/var/{{ .Chart.Name }}/log/log4j2.properties" - name: JAVA_TOOL_OPTIONS value: {{ .Values.javaOpts | quote }} + {{- if and .Values.database .Values.database.mongo.authEnabled }} + {{- with .Values.database.mongo.credentials }} + - name: MONGO_SERVICE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretUsernameKey }} + - name: MONGO_SERVICE_USER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .secretName }} + key: {{ .secretPasswordKey }} + {{- end }} + {{- end }} {{- if and (eq .Values.dataStoreType "postgres") .Values.postgresConfig.data.passwordSecretName .Values.postgresConfig.data.passwordSecretKey }} - name: POSTGRESQL_PASSWORD valueFrom: diff --git a/helm/values.yaml b/helm/values.yaml index 8b19c66..78816a2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -13,6 +13,7 @@ ########### # Job ########### +name: config-bootstrapper image: repository: hypertrace/config-bootstrapper @@ -86,23 +87,6 @@ attributeServiceConfig: host: attribute-service port: 9012 -dataStoreType: "mongo" -mongoConfig: - name: mongo-config - data: - host: mongo - port: 27017 - url: "" - -postgresConfig: - name: postgres-config - data: - host: postgres - port: 5432 - url: "" - passwordSecretName: "" - passwordSecretKey: "" - logConfig: name: config-bootstrapper-log-appender-config rootLogger: