in memory compilation integrated with dynamic config helpers (#1255)

* Rebase against 5.x Co-authored-by: Ayeswarya <[email protected]> Co-authored-by: AvaniMakwana <[email protected]> * pom fix Co-authored-by: Ayeswarya <[email protected]> Co-authored-by: AvaniMakwana <[email protected]> * pom fix Co-authored-by: Ayeswarya <[email protected]> Co-authored-by: AvaniMakwana <[email protected]> * Merging 1220 Co-authored-by: rishi-aga <[email protected]> Co-authored-by: AvaniMakwana <[email protected]> * Spring Boot Changes Co-authored-by: Ayeswarya <[email protected]> * Spring Boot Changes Co-authored-by: Ayeswarya <[email protected]> * Spring Boot Changes Co-authored-by: Ayeswarya <[email protected]> * Review Comments Co-authored-by: Ayeswarya <[email protected]> * Update DynamicConfigTest.java * Review Comments Co-authored-by: Ayeswarya <[email protected]> Co-authored-by: moiz arafat <[email protected]> Co-authored-by: AvaniMakwana <[email protected]> Co-authored-by: rishi-aga <[email protected]>
yahoo · May 15, 2020 · d4af48c · d4af48c
1 parent 39767cd
commit d4af48c
Show file tree

Hide file tree

Showing 64 changed files with 3,671 additions and 12 deletions.
diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml
@@ -14,4 +14,6 @@
   <suppress files="[/\\]src[/\\]main[/\\]webapp[/\\]WEB-INF[/\\]api-docs[/\\]" checks=".*"/>
   <suppress files="[/\\]target[/\\]" checks=".*"/>
   <suppress files="\.csv" checks=".*"/>
+  <suppress files="\.json" checks="LineLength"/>
+  <suppress files="\.hbs" checks="LineLength"/>
 </suppressions>
diff --git a/elide-contrib/elide-dynamic-config-helpers/pom.xml b/elide-contrib/elide-dynamic-config-helpers/pom.xml
@@ -0,0 +1,165 @@
+<!--
+  ~ Copyright 2020, Yahoo Inc.
+  ~ Licensed under the Apache License, Version 2.0
+  ~ See LICENSE file in project root for terms.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>elide-dynamic-config-helpers</artifactId>
+    <packaging>jar</packaging>
+    <name>Elide Dynamic Config Helpers</name>
+    <description>Dynamic config helpers</description>
+    <url>https://github.com/yahoo/elide</url>
+    <parent>
+        <artifactId>elide-contrib-parent-pom</artifactId>
+        <groupId>com.yahoo.elide</groupId>
+        <version>5.0.0-pr9-SNAPSHOT</version>
+    </parent>
+
+    <licenses>
+        <license>
+            <name>The Apache Software License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+
+    <developers>
+        <developer>
+            <name>Yahoo Inc.</name>
+            <url>https://github.com/yahoo</url>
+        </developer>
+    </developers>
+
+    <scm>
+        <developerConnection>scm:git:ssh://[email protected]/yahoo/elide.git</developerConnection>
+        <url>https://github.com/yahoo/elide.git</url>
+        <tag>HEAD</tag>
+    </scm>
+
+    <properties>
+        <hjson.version>3.0.0</hjson.version>
+        <jackson.version>2.10.2</jackson.version>
+        <handlebars.version>4.2.0</handlebars.version>
+        <json-schema-validator.version>2.2.12</json-schema-validator.version>
+        <mdkt.compiler.version>1.3.0</mdkt.compiler.version>
+        <google.collections.version>1.0</google.collections.version>
+        <commons-io.version>2.6</commons-io.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+		<dependency>
+		    <groupId>commons-io</groupId>
+		    <artifactId>commons-io</artifactId>
+		    <version>${commons-io.version}</version>
+		</dependency>
+        <dependency>
+        <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.github.java-json-tools</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>${json-schema-validator.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hjson</groupId>
+            <artifactId>hjson</artifactId>
+            <version>${hjson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.jknack</groupId>
+            <artifactId>handlebars</artifactId>
+            <version>${handlebars.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-launcher</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mdkt.compiler</groupId>
+            <artifactId>InMemoryJavaCompiler</artifactId>
+            <version>${mdkt.compiler.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.collections</groupId>
+            <artifactId>google-collections</artifactId>
+            <version>${google.collections.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/...pers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpers.java b/...pers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpers.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2020, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.contrib.dynamicconfighelpers;
+
+import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig;
+import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig;
+import com.yahoo.elide.contrib.dynamicconfighelpers.model.Table;
+import com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars.HandlebarsHydrator;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.commons.io.FileUtils;
+import org.hjson.JsonValue;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+/**
+ * Util class for Dynamic config helper module.
+ */
+public class DynamicConfigHelpers {
+
+    private static final String TABLE_CONFIG_PATH = "tables" + File.separator;
+    private static final String SECURITY_CONFIG_PATH = "security.hjson";
+    private static final String VARIABLE_CONFIG_PATH = "variables.hjson";
+    private static final String NEW_LINE = "\n";
+
+    /**
+     * Checks whether input is null or empty.
+     * @param input : input string
+     * @return true or false
+     */
+    public static boolean isNullOrEmpty(String input) {
+        return (input == null || input.trim().length() == 0);
+    }
+
+    /**
+     * format config file path.
+     * @param basePath : path to hjson config.
+     * @return formatted file path.
+     */
+    public static String formatFilePath(String basePath) {
+        String path = basePath;
+        if (!path.endsWith(File.separator)) {
+            path += File.separator;
+        }
+        return path;
+    }
+
+    /**
+     * converts variable.hjson to map of variables.
+     * @param basePath : root path to model dir
+     * @return Map of variables
+     * @throws JsonProcessingException
+     */
+    @SuppressWarnings("unchecked")
+    public static Map<String, Object> getVariablesPojo(String basePath) throws JsonProcessingException {
+        String filePath = basePath + VARIABLE_CONFIG_PATH;
+        File variableFile = new File(filePath);
+        if (variableFile.exists()) {
+            String jsonConfig = hjsonToJson(readConfigFile(variableFile));
+            return getModelPojo(jsonConfig, Map.class);
+        } else {
+            log.info("Variables config file not found at " + filePath);
+            return null;
+        }
+    }
+
+    /**
+     * converts all available table config to ElideTableConfig Pojo.
+     * @param basePath : root path to model dir
+     * @param variables : variables to resolve.
+     * @return ElideTableConfig pojo
+     * @throws IOException
+     */
+    public static ElideTableConfig getElideTablePojo(String basePath, Map<String, Object> variables)
+            throws IOException {
+        return getElideTablePojo(basePath, variables, TABLE_CONFIG_PATH);
+    }
+
+    /**
+     * converts all available table config to ElideTableConfig Pojo.
+     * @param basePath : root path to model dir
+     * @param variables : variables to resolve.
+     * @param tableDirName : dir name for table configs
+     * @return ElideTableConfig pojo
+     * @throws IOException
+     */
+    public static ElideTableConfig getElideTablePojo(String basePath, Map<String, Object> variables,
+            String tableDirName) throws IOException {
+        Collection<File> tableConfigs = FileUtils.listFiles(new File(basePath + tableDirName),
+                new String[] {"hjson"}, false);
+        Set<Table> tables = new HashSet<>();
+        for (File tableConfig : tableConfigs) {
+            String jsonConfig = hjsonToJson(resolveVariables(readConfigFile(tableConfig), variables));
+            ElideTableConfig table = getModelPojo(jsonConfig, ElideTableConfig.class);
+            tables.addAll(table.getTables());
+        }
+        ElideTableConfig elideTableConfig = new ElideTableConfig();
+        elideTableConfig.setTables(tables);
+        return elideTableConfig;
+    }
+
+    /**
+     * converts security.hjson to ElideSecurityConfig Pojo.
+     * @param basePath : root path to model dir.
+     * @param variables : variables to resolve.
+     * @return ElideSecurityConfig Pojo
+     * @throws IOException
+     */
+    public static ElideSecurityConfig getElideSecurityPojo(String basePath, Map<String, Object> variables)
+            throws IOException {
+        String filePath = basePath + SECURITY_CONFIG_PATH;
+        File securityFile = new File(filePath);
+        if (securityFile.exists()) {
+            String jsonConfig = hjsonToJson(resolveVariables(readConfigFile(securityFile), variables));
+            return getModelPojo(jsonConfig, ElideSecurityConfig.class);
+        } else {
+            log.info("Security config file not found at " + filePath);
+            return null;
+        }
+    }
+
+    /**
+     * resolves variables in table and security config.
+     * @param jsonConfig of table or security
+     * @param variables map from config
+     * @return json string with resolved variables
+     * @throws IOException
+     */
+    public static String resolveVariables(String jsonConfig, Map<String, Object> variables) throws IOException {
+        HandlebarsHydrator hydrator = new HandlebarsHydrator();
+        return hydrator.hydrateConfigTemplate(jsonConfig, variables);
+    }
+
+    private static String hjsonToJson(String hjson) {
+        return JsonValue.readHjson(hjson).toString();
+    }
+
+    private static <T> T getModelPojo(String jsonConfig, final Class<T> configPojo) throws JsonProcessingException {
+        return new ObjectMapper().readValue(jsonConfig, configPojo);
+    }
+
+    private static String readConfigFile(File configFile) {
+        StringBuffer sb = new StringBuffer();
+        try {
+            for (String line : FileUtils.readLines(configFile, StandardCharsets.UTF_8)) {
+                sb.append(line);
+                sb.append(NEW_LINE);
+            }
+        } catch (IOException e) {
+            log.error("error while reading config file " + configFile.getName());
+            log.error(e.getMessage());
+        }
+        return sb.toString();
+    }
+}