Merge branch 'master' into feat/deployment-firebase

packages/@idemsInternational/env-replace/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+test/**/*.*
+!test/**/*.template.*

packages/@idemsInternational/env-replace/README.md

@@ -0,0 +1,47 @@
+# Env Replace
+
+Populate variables hardcoded into files from environment
+
+## Setup Template Files
+All file formats are supported, however files containing variables should be named with `.template.` in the filename, e.g. `config.template.ts`. 
+
+Within a file variables for replacement should be wrapped with curly braces, e.g.
+
+_config.template.ts_
+```ts
+const config = {
+    appId: ${APP_ID}
+}
+```
+
+## Replace Templated Files
+```ts
+import {replaceFiles} from '@idemsInternational/env-replace'
+
+await replaceFiles()
+```
+
+## Configuration
+The replace method can be customised with various parameters
+
+
+## Future TODOs
+
+**Variable Parsing**
+Future syntax could include utility helpers, e.g.
+```ts
+const config = {
+    nested_config: JSON(${STRING_JSON})
+}
+```
+
+**Variable Fallback**
+Adding support for default/fallback values like some shell interpolations
+```ts
+const config = {
+    appId: ${APP_ID:-debug_app}
+}
+```
+
+**Env file**
+Pass `envFile` path to load variables from

packages/@idemsInternational/env-replace/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@idemsInternational/env-replace",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "scripts": {
+    "test": "ts-mocha -p tsconfig.spec.json src/**/*.spec.ts"
+  },
+  "dependencies": {
+    "glob": "^10.3.10"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.2.22",
+    "@types/expect": "^24.3.0",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^16.18.9",
+    "chai": "^4.3.4",
+    "mocha": "^10.3.0",
+    "ts-mocha": "^10.0.0",
+    "typescript": "~4.2.4"
+  }
+}

packages/@idemsInternational/env-replace/src/index.spec.ts

@@ -0,0 +1,72 @@
+import { resolve } from "path";
+import { expect } from "expect";
+import { envReplace, IEnvReplaceConfig } from "./index";
+import { readFileSync, readdirSync, rmSync } from "fs";
+
+const testDir = resolve(__dirname, "../test");
+
+const TEST_ENV = {
+  STRING_VAR: "example string",
+  BOOL_VAR: true,
+  INT_VAR: 8,
+  CHILD_VAR: "example child",
+};
+
+// Clean any generated output files
+function cleanTestDir(dir: string) {
+  const testFiles = readdirSync(dir, { withFileTypes: true });
+  for (const testFile of testFiles) {
+    if (testFile.isDirectory()) {
+      cleanTestDir(resolve(dir, testFile.name));
+    }
+    if (testFile.isFile() && !testFile.name.includes("template.")) {
+      const fullPath = resolve(dir, testFile.name);
+      rmSync(fullPath);
+    }
+  }
+}
+
+describe("Env Replace", () => {
+  beforeEach(() => {
+    cleanTestDir(testDir);
+  });
+
+  it("Replaces file env", async () => {
+    // populate global process env to use alongside hardcoded env
+    process.env.GLOBAL_VAR = "example global";
+    const res = await envReplace.replaceFiles({
+      cwd: testDir,
+      envAdditional: TEST_ENV,
+      excludeVariables: ["EXCLUDED_VAR"],
+    });
+    // list of replaced values
+    expect(res).toEqual({
+      "test_basic.json": {
+        STRING_VAR: "example string",
+        GLOBAL_VAR: "example global",
+        BOOL_VAR: true,
+        INT_VAR: 8,
+      },
+      "child/.env": { STRING_VAR: "example string" },
+    });
+    // raw file output (including non-replaced)
+    const outputFile = readFileSync(resolve(testDir, "test_basic.json"), { encoding: "utf-8" });
+    expect(JSON.parse(outputFile)).toEqual({
+      test_string: "example string",
+      test_global: "example global",
+      test_excluded: "${EXCLUDED_VAR}",
+      test_non_var: "example non var",
+      test_boolean: true,
+      test_int: 8,
+    });
+  });
+
+  it("Supports file matching glob override", async () => {
+    const res = await envReplace.replaceFiles({
+      rawGlobInput: "child/*.*",
+      envAdditional: TEST_ENV,
+      cwd: testDir,
+    });
+    expect(res).toEqual({ "child/.env": { STRING_VAR: "example string" } });
+  });
+});

packages/@idemsInternational/env-replace/src/index.ts

@@ -0,0 +1,182 @@
+import { readFileSync, writeFileSync } from "fs";
+import { GlobOptionsWithFileTypesUnset, glob } from "glob";
+import { resolve } from "path";
+
+export interface IEnvReplaceConfig {
+  cwd: string;
+
+  /** List of input folder glob patterns to include, default ['**'] */
+  includeFolders: string[];
+
+  /** Filename pattern to find */
+  fileNameFind: string;
+
+  /** String to replace filename pattern match with to output to different file, default '' */
+  fileNameReplace: string;
+
+  /** Additional variables to populate to env. Will be merged with global env */
+  envAdditional: Record<string, any>;
+
+  /** List of folder glob patterns to exclude,  */
+  excludeFolders: string[];
+
+  /** Specify list of variable names to exclude, default includes all */
+  excludeVariables: string[];
+
+  /** Specify list of variable names to include, default includes all */
+  includeVariables: string[];
+
+  /** Override generated globs from input folder and filename patters */
+  rawGlobInput: string;
+
+  /** Additional options to provide directly to input glob match */
+  rawGlobOptions: GlobOptionsWithFileTypesUnset;
+
+  /** Throw an error if environment variable is missing. Default true */
+  throwOnMissing: boolean;
+}
+
+const DEFAULT_CONFIG: IEnvReplaceConfig = {
+  cwd: process.cwd(),
+  envAdditional: {},
+  excludeFolders: ["node_modules/**"],
+  excludeVariables: [],
+  fileNameFind: ".template.",
+  fileNameReplace: ".",
+  includeFolders: ["**"],
+  includeVariables: [],
+  rawGlobInput: "",
+  rawGlobOptions: {},
+  throwOnMissing: true,
+};
+
+/**
+ * Regex pattern used to identify templated variable names.
+ * Supports minimal alphanumeric and underscore characters.
+ * Uses capture group to detect variable name within ${VAR_NAME} syntax
+ */
+const VARIABLE_NAME_REGEX = /\${([a-z0-9_]*)}/gi;
+
+/**
+ * Utility class to handle replacing placeholder templates with environment variables
+ * Inspired by https://github.com/danday74/envsub
+ *
+ * @example
+ * ```ts
+ * import { envReplace, IEnvReplaceConfig } from "@idemsInternational/env-replace";
+ *
+ * // default config processes `.template.` files using process env
+ * const config: IEnvReplaceConfig = {}
+ * envReplace.replaceFiles(config)
+ * ```
+ * @see IEnvReplaceConfig for full configuration options
+ *
+ */
+class EnvReplaceClass {
+  private config: IEnvReplaceConfig;
+  private globalEnv: Record<string, any>;
+  private summary: { [inputName: string]: { [variableName: string]: any } };
+
+  /**
+   * Replace templated variables in files, as matched via config
+   * @returns list of variables replaced, with full output written to file
+   */
+  public async replaceFiles(config: Partial<IEnvReplaceConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.globalEnv = { ...process.env, ...this.config.envAdditional };
+    this.summary = {};
+
+    const { excludeFolders, cwd, rawGlobOptions, fileNameFind, fileNameReplace } = this.config;
+
+    const inputGlob = this.generateInputGlob();
+    const inputNames = await glob(inputGlob, {
+      ignore: excludeFolders,
+      cwd,
+      dot: true,
+      posix: true,
+      ...rawGlobOptions,
+    });
+
+    for (const inputName of inputNames) {
+      const filepath = resolve(cwd, inputName);
+      const sourceContent = readFileSync(filepath, { encoding: "utf-8" });
+
+      // determine variables to replace and values to replace with
+      const variablesToReplace = this.generateReplacementList(sourceContent);
+      const replacementEnv = this.generateReplacementEnv(variablesToReplace);
+
+      // handle replacement
+      const outputName = inputName.replace(fileNameFind, fileNameReplace);
+      this.summary[outputName] = {};
+
+      const replaceContent = this.generateReplaceContent(outputName, sourceContent, replacementEnv);
+      const outputPath = resolve(cwd, outputName);
+      writeFileSync(outputPath, replaceContent);
+    }
+    return this.summary;
+  }
+
+  private generateReplaceContent(
+    outputName: string,
+    sourceContent: string,
+    replacementEnv: Record<string, any>
+  ) {
+    let replaced = new String(sourceContent).toString();
+    for (const [variableName, replaceValue] of Object.entries(replacementEnv)) {
+      this.summary[outputName][variableName] = replaceValue;
+      replaced = replaced.replaceAll("${" + variableName + "}", replaceValue);
+    }
+    return replaced;
+  }
+
+  private generateReplacementEnv(variableNames: string[]) {
+    const { throwOnMissing } = this.config;
+    const replacementEnv: Record<string, any> = {};
+    for (const variableName of variableNames) {
+      let replaceValue = this.globalEnv[variableName];
+      replacementEnv[variableName] = replaceValue;
+      if (replaceValue === undefined) {
+        const msg = `No value for environment variable \${${variableName}}`;
+        if (throwOnMissing) throw new Error(msg);
+        else console.warn(msg);
+      }
+    }
+    return replacementEnv;
+  }
+
+  /**
+   * Extract list of all variables within file contents matching variable regex,
+   * convert to unique list and filter depending on include/exclude config
+   */
+  private generateReplacementList(contents: string) {
+    // generate list of all required matches in advance to ensure
+    // env variables exist as required
+    const matches = Array.from(contents.matchAll(VARIABLE_NAME_REGEX));
+    // use list of unique variable names for replacement
+    const uniqueVariables = [...new Set(matches.map((m) => m[1]))];
+    const { includeVariables, excludeVariables } = this.config;
+    // filter replacement list if named list of variables to include/exclude set
+    if (includeVariables.length > 0) {
+      return uniqueVariables.filter((name) => includeVariables.includes(name));
+    }
+    if (excludeVariables.length > 0) {
+      return uniqueVariables.filter((name) => !excludeVariables.includes(name));
+    }
+    return uniqueVariables;
+  }
+
+  /** Generate a glob matching pattern for all included paths with filename pattern match suffix */
+  private generateInputGlob() {
+    const { includeFolders, fileNameFind, rawGlobInput } = this.config;
+    // use raw glob override if provided
+    if (rawGlobInput) return rawGlobInput;
+    // create match patterns for all included folders and file name patters
+    const globs = [];
+    for (const folder of includeFolders) {
+      globs.push(`${folder}/*${fileNameFind}*`);
+    }
+    return globs.join("|");
+  }
+}
+const envReplace = new EnvReplaceClass();
+export { envReplace };

packages/@idemsInternational/env-replace/test/child/.template.env

@@ -0,0 +1,2 @@
+test_string="${STRING_VAR}"
+

packages/@idemsInternational/env-replace/test/test_basic.template.json

@@ -0,0 +1,8 @@
+{
+  "test_string": "${STRING_VAR}",
+  "test_global": "${GLOBAL_VAR}",
+  "test_excluded": "${EXCLUDED_VAR}",
+  "test_non_var":"example non var",
+  "test_boolean": ${BOOL_VAR},
+  "test_int": ${INT_VAR}
+}

packages/@idemsInternational/env-replace/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "moduleResolution": "node",
+    "lib": ["ESNext"],
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "outDir": "build",
+    "types": ["node"],
+    "typeRoots": ["./node_modules/@types", "./types/**"]
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["src/**/*.spec.ts"]
+}

packages/@idemsInternational/env-replace/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/spec",
+    "types": ["mocha", "chai", "node"]
+  },
+  "files": [],
+  "include": ["src/**/*.spec.ts", "src/**/*.d.ts"]
+}

packages/data-models/flowTypes.ts

@@ -337,11 +337,21 @@ export namespace FlowTypes {
     /** Keep a list of dynamic dependencies used within a template, by reference (e.g. {@local.var1 : ["text_1"]}) */
     _dynamicDependencies?: { [reference: string]: string[] };
     _translatedFields?: { [field: string]: any };
-    _evalContext?: { itemContext: any }; // force specific context variables when calculating eval statements (such as loop items)
+    _evalContext?: { itemContext: TemplateRowItemEvalContext }; // force specific context variables when calculating eval statements (such as loop items)
     __EMPTY?: any; // empty cells (can be removed after pr 679 merged)
   }
   export type IDynamicField = { [key: string]: IDynamicField | TemplateRowDynamicEvaluator[] };
 
+  export interface TemplateRowItemEvalContext {
+    // item metadata
+    _id: string;
+    _index: number;
+    _first: boolean;
+    _last: boolean;
+    // item data
+    [key: string]: any;
+  }
+
   type IDynamicPrefix = IAppConfig["DYNAMIC_PREFIXES"][number];
 
   /** Data passed back from regex match, e.g. expression @local.someField => type:local, fieldName: someField */