Merge pull request #141 from celonis/fisgeci/tn-x-add-datamodel-mapping

Tn-4460: Add dataModelMappings option to import packages command

DOCUMENTATION.md

@@ -9,6 +9,8 @@
         -   [Pull package for EMS Store](#pull-package-for-ems-store)
     -   [Pull/Push individual assets from/to Studio](#pullpush-individual-assets-fromto-studio)
     -   [Overwrite Package In Studio](#overwrite-package-in-studio)
+    -   [Batch Export packages from Studio](#export-multiple-packages-at-once-from-studio)
+    -   [Batch import packages into Studio](#importing-multiple-packages-into-studio)
     -   [List all spaces in Studio](#list-all-spaces-in-studio)
     -   [List all packages in Studio](#list-all-packages-in-studio)
     -   [Asset options for Analysis](#asset-options-for-analysis)
@@ -258,28 +260,28 @@ When you use overwrite the following is to be taken into consideration:
 content-cli push package -p my-profile-name --spaceKey my-space -f <path-to-my-local-package> --overwrite
 ```
 
-### Export multiple packages at once from Studio
+### Batch export packages from Studio
 
-You can use the export packages command to export multiple packages at once from studio.
+You can use the export packages command to batch export packages from studio.
 
 ```
-//Exporting multiple packages at once
+//Batch export packages
 content-cli export packages -p <profileName> --packageKeys <package1> <package2>
 ```
 
 You can use the --includeDependencies flag to also export the dependencies of the specified packages.
 
 ```
-//Exporting multiple packages at once with dependencies
+//Batch export packages with dependencies
 content-cli export packages -p <profileName> --packageKeys <package1> <package2> --includeDependencies
 ```
 
-### Importing multiple packages into Studio
+### Batch import packages into Studio
 
-You can use the `import packages` command to import multiple packages that were exported using `export packages` at once into studio.
+You can use the `import packages` command to batch import packages that were exported using `export packages` at once into studio.
 
 ```
-//Importing multiple packages at once
+//Batch import packages
 content-cli import packages -p <profileName> --file <exportedPackagesFile>
 ```
 
@@ -290,6 +292,14 @@ You can also use the `--spaceMappings` flag to provide a mapping of packages to
 content-cli import packages -p <profileName> --file <exportedPackagesFile> --spaceMappings <packageKey1>:<targetSpaceKey1> <packageKey2>:<targetSpaceKey2> ...
 ```
 
+If you want to update dataModel variables. You can do so by using the --dataModelMappingsFile option and providng the output file from
+the data pool import command.
+
+```
+// Example usage of dataModelMappingsFile
+content-cli import packages -p <profileName> --file <exportedPackagesFile> --dataModelMappingsFile <dataModelMappingsFile>
+```
+
 ### List all spaces in Studio
 With this command you can retrieve a list of all spaces within a team.
 The command takes your permissions into consideration and only lists the

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celonis/content-cli",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "CLI Tool to help manage content in Celonis EMS",
   "main": "content-cli.js",
   "bin": {

src/api/variables-api.ts

@@ -0,0 +1,18 @@
+import {
+    ContentNodeTransport,
+    VariableDefinitionWithValue, VariablesAssignments
+} from "../interfaces/package-manager.interfaces";
+import {httpClientV2} from "../services/http-client-service.v2";
+import {FatalError} from "../util/logger";
+
+class VariablesApi {
+    public static readonly INSTANCE = new VariablesApi();
+
+    public async assignVariableValues(packageKey: string, variablesAssignments: VariablesAssignments[]): Promise<ContentNodeTransport[]> {
+        return httpClientV2.post(`/package-manager/api/nodes/by-package-key/${packageKey}/variables/values`, variablesAssignments).catch(e => {
+            throw new FatalError(`Problem updating variables of package ${packageKey}: ${e}`);
+        });
+    }
+}
+
+export const variablesApi = VariablesApi.INSTANCE;

src/commands/package.command.ts

@@ -48,7 +48,7 @@ export class PackageCommand {
         await packageService.batchExportPackages(packageKeys, includeDependencies);
     }
 
-    public async batchImportPackages(spaceMappings: string[], exportedPackagesFile: string): Promise<void> {
-        await packageService.batchImportPackages(spaceMappings ?? [], exportedPackagesFile);
+    public async batchImportPackages(spaceMappings: string[], dataModelMappingsFilePath: string, exportedPackagesFile: string): Promise<void> {
+        await packageService.batchImportPackages(spaceMappings ?? [], dataModelMappingsFilePath, exportedPackagesFile);
     }
 }

src/content-cli-import.ts

@@ -15,9 +15,10 @@ export class Import {
                 "--spaceMappings <spaceMappings...>",
                 "List of mappings for importing packages to different target spaces. Mappings should follow format 'packageKey:targetSpaceKey'"
             )
+            .option("--dataModelMappingsFile <dataModelMappingsFile>", "DataModel variable mappings file path")
             .requiredOption("-f, --file <file>", "Exported packages file (relative path)")
             .action(async cmd => {
-                await new PackageCommand().batchImportPackages(cmd.spaceMappings, cmd.file);
+                await new PackageCommand().batchImportPackages(cmd.spaceMappings, cmd.dataModelMappingsFile, cmd.file);
                 process.exit();
             });
 

src/interfaces/data-pool-manager.interfaces.ts

@@ -26,5 +26,5 @@ export declare class DataPoolPageTransport {
 }
 
 export declare class DataPoolInstallVersionReport {
-    dataModelIdMappings: Map<String, String>;
+    dataModelIdMappings: Map<string, string>;
 }

src/interfaces/package-manager.interfaces.ts

@@ -20,6 +20,7 @@ export interface ActivatePackageTransport {
     publishMessage: string;
     nodeIdsToExclude: string[];
 }
+
 export interface DataModelTransport {
     id: string;
     name: string,
@@ -56,6 +57,15 @@ export interface VariablesAssignments {
     type: string;
 }
 
+export interface VariableDefinitionWithValue {
+    key: string;
+    type: string;
+    description?: string;
+    source?: string;
+    runtime?: boolean;
+    metadata?: object;
+}
+
 export interface PackageHistoryTransport {
     id: string;
     key: string;

src/services/file-service.ts

@@ -24,11 +24,17 @@ export class FileService {
         return Promise.all(manifest);
     }
 
+    public async readFileToJson<T>(fileName: string): Promise<T> {
+        const fileContent = this.readFile(fileName);
+
+        return JSON.parse(fileContent);
+    }
+
     public readFile(filename: string): string {
         if (!fs.existsSync(path.resolve(process.cwd(), filename))) {
             logger.error(new FatalError(`The provided file '${filename}' does not exit`));
         }
-        return fs.readFileSync(path.resolve(process.cwd(), filename), { encoding: "utf-8" });
+        return fs.readFileSync(path.resolve(process.cwd(), filename), {encoding: "utf-8"});
     }
 
     private getSerializedFileContent(data: any): string {

src/services/package-manager/package-service.ts

@@ -15,6 +15,8 @@ import * as path from "path";
 import {tmpdir} from "os";
 import {SpaceTransport} from "../../interfaces/save-space.interface";
 import {ManifestDependency, ManifestNodeTransport} from "../../interfaces/manifest-transport";
+import {DataPoolInstallVersionReport} from "../../interfaces/data-pool-manager.interfaces";
+import {SemanticVersioning} from "../../util/semantic-versioning";
 import {stringify} from "../../util/yaml";
 
 class PackageService {
@@ -70,14 +72,21 @@ class PackageService {
         this.exportListOfPackages(nodesListToExport, fieldsToInclude);
     }
 
-    public async batchImportPackages(spaceMappings: string[], exportedPackagesFile: string): Promise<void> {
+    public async batchImportPackages(spaceMappings: string[], dataModelMappingsFilePath: string, exportedPackagesFile: string): Promise<void> {
         exportedPackagesFile = exportedPackagesFile + (exportedPackagesFile.includes(".zip") ? "" : ".zip");
         const zip = new AdmZip(exportedPackagesFile);
         const importedFilePath = path.resolve(tmpdir(), "export_" + uuidv4());
         await fs.mkdirSync(importedFilePath);
         await zip.extractAllTo(importedFilePath);
 
         const manifestNodes = await fileService.readManifestFile(importedFilePath);
+
+        let dmTargetIdsBySourceIds: Map<string, string>;
+        if (dataModelMappingsFilePath) {
+            const dataModelMappings: DataPoolInstallVersionReport = await fileService.readFileToJson<DataPoolInstallVersionReport>(dataModelMappingsFilePath);
+            dmTargetIdsBySourceIds = new Map(Object.entries(dataModelMappings.dataModelIdMappings));
+        }
+
         manifestNodes.map(node => node.dependenciesByVersion = new Map(Object.entries(node.dependenciesByVersion)));
 
         const importedVersionsByNodeKey = new Map<string, string[]>();
@@ -97,7 +106,7 @@ class PackageService {
 
         const draftIdsByPackageKeyAndVersion = new Map<string, string>();
         for (const node of manifestNodes) {
-            await this.importPackage(node, manifestNodes, sourceToTargetVersionsByNodeKey, customSpacesMap, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath)
+            await this.importPackage(node, manifestNodes, sourceToTargetVersionsByNodeKey, customSpacesMap, dmTargetIdsBySourceIds, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath)
         }
     }
 
@@ -121,49 +130,40 @@ class PackageService {
                                 manifestNodes: ManifestNodeTransport[],
                                 sourceToTargetVersionsByNodeKey: Map<string, Map<string, string>>,
                                 spaceMappings: Map<string, string>,
+                                dmTargetIdsBySourceIds: Map<string, string>,
                                 importedVersionsByNodeKey: Map<string, string[]>,
                                 draftIdsByPackageKeyAndVersion: Map<string, string>,
                                 importedFilePath: string): Promise<void> {
         const importedPackageVersion = importedVersionsByNodeKey.get(packageToImport.packageKey) ?? [];
-        const versionsOfPackage = [...packageToImport.dependenciesByVersion.keys()].sort((k1, k2) => this.compareVersions(k1, k2)).filter(version => !importedPackageVersion.includes(version));
+        const versionsOfPackage = [...packageToImport.dependenciesByVersion.keys()].sort((k1, k2) => {
+            const version1 = new SemanticVersioning(k1);
+            const version2 = new SemanticVersioning(k1);
+            return version1.isGreaterThan(version2) ? 1 : -1;
+        }).filter(version => !importedPackageVersion.includes(version));
 
         for (const version of versionsOfPackage) {
             try {
-                await this.importPackageVersion(packageToImport, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath, version);
+                await this.importPackageVersion(packageToImport, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, dmTargetIdsBySourceIds, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath, version);
             } catch (e) {
                 logger.error(`Problem import package with key: ${packageToImport.packageKey} ${version} ${e}`);
             }
         }
     }
 
-    private compareVersions(version1: string, version2: string): number {
-        const splitVersion1 = version1.split(".");
-        const splitVersion2 = version2.split(".");
-
-        const majorVersion1 = splitVersion1[0];
-        const majorVersion2 = splitVersion2[0];
-
-        const minorVersion1 = splitVersion1[1];
-        const minorVersion2 = splitVersion2[1];
-
-        const patchVersion1 = splitVersion1[2];
-        const patchVersion2 = splitVersion2[2];
-
-        const firstVersionIsGreaterThanFirst = (majorVersion1 > majorVersion2) || (minorVersion1 > minorVersion2) || (patchVersion1 > patchVersion2);
-        return firstVersionIsGreaterThanFirst ? 1 : -1;
-    }
-
     private async importPackageVersion(packageToImport: ManifestNodeTransport,
                                        manifestNodes: ManifestNodeTransport[],
                                        sourceToTargetVersionsByNodeKey: Map<string, Map<string, string>>,
                                        spaceMappings: Map<string, string>,
+                                       dmTargetIdsBySourceIds: Map<string, string>,
                                        importedVersionsByNodeKey: Map<string, string[]>,
                                        draftIdsByPackageKeyAndVersion: Map<string, string>,
                                        importedFilePath: string,
                                        versionOfPackageBeingImported: string): Promise<void> {
         if (packageToImport.dependenciesByVersion.get(versionOfPackageBeingImported).length) {
             const dependenciesOfPackageVersion = packageToImport.dependenciesByVersion.get(versionOfPackageBeingImported);
-            await this.importDependencyPackages(dependenciesOfPackageVersion, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath);
+            await this.importDependencyPackages(dependenciesOfPackageVersion, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, dmTargetIdsBySourceIds,
+                importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath
+            );
         }
 
         if (this.checkIfPackageVersionHasBeenImported(packageToImport.packageKey, versionOfPackageBeingImported, importedVersionsByNodeKey)) {
@@ -185,6 +185,16 @@ class PackageService {
 
         nodeInTargetTeam = await nodeApi.findOneByKeyAndRootNodeKey(packageToImport.packageKey, packageToImport.packageKey);
 
+        if (this.isLatestVersion(versionOfPackageBeingImported, [...packageToImport.dependenciesByVersion.keys()])) {
+            const variableAssignments = packageToImport.variables
+                .filter(variable => variable.type === "DATA_MODEL").map(variable => {
+                    variable.value = dmTargetIdsBySourceIds.get(variable.value.toString()) as unknown as object;
+                    return variable;
+                })
+
+            await variableService.assignVariableValues(nodeInTargetTeam.key, variableAssignments);
+        }
+
         draftIdsByPackageKeyAndVersion.set(`${nodeInTargetTeam.key}_${versionOfPackageBeingImported}`, nodeInTargetTeam.workingDraftId);
 
         await this.updateDependencyVersions(packageToImport, versionOfPackageBeingImported, sourceToTargetVersionsByNodeKey, draftIdsByPackageKeyAndVersion);
@@ -205,10 +215,27 @@ class PackageService {
         logger.info(`Imported package with key: ${packageToImport.packageKey} ${versionOfPackageBeingImported} successfully. New version: ${mappedVersion}`)
     }
 
+    private isLatestVersion(packageVersion: string, allPackageVersions: string[]): boolean {
+        let isLatestVersion = true;
+
+        for (const version of allPackageVersions) {
+            const version1 = new SemanticVersioning(packageVersion);
+            const version2 = new SemanticVersioning(version);
+
+            if (version2.isGreaterThan(version1)) {
+                isLatestVersion = false;
+                break;
+            }
+        }
+
+        return isLatestVersion;
+    }
+
     private async importDependencyPackages(dependenciesToImport: ManifestDependency[],
                                            manifestNodes: ManifestNodeTransport[],
                                            sourceToTargetVersionsByNodeKey: Map<string, Map<string, string>>,
                                            spaceMappings: Map<string, string>,
+                                           dmTargetIdsBySourceIds: Map<string, string>,
                                            importedVersionsByNodeKey: Map<string, string[]>,
                                            draftIdsByPackageKeyAndVersion: Map<string, string>,
                                            importedFilePath: string): Promise<void> {
@@ -218,7 +245,7 @@ class PackageService {
             }
 
             const dependentPackage = manifestNodes.find((node) => node.packageKey === dependency.key);
-            await this.importPackage(dependentPackage, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath);
+            await this.importPackage(dependentPackage, manifestNodes, sourceToTargetVersionsByNodeKey, spaceMappings, dmTargetIdsBySourceIds, importedVersionsByNodeKey, draftIdsByPackageKeyAndVersion, importedFilePath);
         }
     }
 

src/services/package-manager/variable-service.ts

@@ -1,12 +1,21 @@
 import {packageApi} from "../../api/package-api";
-import {PackageWithVariableAssignments, VariablesAssignments} from "../../interfaces/package-manager.interfaces";
+import {
+    PackageWithVariableAssignments,
+    VariableDefinitionWithValue,
+    VariablesAssignments
+} from "../../interfaces/package-manager.interfaces";
 import {BatchExportNodeTransport} from "../../interfaces/batch-export-node-transport";
+import {variablesApi} from "../../api/variables-api";
 
 class VariableService {
 
     public async getVariableAssignmentsForNodes(nodes: BatchExportNodeTransport[]): Promise<PackageWithVariableAssignments[]> {
         return await packageApi.findAllPackagesWithVariableAssignments();
     }
+
+    public async assignVariableValues(packageKey: string, variablesAssignments: VariablesAssignments[]): Promise<void> {
+        await variablesApi.assignVariableValues(packageKey, variablesAssignments);
+    }
 }
 
 export const variableService = new VariableService();

src/util/semantic-versioning.ts

@@ -0,0 +1,23 @@
+export class SemanticVersioning {
+    private version: string;
+
+    constructor(version: string) {
+        this.version = version;
+    }
+
+    public isGreaterThan(versionToCompare: SemanticVersioning): boolean {
+        const splitVersion1 = this.version.split(".");
+        const splitVersion2 = versionToCompare.version.split(".");
+
+        const majorVersion1 = splitVersion1[0];
+        const majorVersion2 = splitVersion2[0];
+
+        const minorVersion1 = splitVersion1[1];
+        const minorVersion2 = splitVersion2[1];
+
+        const patchVersion1 = splitVersion1[2];
+        const patchVersion2 = splitVersion2[2];
+
+        return (majorVersion1 > majorVersion2) || (minorVersion1 > minorVersion2) || (patchVersion1 > patchVersion2);
+    }
+}