diff --git a/src/api/batch-import-export-api.ts b/src/api/batch-import-export-api.ts index c09f7ef..e74db3c 100644 --- a/src/api/batch-import-export-api.ts +++ b/src/api/batch-import-export-api.ts @@ -1,4 +1,8 @@ -import {PackageExportTransport} from "../interfaces/package-export-transport"; +import { + PackageExportTransport, + PackageKeyAndVersionPair, + VariableManifestTransport +} from "../interfaces/package-export-transport"; import {httpClientV2} from "../services/http-client-service.v2"; import {FatalError} from "../util/logger"; @@ -26,6 +30,22 @@ class BatchImportExportApi { throw new FatalError(`Problem getting active packages by keys: ${e}`); }); } + + public exportPackages(packageKeys: string[], withDependencies: boolean = false): Promise { + const queryParams = new URLSearchParams(); + packageKeys.forEach(packageKey => queryParams.append("packageKeys", packageKey)); + queryParams.set("withDependencies", withDependencies.toString()); + + return httpClientV2.getFile(`/package-manager/api/core/packages/export/batch?${queryParams.toString()}`).catch(e => { + throw new FatalError(`Problem exporting packages: ${e}`); + }); + } + + public findVariablesWithValuesByPackageKeysAndVersion(packagesByKeyAndVersion: PackageKeyAndVersionPair[]): Promise { + return httpClientV2.post("/package-manager/api/core/packages/export/batch/variables-with-assignments", packagesByKeyAndVersion).catch(e => { + throw new FatalError(`Problem exporting package variables: ${e}`); + }) + } } export const batchImportExportApi = BatchImportExportApi.INSTANCE; \ No newline at end of file diff --git a/src/api/variables-api.ts b/src/api/variables-api.ts index 09c0e63..8c57e4d 100644 --- a/src/api/variables-api.ts +++ b/src/api/variables-api.ts @@ -29,6 +29,12 @@ class VariablesApi { throw new FatalError(`Problem getting variables assignment values for type ${type}: ${e}`); }); } + + public getRuntimeVariableValues(packageKey: string): Promise { + return httpClientV2.get(`/package-manager/api/nodes/by-package-key/${packageKey}/variables/runtime-values`).catch(e => { + throw new FatalError(`Problem getting runtime variables of package ${packageKey}: ${e}`); + }); + } } export const variablesApi = VariablesApi.INSTANCE; \ No newline at end of file diff --git a/src/commands/config.command.ts b/src/commands/config.command.ts index fbf3230..8902497 100644 --- a/src/commands/config.command.ts +++ b/src/commands/config.command.ts @@ -9,4 +9,8 @@ export class ConfigCommand { await batchImportExportService.listActivePackages(flavors ?? []); } } + + public batchExportPackages(packageKeys: string[], withDependencies: boolean = false): Promise { + return batchImportExportService.batchExportPackages(packageKeys, withDependencies); + } } \ No newline at end of file diff --git a/src/content-cli-config.ts b/src/content-cli-config.ts index a24b7f7..679ca47 100644 --- a/src/content-cli-config.ts +++ b/src/content-cli-config.ts @@ -22,6 +22,21 @@ export class Config { return program; } + + public static export(program: CommanderStatic): CommanderStatic { + program + .command("export") + .description("Command to export package configs") + .option("-p, --profile ", "Profile which you want to use to list packages") + .requiredOption("--packageKeys ", "Keys of packages to export") + .option("--withDependencies", "Include variables and dependencies", "") + .action(async cmd => { + await new ConfigCommand().batchExportPackages(cmd.packageKeys, cmd.withDependencies); + process.exit(); + }); + + return program; + } } process.on("unhandledRejection", (e, promise) => { @@ -30,6 +45,7 @@ process.on("unhandledRejection", (e, promise) => { const loadAllCommands = () => { Config.list(commander); + Config.export(commander); commander.parse(process.argv); }; diff --git a/src/interfaces/package-export-transport.ts b/src/interfaces/package-export-transport.ts index efdc4b6..74d222e 100644 --- a/src/interfaces/package-export-transport.ts +++ b/src/interfaces/package-export-transport.ts @@ -1,4 +1,5 @@ -import {StudioComputeNodeDescriptor} from "./package-manager.interfaces"; +import {StudioComputeNodeDescriptor, VariableDefinition, VariablesAssignments} from "./package-manager.interfaces"; +import {SpaceTransport} from "./save-space.interface"; export interface DependencyTransport { key: string; @@ -17,4 +18,58 @@ export interface PackageExportTransport { dependencies: DependencyTransport[]; spaceId?: string; datamodels?: StudioComputeNodeDescriptor[]; +} + +export interface PackageManifestTransport { + packageKey: string; + flavor: string; + activeVersion: string; + space?: SpaceTransport; + variableAssignments?: VariablesAssignments[]; + dependenciesByVersion: Map; +} + +export interface VariableExportTransport { + key: string; + value: object; + type: string; + metadata: object; +} + +export interface VariableManifestTransport { + packageKey: string; + version: string; + variables?: VariableExportTransport[]; +} + +export interface PackageKeyAndVersionPair { + packageKey: string; + version: string; +} + +export interface NodeExportTransport { + key: string; + parentNodeKey: string; + packageNodeKey: string; + name: string; + type: string; + exportSerializationType: string; + serializedContent: string; + schemaVersion: number; + + unversionedMetadata: object; + versionedMetdata: object; + + invalidContent?: boolean; + serializedDocument: Buffer; +} + +export interface NodeSerializedContent { + variables: VariableDefinition[] +} + +export interface StudioPackageManifest { + packageKey: string; + space: Partial; + runtimeVariableAssignments: VariablesAssignments[]; } \ No newline at end of file diff --git a/src/interfaces/package-manager.interfaces.ts b/src/interfaces/package-manager.interfaces.ts index 26aa99b..07b3eb1 100644 --- a/src/interfaces/package-manager.interfaces.ts +++ b/src/interfaces/package-manager.interfaces.ts @@ -58,7 +58,7 @@ export interface VariablesAssignments { type: string; } -export interface VariableDefinitionWithValue { +export interface VariableDefinition { key: string; type: PackageManagerVariableType; description?: string; diff --git a/src/services/http-client-service.v2.ts b/src/services/http-client-service.v2.ts index 94195be..89cf763 100644 --- a/src/services/http-client-service.v2.ts +++ b/src/services/http-client-service.v2.ts @@ -22,6 +22,29 @@ class HttpClientServiceV2 { }) } + public async getFile(url: string): Promise { + return new Promise((resolve, reject) => { + axios.get(this.resolveUrl(url), { + headers: this.buildHeaders(contextService.getContext().profile), + responseType: "stream" + }).then(response => { + const data: Buffer[] = []; + response.data.on("data", (chunk: Buffer) => { + data.push(chunk); + }); + response.data.on("end", () => { + if (this.checkBadRequest(response.status)) { + this.handleBadRequest(response.status, response.data, reject); + } else { + this.handleResponseStreamData(Buffer.concat(data), resolve, reject); + } + }); + }).catch(err => { + this.handleError(err, resolve, reject); + }) + }); + } + public async postFile(url: string, body: any, parameters?: {}): Promise { return new Promise((resolve, reject) => { const formData = new FormData(); diff --git a/src/services/package-manager/batch-import-export-service.ts b/src/services/package-manager/batch-import-export-service.ts index c4e8486..6886598 100644 --- a/src/services/package-manager/batch-import-export-service.ts +++ b/src/services/package-manager/batch-import-export-service.ts @@ -1,9 +1,16 @@ import {batchImportExportApi} from "../../api/batch-import-export-api"; import {logger} from "../../util/logger"; import {v4 as uuidv4} from "uuid"; -import {PackageExportTransport} from "../../interfaces/package-export-transport"; +import { + PackageExportTransport, + PackageKeyAndVersionPair, + PackageManifestTransport, + VariableManifestTransport +} from "../../interfaces/package-export-transport"; import {FileService, fileService} from "../file-service"; import {studioService} from "../studio/studio.service"; +import {parse, stringify} from "../../util/yaml" +import AdmZip = require("adm-zip"); class BatchImportExportService { @@ -28,11 +35,68 @@ class BatchImportExportService { this.exportListOfPackages(packagesToExport); } + public async batchExportPackages(packageKeys: string[], withDependencies: boolean = false): Promise { + const exportedPackagesData: Buffer = await batchImportExportApi.exportPackages(packageKeys, withDependencies); + const exportedPackagesZip: AdmZip = new AdmZip(exportedPackagesData); + + const manifest: PackageManifestTransport[] = parse( + exportedPackagesZip.getEntry("manifest.yml").getData().toString() + ); + + const versionsByPackageKey = this.getVersionsByPackageKey(manifest); + + let exportedVariables = await this.getVersionedVariablesForPackagesWithKeys(versionsByPackageKey); + exportedVariables = studioService.fixConnectionVariables(exportedVariables); + exportedPackagesZip.addFile("variables.yml", Buffer.from(stringify(exportedVariables), "utf8")); + + const studioPackageKeys = manifest.filter(packageManifest => packageManifest.flavor === "STUDIO") + .map(packageManifest => packageManifest.packageKey); + + const studioData = await studioService.getStudioPackageManifests(studioPackageKeys); + exportedPackagesZip.addFile("studio.yml", Buffer.from(stringify(studioData), "utf8")); + + exportedPackagesZip.getEntries().forEach(entry => { + if (entry.name.endsWith(".zip") && studioPackageKeys.includes(entry.name.split("_")[0])) { + const updatedPackage = studioService.processPackageForExport(entry, exportedVariables); + + exportedPackagesZip.updateFile(entry, updatedPackage.toBuffer()); + } + }); + + const fileDownloadedMessage = "File downloaded successfully. New filename: "; + const filename = `export_${uuidv4()}.zip`; + exportedPackagesZip.writeZip(filename); + logger.info(fileDownloadedMessage + filename); + } + private exportListOfPackages(packages: PackageExportTransport[]): void { const filename = uuidv4() + ".json"; fileService.writeToFileWithGivenName(JSON.stringify(packages), filename); logger.info(FileService.fileDownloadedMessage + filename); } + + private getVersionsByPackageKey(manifests: PackageManifestTransport[]): Map { + const versionsByPackageKey = new Map(); + manifests.forEach(packageManifest => { + versionsByPackageKey.set(packageManifest.packageKey, Object.keys(packageManifest.dependenciesByVersion)); + }) + + return versionsByPackageKey; + } + + private getVersionedVariablesForPackagesWithKeys(versionsByPackageKey: Map): Promise { + const variableExportRequest: PackageKeyAndVersionPair[] = []; + versionsByPackageKey?.forEach((versions, key) => { + versions?.forEach(version => { + variableExportRequest.push({ + packageKey: key, + version: version, + }) + }) + }); + + return batchImportExportApi.findVariablesWithValuesByPackageKeysAndVersion(variableExportRequest) + } } export const batchImportExportService = new BatchImportExportService(); \ No newline at end of file diff --git a/src/services/studio/studio.service.ts b/src/services/studio/studio.service.ts index 35223b9..8899888 100644 --- a/src/services/studio/studio.service.ts +++ b/src/services/studio/studio.service.ts @@ -1,4 +1,11 @@ -import {PackageExportTransport} from "../../interfaces/package-export-transport"; +import { + NodeExportTransport, + NodeSerializedContent, + PackageExportTransport, + StudioPackageManifest, + VariableExportTransport, + VariableManifestTransport +} from "../../interfaces/package-export-transport"; import {packageApi} from "../../api/package-api"; import { PackageManagerVariableType, @@ -6,6 +13,13 @@ import { StudioComputeNodeDescriptor } from "../../interfaces/package-manager.interfaces"; import {dataModelService} from "../package-manager/datamodel-service"; +import {IZipEntry} from "adm-zip"; +import {parse, stringify} from "../../util/yaml"; +import AdmZip = require("adm-zip"); +import {nodeApi} from "../../api/node-api"; +import {variablesApi} from "../../api/variables-api"; +import {spaceApi} from "../../api/space-api"; +import {SpaceTransport} from "../../interfaces/save-space.interface"; class StudioService { @@ -22,6 +36,48 @@ class StudioService { return packagesToExport; } + public fixConnectionVariables(variables: VariableManifestTransport[]): VariableManifestTransport[] { + return variables.map(variableManifest => ({ + ...variableManifest, + variables: variableManifest.variables.map(variable => ({ + ...variable, + metadata: variable.type === PackageManagerVariableType.CONNECTION ? { + ...variable.metadata, + appName: variable.value["appName"] || "" + } : { + ...variable.metadata + } + })) + })); + } + + public async getStudioPackageManifests(studioPackageKeys: string[]): Promise { + return Promise.all(studioPackageKeys.map(async packageKey => { + const node = await nodeApi.findOneByKeyAndRootNodeKey(packageKey, packageKey); + const nodeSpace: SpaceTransport = await spaceApi.findOne(node.spaceId); + const variableAssignments = await variablesApi.getRuntimeVariableValues(packageKey); + + return { + packageKey: packageKey, + space: { + name: nodeSpace.name, + iconReference: nodeSpace.iconReference + }, + runtimeVariableAssignments: variableAssignments + } + })); + } + + public processPackageForExport(exportedPackage: IZipEntry, exportedVariables: VariableManifestTransport[]): AdmZip { + const packageZip = new AdmZip(exportedPackage.getData()); + packageZip.getEntries().forEach(entry => { + this.deleteFileIfTypeScenario(packageZip, entry); + this.fixConnectionVariablesIfRootNodeFile(packageZip, entry, exportedPackage.name, exportedVariables); + }); + + return packageZip; + } + private setSpaceIdForStudioPackages(packages: PackageExportTransport[], studioPackages: PackageWithVariableAssignments[]): PackageExportTransport[] { const studioPackageByKey = new Map(); studioPackages.forEach(pkg => studioPackageByKey.set(pkg.key, pkg)); @@ -52,6 +108,48 @@ class StudioService { } : pkg; }); } + + private deleteFileIfTypeScenario(packageZip: AdmZip, entry: IZipEntry): void { + if (entry.entryName.startsWith("nodes/") && entry.entryName.endsWith(".yml")) { + const node: NodeExportTransport = parse(entry.getData().toString()); + if (node.type === "SCENARIO") { + packageZip.deleteFile(entry); + } + } + } + + private fixConnectionVariablesIfRootNodeFile(packageZip: AdmZip, entry: IZipEntry, zipName: string, exportedVariables: VariableManifestTransport[]): void { + if (entry.name === "package.yml") { + const packageKeyAndVersion = zipName.replace(".zip", "").split("_"); + const connectionVariablesByKey = this.getConnectionVariablesByKeyForPackage(packageKeyAndVersion[0], packageKeyAndVersion[1], exportedVariables); + + if (connectionVariablesByKey.size) { + const exportedNode: NodeExportTransport = parse(entry.getData().toString()); + const nodeContent: NodeSerializedContent = parse(exportedNode.serializedContent); + + nodeContent.variables = nodeContent.variables.map(variable => ({ + ...variable, + metadata: variable.type === PackageManagerVariableType.CONNECTION ? + connectionVariablesByKey.get(variable.key).metadata : variable.metadata + })); + + exportedNode.serializedContent = stringify(nodeContent); + packageZip.updateFile(entry, Buffer.from(stringify(exportedNode))); + } + } + } + + private getConnectionVariablesByKeyForPackage(packageKey: string, packageVersion: string, variables: VariableManifestTransport[]): Map { + const variablesByKey = new Map(); + const packageVariables = variables.find(exportedVariable => exportedVariable.packageKey === packageKey && exportedVariable.version === packageVersion); + + if (packageVariables && packageVariables.variables.length) { + packageVariables.variables.filter(variable => variable.type === PackageManagerVariableType.CONNECTION) + .forEach(variable => variablesByKey.set(variable.key, variable)); + } + + return variablesByKey; + } } export const studioService = new StudioService(); \ No newline at end of file diff --git a/tests/config/config-export.spec.ts b/tests/config/config-export.spec.ts new file mode 100644 index 0000000..6f90fbe --- /dev/null +++ b/tests/config/config-export.spec.ts @@ -0,0 +1,410 @@ +import {ConfigUtils} from "../utls/config-utils"; +import { + DependencyTransport, NodeExportTransport, NodeSerializedContent, + PackageManifestTransport, + StudioPackageManifest, + VariableManifestTransport +} from "../../src/interfaces/package-export-transport"; +import {mockAxiosGet, mockAxiosPost, mockedPostRequestBodyByUrl} from "../utls/http-requests-mock"; +import {ConfigCommand} from "../../src/commands/config.command"; +import {PackageManagerApiUtils} from "../utls/package-manager-api.utils"; +import {mockWriteSync, testTransport} from "../jest.setup"; +import {FileService} from "../../src/services/file-service"; +import * as fs from "fs"; +import AdmZip = require("adm-zip"); + +import { parse, stringify } from "../../src/util/yaml"; +import { + PackageManagerVariableType, + VariableDefinition, + VariablesAssignments +} from "../../src/interfaces/package-manager.interfaces"; + +describe("Config export", () => { + + const firstSpace = PackageManagerApiUtils.buildSpaceTransport("space-1", "First space", "Icon1"); + const secondSpace = PackageManagerApiUtils.buildSpaceTransport("space-2", "Second space", "Icon2"); + + beforeEach(() => { + (fs.openSync as jest.Mock).mockReturnValue(100); + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/spaces/space-1", {...firstSpace}); + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/spaces/space-2", {...secondSpace}); + }); + + it("Should export studio file for studio packageKeys", async () => { + const manifest: PackageManifestTransport[] = []; + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO")); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO")); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-3", "TEST")); + const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, []); + + const firstStudioPackage = PackageManagerApiUtils.buildContentNodeTransport("key-1", "space-1"); + const firstPackageRuntimeVariable: VariablesAssignments = { + key: "varKey", + type: PackageManagerVariableType.PLAIN_TEXT, + value: "default-value" as unknown as object + }; + + const secondStudioPackage = PackageManagerApiUtils.buildContentNodeTransport("key-2", "space-2"); + + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch?packageKeys=key-1&packageKeys=key-2&packageKeys=key-3&withDependencies=true", exportedPackagesZip.toBuffer()); + mockAxiosPost("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments", []); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${firstStudioPackage.key}/${firstStudioPackage.key}`, firstStudioPackage); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${secondStudioPackage.key}/${secondStudioPackage.key}`, secondStudioPackage); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${firstStudioPackage.key}/variables/runtime-values`, [firstPackageRuntimeVariable]); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${secondStudioPackage.key}/variables/runtime-values`, []); + + await new ConfigCommand().batchExportPackages(["key-1", "key-2", "key-3"], true); + + const expectedFileName = testTransport.logMessages[0].message.split(FileService.fileDownloadedMessage)[1]; + expect(fs.openSync).toHaveBeenCalledWith(expectedFileName, expect.anything(), expect.anything()); + expect(mockWriteSync).toHaveBeenCalled(); + + const fileBuffer = mockWriteSync.mock.calls[0][1]; + const actualZip = new AdmZip(fileBuffer); + + const studioManifest: StudioPackageManifest[] = parse(actualZip.getEntry("studio.yml").getData().toString()); + expect(studioManifest).toHaveLength(2); + expect(studioManifest).toContainEqual({ + packageKey: firstStudioPackage.key, + space: { + name: firstSpace.name, + iconReference: firstSpace.iconReference + }, + runtimeVariableAssignments: [firstPackageRuntimeVariable] + }); + expect(studioManifest).toContainEqual({ + packageKey: secondStudioPackage.key, + space: { + name: secondSpace.name, + iconReference: secondSpace.iconReference + }, + runtimeVariableAssignments: [] + }); + }) + + it("Should export variables file with connection variables fixed", async () => { + const firstPackageDependencies = new Map(); + firstPackageDependencies.set("1.0.0", []); + + const secondPackageDependencies = new Map(); + secondPackageDependencies.set("1.0.0", []); + secondPackageDependencies.set("1.1.1", []); + + const manifest: PackageManifestTransport[] = []; + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO", firstPackageDependencies)); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO", secondPackageDependencies)); + + const firstPackageVariableDefinition: VariableDefinition[] = [ + { + key: "key1-var", + type: PackageManagerVariableType.DATA_MODEL, + runtime: false + }, + { + key: "key-1-connection", + type: PackageManagerVariableType.CONNECTION, + runtime: false + } + ]; + + const firstPackageNode = ConfigUtils.buildPackageNode("key-1", stringify({variables: firstPackageVariableDefinition})); + const firstPackageZip = ConfigUtils.buildExportPackageZip(firstPackageNode, [], "1.0.0"); + + const secondPackageVariableDefinition: VariableDefinition[] = [ + { + key: "key2-var", + type: PackageManagerVariableType.DATA_MODEL, + runtime: false + } + ]; + + const secondPackageNode = ConfigUtils.buildPackageNode("key-2", stringify({variables: secondPackageVariableDefinition})); + const secondPackageZip = ConfigUtils.buildExportPackageZip(secondPackageNode, [], "1.0.0"); + + const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, [firstPackageZip, secondPackageZip]); + + const exportedVariables: VariableManifestTransport[] = [ + { + packageKey: "key-1", + version: "1.0.0", + variables: [ + { + key: "key1-var", + type: PackageManagerVariableType.DATA_MODEL, + value: "dm-id" as unknown as object, + metadata: {} + }, + { + key: "key-1-connection", + type: PackageManagerVariableType.CONNECTION, + value: { + appName: "celonis", + connectionId: "connection-id" + } as unknown as object, + metadata: null + } + ] + }, + { + packageKey: "key-2", + version: "1.0.0", + variables: [ + { + key: "key2-var", + type: PackageManagerVariableType.DATA_MODEL, + value: "dm-id" as unknown as object, + metadata: {} + } + ] + }, + ]; + + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch?packageKeys=key-1&packageKeys=key-2&withDependencies=true", exportedPackagesZip.toBuffer()); + mockAxiosPost("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments", [...exportedVariables]); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${firstPackageNode.key}/${firstPackageNode.key}`, {...firstPackageNode, spaceId: "space-1"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${secondPackageNode.key}/${secondPackageNode.key}`, {...secondPackageNode, spaceId: "space-2"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${firstPackageNode.key}/variables/runtime-values`, []); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${secondPackageNode.key}/variables/runtime-values`, []); + + await new ConfigCommand().batchExportPackages(["key-1", "key-2"], true); + + const expectedFileName = testTransport.logMessages[0].message.split(FileService.fileDownloadedMessage)[1]; + expect(fs.openSync).toHaveBeenCalledWith(expectedFileName, expect.anything(), expect.anything()); + expect(mockWriteSync).toHaveBeenCalled(); + + const fileBuffer = mockWriteSync.mock.calls[0][1]; + const actualZip = new AdmZip(fileBuffer); + + const exportedVariablesFileContent: VariableManifestTransport[] = parse(actualZip.getEntry("variables.yml").getData().toString()); + expect(exportedVariablesFileContent).toHaveLength(2); + expect(exportedVariablesFileContent).toContainEqual({ + packageKey: "key-1", + version: "1.0.0", + variables: [ + { + key: "key1-var", + type: PackageManagerVariableType.DATA_MODEL, + value: "dm-id", + metadata: {} + }, + { + key: "key-1-connection", + type: PackageManagerVariableType.CONNECTION, + value: { + appName: "celonis", + connectionId: "connection-id" + }, + metadata: { + appName: "celonis" + } + } + ] + }); + expect(exportedVariablesFileContent).toContainEqual({ + packageKey: "key-2", + version: "1.0.0", + variables: [ + { + key: "key2-var", + type: PackageManagerVariableType.DATA_MODEL, + value: "dm-id", + metadata: {} + } + ] + }); + + const variableExportRequest = parse(mockedPostRequestBodyByUrl.get("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments")); + expect(variableExportRequest).toBeTruthy(); + expect(variableExportRequest).toHaveLength(3); + expect(variableExportRequest).toContainEqual({ + packageKey: "key-1", + version: "1.0.0" + }); + expect(variableExportRequest).toContainEqual({ + packageKey: "key-2", + version: "1.0.0" + }); + expect(variableExportRequest).toContainEqual({ + packageKey: "key-2", + version: "1.1.1" + }); + }) + + it("Should remove SCENARIO asset files of packages", async () => { + const manifest: PackageManifestTransport[] = []; + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO")); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO")); + + const firstPackageNode = ConfigUtils.buildPackageNode("key-1", ""); + const firstPackageScenarioChild = ConfigUtils.buildChildNode("child-1-scenario", firstPackageNode.key, "SCENARIO"); + const firstPackageTestChild = ConfigUtils.buildChildNode("child-2", firstPackageNode.key, "TEST"); + const firstPackageZip = ConfigUtils.buildExportPackageZip(firstPackageNode, [firstPackageScenarioChild, firstPackageTestChild], "1.0.0"); + + const secondPackageNode = ConfigUtils.buildPackageNode("key-2", ""); + const secondPackageScenarioChild = ConfigUtils.buildChildNode("child-3-scenario", secondPackageNode.key, "SCENARIO"); + const secondPackageTestChild = ConfigUtils.buildChildNode("child-4", secondPackageNode.key, "TEST"); + const secondPackageZip = ConfigUtils.buildExportPackageZip(secondPackageNode, [secondPackageScenarioChild, secondPackageTestChild], "1.0.0"); + + const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, [firstPackageZip, secondPackageZip]); + + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch?packageKeys=key-1&packageKeys=key-2&withDependencies=true", exportedPackagesZip.toBuffer()); + mockAxiosPost("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments", []); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${firstPackageNode.key}/${firstPackageNode.key}`, {...firstPackageNode, spaceId: "space-1"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${secondPackageNode.key}/${secondPackageNode.key}`, {...secondPackageNode, spaceId: "space-2"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${firstPackageNode.key}/variables/runtime-values`, []); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${secondPackageNode.key}/variables/runtime-values`, []); + + await new ConfigCommand().batchExportPackages(["key-1", "key-2"], true); + + const expectedFileName = testTransport.logMessages[0].message.split(FileService.fileDownloadedMessage)[1]; + expect(fs.openSync).toHaveBeenCalledWith(expectedFileName, expect.anything(), expect.anything()); + expect(mockWriteSync).toHaveBeenCalled(); + + const fileBuffer = mockWriteSync.mock.calls[0][1]; + const actualZip = new AdmZip(fileBuffer); + + const firstPackageExportedZip = new AdmZip(actualZip.getEntry("key-1_1.0.0.zip").getData()); + expect(firstPackageExportedZip.getEntry("nodes/child-1-scenario.yml")).toBeNull(); + expect(firstPackageExportedZip.getEntry("nodes/child-2.yml").getData().toString()).toEqual(stringify(firstPackageTestChild)); + + const secondPackageExportedZip = new AdmZip(actualZip.getEntry("key-2_1.0.0.zip").getData()); + expect(secondPackageExportedZip.getEntry("nodes/child-3-scenario.yml")).toBeNull(); + expect(secondPackageExportedZip.getEntry("nodes/child-4.yml").getData().toString()).toEqual(stringify(secondPackageTestChild)); + }) + + it("Should add appName to metadata for CONNECTION variables of package.yml files", async () => { + const manifest: PackageManifestTransport[] = []; + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "STUDIO")); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "STUDIO")); + + const firstPackageVariableDefinition: VariableDefinition[] = [ + { + key: "key1-var", + type: PackageManagerVariableType.DATA_MODEL, + runtime: false + }, + { + key: "key-1-connection", + type: PackageManagerVariableType.CONNECTION, + runtime: false + } + ]; + + const firstPackageNode = ConfigUtils.buildPackageNode("key-1", stringify({variables: firstPackageVariableDefinition})); + const firstPackageZip = ConfigUtils.buildExportPackageZip(firstPackageNode, [], "1.0.0"); + + const secondPackageVariableDefinition: VariableDefinition[] = [ + { + key: "key2-var", + type: PackageManagerVariableType.CONNECTION, + runtime: false, + metadata: { + appName: "celonis" + } + } + ]; + + const secondPackageNode = ConfigUtils.buildPackageNode("key-2", stringify({variables: secondPackageVariableDefinition})); + const secondPackageZip = ConfigUtils.buildExportPackageZip(secondPackageNode, [], "1.0.0"); + + const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, [firstPackageZip, secondPackageZip]); + + const exportedVariables: VariableManifestTransport[] = [ + { + packageKey: "key-1", + version: "1.0.0", + variables: [ + { + key: "key1-var", + type: PackageManagerVariableType.DATA_MODEL, + value: "dm-id" as unknown as object, + metadata: {} + }, + { + key: "key-1-connection", + type: PackageManagerVariableType.CONNECTION, + value: { + appName: "celonis", + connectionId: "connection-id" + } as unknown as object, + metadata: null + } + ] + }, + { + packageKey: "key-2", + version: "1.0.0", + variables: [ + { + key: "key2-var", + type: PackageManagerVariableType.CONNECTION, + value: { + appName: "celonis", + connectionId: "connection-id" + } as unknown as object, + metadata: { + appName: "celonis" + } + } + ] + }, + ]; + + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch?packageKeys=key-1&packageKeys=key-2&withDependencies=true", exportedPackagesZip.toBuffer()); + mockAxiosPost("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments", exportedVariables); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${firstPackageNode.key}/${firstPackageNode.key}`, {...firstPackageNode, spaceId: "space-1"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/${secondPackageNode.key}/${secondPackageNode.key}`, {...secondPackageNode, spaceId: "space-2"}); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${firstPackageNode.key}/variables/runtime-values`, []); + mockAxiosGet(`https://myTeam.celonis.cloud/package-manager/api/nodes/by-package-key/${secondPackageNode.key}/variables/runtime-values`, []); + + await new ConfigCommand().batchExportPackages(["key-1", "key-2"], true); + + const expectedFileName = testTransport.logMessages[0].message.split(FileService.fileDownloadedMessage)[1]; + expect(fs.openSync).toHaveBeenCalledWith(expectedFileName, expect.anything(), expect.anything()); + expect(mockWriteSync).toHaveBeenCalled(); + + const fileBuffer = mockWriteSync.mock.calls[0][1]; + const actualZip = new AdmZip(fileBuffer); + + const firstPackageExportedZip = new AdmZip(actualZip.getEntry("key-1_1.0.0.zip").getData()); + const firstPackageExportedNode: NodeExportTransport = parse(firstPackageExportedZip.getEntry("package.yml").getData().toString()); + expect(firstPackageExportedNode).toBeTruthy(); + const firstPackageContent: NodeSerializedContent = parse(firstPackageExportedNode.serializedContent); + expect(firstPackageContent.variables).toHaveLength(2); + expect(firstPackageContent.variables).toEqual([ + { + ...firstPackageVariableDefinition[0], + }, + { + ...firstPackageVariableDefinition[1], + metadata: { + appName: "celonis" + } + } + ]); + + const secondPackageExportedZip = new AdmZip(actualZip.getEntry("key-2_1.0.0.zip").getData()); + const secondPackageExportedNode: NodeExportTransport = parse(secondPackageExportedZip.getEntry("package.yml").getData().toString()); + expect(secondPackageExportedNode).toBeTruthy(); + const secondPackageContent: NodeSerializedContent = parse(secondPackageExportedNode.serializedContent); + expect(secondPackageContent.variables).toHaveLength(1); + expect(secondPackageContent.variables).toEqual([...secondPackageVariableDefinition]); + }) + + it("Should export by packageKeys without dependencies", async () => { + const manifest: PackageManifestTransport[] = []; + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-1", "TEST")); + manifest.push(ConfigUtils.buildManifestForKeyAndFlavor("key-2", "TEST")); + const exportedPackagesZip = ConfigUtils.buildBatchExportZip(manifest, []); + + mockAxiosGet("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch?packageKeys=key-1&packageKeys=key-2&packageKeys=key-3&withDependencies=false", exportedPackagesZip.toBuffer()); + mockAxiosPost("https://myTeam.celonis.cloud/package-manager/api/core/packages/export/batch/variables-with-assignments", []); + + await new ConfigCommand().batchExportPackages(["key-1", "key-2", "key-3"], false); + + const expectedFileName = testTransport.logMessages[0].message.split(FileService.fileDownloadedMessage)[1]; + expect(fs.openSync).toHaveBeenCalledWith(expectedFileName, expect.anything(), expect.anything()); + expect(mockWriteSync).toHaveBeenCalled(); + }) +}) \ No newline at end of file diff --git a/tests/jest.setup.ts b/tests/jest.setup.ts index 1f18a63..56c006b 100644 --- a/tests/jest.setup.ts +++ b/tests/jest.setup.ts @@ -10,8 +10,12 @@ jest.mock('fs'); const mockWriteFileSync = jest.fn(); (fs.writeFileSync as jest.Mock).mockImplementation(mockWriteFileSync); +const mockWriteSync = jest.fn(); +(fs.writeSync as jest.Mock).mockImplementation(mockWriteSync); + afterEach(() => { mockWriteFileSync.mockClear(); + mockWriteSync.mockClear(); }) beforeAll(() => { @@ -25,4 +29,4 @@ beforeEach(() => { logger.add(testTransport); }) -export {testTransport, mockWriteFileSync}; \ No newline at end of file +export {testTransport, mockWriteFileSync, mockWriteSync}; \ No newline at end of file diff --git a/tests/utls/config-utils.ts b/tests/utls/config-utils.ts new file mode 100644 index 0000000..1854a30 --- /dev/null +++ b/tests/utls/config-utils.ts @@ -0,0 +1,81 @@ +import AdmZip = require("adm-zip"); +import { + DependencyTransport, + NodeExportTransport, + PackageManifestTransport +} from "../../src/interfaces/package-export-transport"; +import {stringify} from "../../src/util/yaml"; + +export class ConfigUtils { + + public static buildBatchExportZip(manifest: PackageManifestTransport[], packageZips: AdmZip[]): AdmZip { + + const zipExport = new AdmZip(); + zipExport.addFile("manifest.yml", Buffer.from(stringify(manifest))); + packageZips.forEach(packageZip => { + const fileName = `${packageZip.getZipComment()}.zip` + packageZip.addZipComment("") + zipExport.addFile(fileName, packageZip.toBuffer()); + }) + + return zipExport; + } + + public static buildExportPackageZip(packageNode: NodeExportTransport, childNodes: NodeExportTransport[], version: string): AdmZip { + const zipExport = new AdmZip(); + + zipExport.addFile("package.yml", Buffer.from(stringify(packageNode))); + zipExport.addFile("nodes/", Buffer.alloc(0)); + + childNodes.forEach(child => { + zipExport.addFile(`nodes/${child.key}.yml`, Buffer.from(stringify(child))); + }); + + zipExport.addZipComment(`${packageNode.key}_${version}`); + + return zipExport; + } + + public static buildManifestForKeyAndFlavor(key: string, flavor: string, dependenciesByVersion?: Map): PackageManifestTransport { + return { + packageKey: key, + flavor: flavor, + activeVersion: "", + dependenciesByVersion: dependenciesByVersion ?? {} as Map + }; + } + + public static buildPackageNode(key: string, serializedContent: string): NodeExportTransport { + return { + key, + parentNodeKey: key, + packageNodeKey: key, + name: "name", + type: "PACKAGE", + exportSerializationType: "YAML", + serializedContent, + schemaVersion: 1, + unversionedMetadata: {}, + versionedMetdata: {}, + invalidContent: false, + serializedDocument: null + }; + } + + public static buildChildNode(key: string, parentKey: string, type: string): NodeExportTransport { + return { + key, + parentNodeKey: parentKey, + packageNodeKey: parentKey, + name: "name", + type: type, + exportSerializationType: "YAML", + serializedContent: "", + schemaVersion: 1, + unversionedMetadata: {}, + versionedMetdata: {}, + invalidContent: false, + serializedDocument: null + }; + } +} \ No newline at end of file diff --git a/tests/utls/http-requests-mock.ts b/tests/utls/http-requests-mock.ts index 49df3c3..867e31d 100644 --- a/tests/utls/http-requests-mock.ts +++ b/tests/utls/http-requests-mock.ts @@ -1,21 +1,50 @@ import axios from "axios"; +import {Readable} from "stream"; const mockedResponseByUrl = new Map(); +const mockedPostRequestBodyByUrl = new Map(); const mockAxiosGet = (url: string, responseData: any) => { mockedResponseByUrl.set(url, responseData); (axios.get as jest.Mock).mockImplementation(requestUrl => { if (mockedResponseByUrl.has(requestUrl)) { - return Promise.resolve({ data: mockedResponseByUrl.get(requestUrl) }); + const response = { data: mockedResponseByUrl.get(requestUrl) }; + + if (response.data instanceof Buffer) { + const readableStream = new Readable(); + readableStream.push(response.data) + readableStream.push(null); + return Promise.resolve({ + data: readableStream, + }); + } else { + return Promise.resolve(response); + } } else { fail("API call not mocked.") } }); }; +const mockAxiosPost = (url: string, responseData: any) => { + mockedResponseByUrl.set(url, responseData); + + (axios.post as jest.Mock).mockImplementation((requestUrl: string, data: any) => { + if (mockedResponseByUrl.has(requestUrl)) { + const response = { data: mockedResponseByUrl.get(requestUrl) }; + mockedPostRequestBodyByUrl.set(requestUrl, data); + + return Promise.resolve(response); + } else { + fail("API call not mocked.") + } + }) +} + afterEach(() => { mockedResponseByUrl.clear(); + mockedPostRequestBodyByUrl.clear(); }) -export {mockAxiosGet}; \ No newline at end of file +export {mockAxiosGet, mockAxiosPost, mockedPostRequestBodyByUrl}; \ No newline at end of file diff --git a/tests/utls/package-manager-api.utils.ts b/tests/utls/package-manager-api.utils.ts index be92715..163f3c3 100644 --- a/tests/utls/package-manager-api.utils.ts +++ b/tests/utls/package-manager-api.utils.ts @@ -1,4 +1,5 @@ import {PackageExportTransport} from "../../src/interfaces/package-export-transport"; +import {SpaceTransport} from "../../src/interfaces/save-space.interface"; import {ContentNodeTransport} from "../../src/interfaces/package-manager.interfaces"; export class PackageManagerApiUtils { @@ -29,4 +30,12 @@ export class PackageManagerApiUtils { spaceId } } + + public static buildSpaceTransport = (id: string, name: string = "space-name", iconReference: string = "icon"): SpaceTransport => { + return { + id, + name, + iconReference, + }; + } } \ No newline at end of file