diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d9180a36a..5f5154cb1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -200,20 +200,8 @@ jobs: node-version: ${{ env.node-version }} - name: Install NPM dependencies run: npm ci - - name: Mock Docker Images for CDK Tests - run: | - image=$(docker images --format '{{.Repository}}:{{.Tag}}' | head -n 1) - echo "Using image for mocked tags: $image" - docker tag $image migrations/capture_proxy:latest - docker tag $image migrations/capture_proxy_es:latest - docker tag $image opensearchproject/opensearch:2 - docker tag $image migrations/elasticsearch_searchguard:latest - docker tag $image docker.io/apache/kafka:3.7.0 - docker tag $image migrations/migration_console:latest - docker tag $image migrations/reindex_from_snapshot:latest - docker tag $image migrations/traffic_replayer:latest - name: Run CDK Jest Tests (using mocked images) - run: npm run test:withoutBuildImages + run: npm run test link-checker: runs-on: ubuntu-latest diff --git a/deployment/cdk/opensearch-service-migration/jest.config.js b/deployment/cdk/opensearch-service-migration/jest.config.js index 8e8c1d0d8..a55fae2c1 100644 --- a/deployment/cdk/opensearch-service-migration/jest.config.js +++ b/deployment/cdk/opensearch-service-migration/jest.config.js @@ -9,4 +9,5 @@ module.exports = { transform: { '^.+\\.tsx?$': 'ts-jest' }, + setupFilesAfterEnv: ['/test/jest.setup.ts'], }; diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index 5d9b7d1ed..e6d55869e 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -6,7 +6,11 @@ import { IStringParameter, StringParameter } from "aws-cdk-lib/aws-ssm"; import * as forge from 'node-forge'; import { ClusterYaml } from "./migration-services-yaml"; import { CdkLogger } from "./cdk-logger"; - +import { mkdtempSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { DockerImageAsset } from "aws-cdk-lib/aws-ecr-assets"; +import { execSync } from 'child_process'; export function getSecretAccessPolicy(secretArn: string): PolicyStatement { return new PolicyStatement({ effect: Effect.ALLOW, @@ -16,10 +20,6 @@ export function getSecretAccessPolicy(secretArn: string): PolicyStatement { ] }) } -import { mkdtempSync, writeFileSync } from 'fs'; -import { join } from 'path'; -import { tmpdir } from 'os'; -import { DockerImageAsset } from "aws-cdk-lib/aws-ecr-assets"; export function appendArgIfNotInExtraArgs( baseCommand: string, @@ -453,17 +453,41 @@ export function isRegionGovCloud(region: string): boolean { * * @param {string} imageName - The name of the Docker image to save as a tarball and use in CDK. * @returns {ContainerImage} - A `ContainerImage` object representing the Docker image asset. - */ + */ export function makeLocalAssetContainerImage(scope: Construct, imageName: string): ContainerImage { - const sanitizedImageName = imageName.replace(/[^a-zA-Z0-9-_]/g, '_'); - const tempDir = mkdtempSync(join(tmpdir(), 'docker-build-' + sanitizedImageName)); - const dockerfilePath = join(tempDir, 'Dockerfile'); - const dockerfileContent = ` - FROM ${imageName} - `; - writeFileSync(dockerfilePath, dockerfileContent); - const asset = new DockerImageAsset(scope, 'ServiceImage', { - directory: tempDir, - }); - return ContainerImage.fromDockerImageAsset(asset); -} \ No newline at end of file + const sanitizedImageName = imageName.replace(/[^a-zA-Z0-9-_]/g, '_'); + const tempDir = mkdtempSync(join(tmpdir(), 'docker-build-' + sanitizedImageName)); + const dockerfilePath = join(tempDir, 'Dockerfile'); + + let imageHash = null; + try { + // Update the image if it is not a local image + if (!imageName.startsWith('migrations/')) { + execSync(`docker pull ${imageName}`); + } + // Get the actual hash for the image + const imageId = execSync(`docker inspect --format='{{.Id}}' ${imageName}`).toString().trim(); + if (!imageId) { + throw new Error(`No RepoDigests found for image: ${imageName}`); + } + imageHash = imageId.split(':')[1]; + CdkLogger.info('For image: ' + imageName + ' found hash: ' + imageHash); + } catch (error) { + CdkLogger.error('Error fetching the actual hash for the image: ' + imageName + ' Error: ' + error); + throw new Error('Error fetching the image hash for the image: ' + imageName + ' Error: ' + error); + } + + const dockerfileContent = ` + FROM ${imageName} + `; + + writeFileSync(dockerfilePath, dockerfileContent); + const assetName = `${sanitizedImageName.charAt(0).toUpperCase() + sanitizedImageName.slice(1)}ImageAsset`; + const asset = new DockerImageAsset(scope, assetName, { + directory: tempDir, + // add the tag to the hash so that the asset is invalidated when the tag changes + extraHash: imageHash, + assetName: assetName, + }); + return ContainerImage.fromDockerImageAsset(asset); + } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts index a23d44daa..dc851b146 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts @@ -7,7 +7,7 @@ import { TaskDefinition } from "aws-cdk-lib/aws-ecs"; import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs"; -import {createAwsDistroForOtelPushInstrumentationPolicy} from "../common-utilities"; +import {createAwsDistroForOtelPushInstrumentationPolicy, makeLocalAssetContainerImage} from "../common-utilities"; import {Duration, RemovalPolicy} from "aws-cdk-lib"; import { DockerImageAsset } from "aws-cdk-lib/aws-ecr-assets"; import { join } from "path"; @@ -41,9 +41,7 @@ export class OtelCollectorSidecar { logGroupName: `${logGroupPrefix}/otel-collector` }); const otelCollectorContainer = taskDefinition.addContainer("OtelCollectorContainer", { - image: ContainerImage.fromDockerImageAsset(new DockerImageAsset(taskDefinition.stack, "OtelCollectorImage", { - directory: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/otelCollector") - })), + image: makeLocalAssetContainerImage(taskDefinition.stack, "migrations/otel_collector:latest"), containerName: "otel-collector", command: ["--config=/etc/otel-config-aws.yaml"], portMappings: [otelCollectorPort, otelCollectorHealthcheckPort], diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index e5c2b48a2..67db9e0ef 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -36,8 +36,7 @@ "scripts": { "build": "tsc", "watch": "tsc -w", - "test": "./buildDockerImages.sh && npm run test:withoutBuildImages", - "test:withoutBuildImages": "jest --coverage", + "test": "jest --coverage", "clean": "rm -rf dist", "release": "npm run build", "cdk": "cdk", diff --git a/deployment/cdk/opensearch-service-migration/test/jest.setup.ts b/deployment/cdk/opensearch-service-migration/test/jest.setup.ts new file mode 100644 index 000000000..59d58eacf --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/jest.setup.ts @@ -0,0 +1,34 @@ + +jest.mock('child_process', () => { + const child_process = jest.requireActual('child_process'); + + // Define the list of expected Docker images as per CI.yml + const expectedDockerImages = [ + 'docker.io/apache/kafka:3.7.0', + 'migrations/capture_proxy_es:latest', + 'migrations/capture_proxy:latest', + 'migrations/elasticsearch_searchguard:latest', + 'migrations/migration_console:latest', + 'migrations/reindex_from_snapshot:latest', + 'migrations/traffic_replayer:latest', + 'migrations/otel_collector:latest', + 'opensearchproject/opensearch:2', + ]; + + return { + execSync: jest.fn().mockImplementation((command: string) => { + if (expectedDockerImages.some(expectedImage => command.endsWith(expectedImage))) { + if (command.startsWith('docker inspect')) { + return Buffer.from("sha256:a6cdbbcf16c510f8e24c6b993"); + } + if (command.startsWith('docker pull')) { + return Buffer.from(""); + } + } + throw new Error(`Command not supported: ${command}`); + }), + // Uncomment and replace above to use the actual execSync, requires ./buildDockerImages.sh to be run locally + // execSync: child_process.execSync, + spawnSync: child_process.spawnSync, + }; +}); \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts b/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts index 1e1baaca3..042813107 100644 --- a/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts @@ -44,7 +44,9 @@ describe('ACM Certificate Importer Handler', () => { }); afterEach(() => { - jest.resetAllMocks(); + jest.clearAllMocks(); + jest.resetModules(); + jest.restoreAllMocks(); }); test('Create: should generate and import a self-signed certificate', async () => { diff --git a/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts index 00c8c85c8..5c4e72e08 100644 --- a/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts @@ -5,9 +5,10 @@ import {ContainerImage} from "aws-cdk-lib/aws-ecs"; import {describe, beforeEach, afterEach, test, expect, jest} from '@jest/globals'; import {ReindexFromSnapshotStack} from "../lib"; + describe('Migration Console Stack Tests', () => { - // Mock the implementation of fromDockerImageAsset before all tests beforeEach(() => { + // Mock the implementation of fromDockerImageAsset before all tests jest.spyOn(ContainerImage, 'fromDockerImageAsset').mockImplementation(() => ContainerImage.fromRegistry("ServiceImage")); }); @@ -16,7 +17,6 @@ describe('Migration Console Stack Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Migration Console task definition is updated when services.yaml inputs change', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts index 10b15abbb..1d946d892 100644 --- a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts @@ -16,7 +16,6 @@ describe('Migration Services YAML Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Test default servicesYaml can be stringified', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts index 681559447..e7284036e 100644 --- a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts @@ -40,7 +40,6 @@ describe('NetworkStack Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Test vpcEnabled setting that is disabled does not create stack', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts index e309b26f6..c9ed741ab 100644 --- a/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/opensearch-domain-stack.test.ts @@ -15,7 +15,6 @@ describe('OpenSearch Domain Stack Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Test primary context options are mapped with standard data type', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts index da75e12e7..e8aeed273 100644 --- a/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts @@ -31,7 +31,6 @@ describe('ReindexFromSnapshotStack Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('ReindexFromSnapshotStack creates expected resources', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts index ad5ab2a36..8e906de1c 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts @@ -20,7 +20,6 @@ describe('Stack Composer Ordering Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Test all migration services with MSK get created when enabled', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts index b1cf706d5..285b24ec5 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts @@ -16,7 +16,6 @@ describe('Stack Composer Tests', () => { jest.clearAllMocks(); jest.resetModules(); jest.restoreAllMocks(); - jest.resetAllMocks(); }); test('Test empty string provided for a parameter which has a default value, uses the default value', () => {