diff --git a/package-lock.json b/package-lock.json index 37ad81a..2334cf5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,6 +56,7 @@ "@swc-node/register": "^1.6.5", "@swc/cli": "0.1.62", "@swc/core": "^1.3.57", + "@types/aws-lambda": "^8.10.125", "@types/cross-spawn": "^6.0.2", "@types/fs-extra": "^11.0.1", "@types/jest": "29.4.0", @@ -6483,6 +6484,12 @@ "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", "dev": true }, + "node_modules/@types/aws-lambda": { + "version": "8.10.125", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.125.tgz", + "integrity": "sha512-Vqw/WMlV4O1fJT6capim01v7VLDZkcX1n6Yhb52E7IfnMqYbNfwHfyDV8rRN42NLBtdDvfaqcCqs2K0fr5ljZw==", + "dev": true + }, "node_modules/@types/babel__core": { "version": "7.1.20", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.20.tgz", @@ -30257,6 +30264,12 @@ "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", "dev": true }, + "@types/aws-lambda": { + "version": "8.10.125", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.125.tgz", + "integrity": "sha512-Vqw/WMlV4O1fJT6capim01v7VLDZkcX1n6Yhb52E7IfnMqYbNfwHfyDV8rRN42NLBtdDvfaqcCqs2K0fr5ljZw==", + "dev": true + }, "@types/babel__core": { "version": "7.1.20", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.20.tgz", diff --git a/package.json b/package.json index b751e38..b226ac0 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,13 @@ "@commitlint/config-conventional": "^17.4.4", "@commitlint/config-nx-scopes": "^17.6.4", "@commitlint/cz-commitlint": "^17.5.0", + "@nx/devkit": "16.3.2", + "@nx/eslint-plugin": "16.3.2", + "@nx/jest": "16.3.2", + "@nx/js": "16.3.2", + "@nx/linter": "16.3.2", + "@nx/plugin": "16.3.2", + "@nx/workspace": "16.3.2", "@semantic-release/changelog": "^6.0.3", "@semantic-release/exec": "^6.0.3", "@semantic-release/git": "^10.0.1", @@ -19,6 +26,7 @@ "@swc-node/register": "^1.6.5", "@swc/cli": "0.1.62", "@swc/core": "^1.3.57", + "@types/aws-lambda": "^8.10.125", "@types/cross-spawn": "^6.0.2", "@types/fs-extra": "^11.0.1", "@types/jest": "29.4.0", @@ -39,21 +47,14 @@ "lint-staged": "^13.1.2", "mock-fs": "^5.2.0", "nx": "16.3.2", + "nx-cloud": "16.0.5", "prettier": "^2.8.4", "semantic-release-npm": "^0.0.5", "semantic-release-plus": "^20.0.0", "string-dedent": "^3.0.1", "ts-jest": "29.1.0", "ts-node": "10.9.1", - "typescript": "5.0.4", - "nx-cloud": "16.0.5", - "@nx/devkit": "16.3.2", - "@nx/workspace": "16.3.2", - "@nx/js": "16.3.2", - "@nx/linter": "16.3.2", - "@nx/eslint-plugin": "16.3.2", - "@nx/jest": "16.3.2", - "@nx/plugin": "16.3.2" + "typescript": "5.0.4" }, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "^3.348.0", diff --git a/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.stream.ts b/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.stream.ts index 8ec4a55..acba6c3 100644 --- a/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.stream.ts +++ b/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.stream.ts @@ -10,6 +10,8 @@ import { const client = new DynamoDBClient({}); const docClient = DynamoDBDocumentClient.from(client); +const TARGET_TABLE_NAME = 'example-users-v2'; + export const handler: DynamoDBStreamHandler = async (event) => { console.debug('Event', JSON.stringify(event)); @@ -22,7 +24,7 @@ export const handler: DynamoDBStreamHandler = async (event) => { await docClient.send( new PutCommand({ - TableName: process.env['TARGET_TABLE_NAME'], + TableName: TARGET_TABLE_NAME, Item: { userId: item['id'], name: item['name'], @@ -37,7 +39,7 @@ export const handler: DynamoDBStreamHandler = async (event) => { await docClient.send( new DeleteCommand({ - TableName: process.env['TARGET_TABLE_NAME'], + TableName: TARGET_TABLE_NAME, Key: { userId: keys['id'], }, diff --git a/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.ts b/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.ts index 23336d2..3521d1b 100644 --- a/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.ts +++ b/packages/data-migration-example/src/migrations/users/202305264-migrate-v1-to-v2.ts @@ -27,7 +27,7 @@ export default class extends DynamoDBMigrationBase { } async up(): Promise { - await this.enableStream('example-users', 'example-users-v2'); + await this.enableStream('example-users'); this.logger.info('Migrating users from v1 to v2'); let count = 0; diff --git a/packages/data-migration/README.md b/packages/data-migration/README.md index 3d6ddf1..a369be7 100644 --- a/packages/data-migration/README.md +++ b/packages/data-migration/README.md @@ -413,7 +413,7 @@ import { }) export default class extends DynamoDBMigrationBase { async up(): Promise { - await this.enableStream('my-table-v1', 'my-table-v2'); + await this.enableStream('my-table-v1'); const items = await MyTableModel.scan().exec(); for (const chunk of chunks(items, 25)) { diff --git a/packages/data-migration/src/migrator/dynamodb/migration.spec.ts b/packages/data-migration/src/migrator/dynamodb/migration.spec.ts index 013430e..a4b2156 100644 --- a/packages/data-migration/src/migrator/dynamodb/migration.spec.ts +++ b/packages/data-migration/src/migrator/dynamodb/migration.spec.ts @@ -1293,7 +1293,7 @@ describe('DynamoDBMigrationBase', () => { } async up() { - await this.enableStream('source', 'destination'); + await this.enableStream('source'); } async down() { @@ -1306,21 +1306,12 @@ describe('DynamoDBMigrationBase', () => { const dynamoDbMock = mockClient(DynamoDBClient); const dynamodbStreamsMock = mockClient(DynamoDBStreamsClient); - dynamoDbMock - .on(DescribeTableCommand) - .resolvesOnce({ - Table: { - TableName: 'source', - TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - }, - }) - .resolvesOnce({ - Table: { - TableName: 'destination', - TableArn: - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', - }, - }); + dynamoDbMock.on(DescribeTableCommand).resolvesOnce({ + Table: { + TableName: 'source', + TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', + }, + }); dynamoDbMock.on(ListTagsOfResourceCommand).resolvesOnce({ Tags: [], @@ -1341,15 +1332,8 @@ describe('DynamoDBMigrationBase', () => { TableName: 'source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 2, - DescribeTableCommand, - { - TableName: 'destination', - } - ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(3, UpdateTableCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(2, UpdateTableCommand, { TableName: 'source', StreamSpecification: { StreamEnabled: true, @@ -1358,14 +1342,14 @@ describe('DynamoDBMigrationBase', () => { }); expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 4, + 3, ListTagsOfResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(5, TagResourceCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(4, TagResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', Tags: [ { @@ -1423,7 +1407,7 @@ describe('DynamoDBMigrationBase', () => { } async up() { - await this.enableStream('source', 'destination'); + await this.enableStream('source'); } async down() { @@ -1436,25 +1420,16 @@ describe('DynamoDBMigrationBase', () => { const dynamoDbMock = mockClient(DynamoDBClient); const dynamodbStreamsMock = mockClient(DynamoDBStreamsClient); - dynamoDbMock - .on(DescribeTableCommand) - .resolvesOnce({ - Table: { - TableName: 'source', - TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - StreamSpecification: { - StreamEnabled: true, - StreamViewType: 'NEW_AND_OLD_IMAGES', - }, - }, - }) - .resolvesOnce({ - Table: { - TableName: 'destination', - TableArn: - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', + dynamoDbMock.on(DescribeTableCommand).resolvesOnce({ + Table: { + TableName: 'source', + TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', + StreamSpecification: { + StreamEnabled: true, + StreamViewType: 'NEW_AND_OLD_IMAGES', }, - }); + }, + }); dynamoDbMock.on(ListTagsOfResourceCommand).resolvesOnce({ Tags: [], @@ -1494,25 +1469,17 @@ describe('DynamoDBMigrationBase', () => { TableName: 'source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 2, - DescribeTableCommand, - { - TableName: 'destination', - } - ); - expect(dynamoDbMock).not.toHaveReceivedCommand(UpdateTableCommand); expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 3, + 2, ListTagsOfResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(4, TagResourceCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(3, TagResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', Tags: [ { @@ -1593,7 +1560,7 @@ describe('DynamoDBMigrationBase', () => { } async up() { - await this.enableStream('source', 'destination'); + await this.enableStream('source'); } async down() { @@ -1608,25 +1575,16 @@ describe('DynamoDBMigrationBase', () => { const iamMock = mockClient(IAMClient); const lambdaMock = mockClient(LambdaClient); - dynamoDbMock - .on(DescribeTableCommand) - .resolvesOnce({ - Table: { - TableName: 'source', - TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - StreamSpecification: { - StreamEnabled: true, - StreamViewType: 'NEW_AND_OLD_IMAGES', - }, - }, - }) - .resolvesOnce({ - Table: { - TableName: 'destination', - TableArn: - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', + dynamoDbMock.on(DescribeTableCommand).resolvesOnce({ + Table: { + TableName: 'source', + TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', + StreamSpecification: { + StreamEnabled: true, + StreamViewType: 'NEW_AND_OLD_IMAGES', }, - }); + }, + }); dynamoDbMock.on(ListTagsOfResourceCommand).resolves({ Tags: [], @@ -1697,25 +1655,18 @@ describe('DynamoDBMigrationBase', () => { TableName: 'source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 2, - DescribeTableCommand, - { - TableName: 'destination', - } - ); expect(dynamoDbMock).not.toHaveReceivedCommand(UpdateTableCommand); expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 3, + 2, ListTagsOfResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(4, TagResourceCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(3, TagResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', Tags: [ { @@ -1859,23 +1810,12 @@ describe('DynamoDBMigrationBase', () => { 'dynamodb:Query', 'dynamodb:Scan', 'dynamodb:Describe*', - ], - Resource: [ - 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', - ], - }, - { - Effect: 'Allow', - Action: [ 'dynamodb:DeleteItem', 'dynamodb:PutItem', 'dynamodb:UpdateItem', 'dynamodb:BatchWrite*', ], - Resource: [ - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', - ], + Resource: ['*'], }, { Effect: 'Allow', @@ -1932,8 +1872,6 @@ describe('DynamoDBMigrationBase', () => { Environment: { Variables: { ENV: 'test', - TARGET_TABLE_NAME: 'destination', - TRANSFORM_MODULE_PATH: './transform.js', }, }, FunctionName: 'migration-namespace-name-202304031-stream', @@ -1978,7 +1916,7 @@ describe('DynamoDBMigrationBase', () => { } async up() { - await this.enableStream('source', 'destination'); + await this.enableStream('source'); } async down() { @@ -1993,25 +1931,16 @@ describe('DynamoDBMigrationBase', () => { const iamMock = mockClient(IAMClient); const lambdaMock = mockClient(LambdaClient); - dynamoDbMock - .on(DescribeTableCommand) - .resolvesOnce({ - Table: { - TableName: 'source', - TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - StreamSpecification: { - StreamEnabled: true, - StreamViewType: 'KEYS_ONLY', - }, - }, - }) - .resolvesOnce({ - Table: { - TableName: 'destination', - TableArn: - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', + dynamoDbMock.on(DescribeTableCommand).resolvesOnce({ + Table: { + TableName: 'source', + TableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', + StreamSpecification: { + StreamEnabled: true, + StreamViewType: 'KEYS_ONLY', }, - }); + }, + }); dynamoDbMock.on(ListTagsOfResourceCommand).resolves({ Tags: [], @@ -2082,15 +2011,8 @@ describe('DynamoDBMigrationBase', () => { TableName: 'source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 2, - DescribeTableCommand, - { - TableName: 'destination', - } - ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(3, UpdateTableCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(2, UpdateTableCommand, { TableName: 'source', StreamSpecification: { StreamEnabled: true, @@ -2099,14 +2021,14 @@ describe('DynamoDBMigrationBase', () => { }); expect(dynamoDbMock).toHaveReceivedNthCommandWith( - 4, + 3, ListTagsOfResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', } ); - expect(dynamoDbMock).toHaveReceivedNthCommandWith(5, TagResourceCommand, { + expect(dynamoDbMock).toHaveReceivedNthCommandWith(4, TagResourceCommand, { ResourceArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/source', Tags: [ { @@ -2250,23 +2172,12 @@ describe('DynamoDBMigrationBase', () => { 'dynamodb:Query', 'dynamodb:Scan', 'dynamodb:Describe*', - ], - Resource: [ - 'arn:aws:dynamodb:us-east-1:123456789012:table/source', - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', - ], - }, - { - Effect: 'Allow', - Action: [ 'dynamodb:DeleteItem', 'dynamodb:PutItem', 'dynamodb:UpdateItem', 'dynamodb:BatchWrite*', ], - Resource: [ - 'arn:aws:dynamodb:us-east-1:123456789012:table/destination', - ], + Resource: ['*'], }, { Effect: 'Allow', @@ -2323,8 +2234,6 @@ describe('DynamoDBMigrationBase', () => { Environment: { Variables: { ENV: 'test', - TARGET_TABLE_NAME: 'destination', - TRANSFORM_MODULE_PATH: './transform.js', }, }, FunctionName: 'migration-namespace-name-202304031-stream', diff --git a/packages/data-migration/src/migrator/dynamodb/migration.ts b/packages/data-migration/src/migrator/dynamodb/migration.ts index 6588e13..213f3eb 100644 --- a/packages/data-migration/src/migrator/dynamodb/migration.ts +++ b/packages/data-migration/src/migrator/dynamodb/migration.ts @@ -309,7 +309,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { protected async enableStream( sourceTableName: string, - destinationTableName: string, options: StreamLambdaOptions = {} ) { const defaults: StreamLambdaOptions = { @@ -329,13 +328,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { }) ); - this.logger.debug(`Describing table ${destinationTableName}`); - const { Table: destinationTable } = await this.client.send( - new DescribeTableCommand({ - TableName: destinationTableName, - }) - ); - await this.enableDynamoDBStreams(sourceTable); const streamArn = await this.getStreamArn(sourceTableName); @@ -362,8 +354,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { if (!resolvedOptions.iamRoleArn) { resolvedOptions.iamRoleArn = await this.createLambdaStreamIamRole( streamArn, - sourceTable, - destinationTable, resolvedOptions.iamPolicyStatements ); } @@ -371,7 +361,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { await this.createStreamLambdaFunction( functionName, zipPath, - destinationTableName, streamArn, resolvedOptions ); @@ -522,7 +511,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { private async createStreamLambdaFunction( functionName: string, zipPath: string, - destinationTableName: string, streamArn: string, lambdaOptions: StreamLambdaOptions ) { @@ -547,8 +535,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { Tags: this.getLambdaTags(), Environment: { Variables: { - TARGET_TABLE_NAME: destinationTableName, - TRANSFORM_MODULE_PATH: './transform.js', ENV: process.env.ENV, ...lambdaOptions.envVars, }, @@ -570,8 +556,6 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { private async createLambdaStreamIamRole( streamArn: string, - sourceTable: TableDescription, - destinationTable: TableDescription, additionalStatments: Record[] ) { const iamClient = new IAMClient({ region: this.region }); @@ -640,18 +624,12 @@ export abstract class DynamoDBMigrationBase extends MigrationBase { 'dynamodb:Query', 'dynamodb:Scan', 'dynamodb:Describe*', - ], - Resource: [sourceTable.TableArn, destinationTable.TableArn], - }, - { - Effect: 'Allow', - Action: [ 'dynamodb:DeleteItem', 'dynamodb:PutItem', 'dynamodb:UpdateItem', 'dynamodb:BatchWrite*', ], - Resource: [destinationTable.TableArn], + Resource: ['*'], }, { Effect: 'Allow',