Merge pull request #1410 from simlu/dev2

feat: added healing for sqs xml

.structignore

@@ -1 +1,2 @@
 test/util/cache-clearer-disabled.spec.js
+test/modules/response-healing.spec.js

package.json

@@ -86,6 +86,7 @@
     "object-scan": "19.0.5",
     "smart-fs": "4.0.1",
     "timekeeper": "2.3.1",
-    "tmp": "0.2.1"
+    "tmp": "0.2.1",
+    "xml2js": "0.6.2"
   }
 }

src/modules/request-recorder.js

@@ -11,7 +11,7 @@ import nockCommon from 'nock/lib/common.js';
 import compareUrls from '../util/compare-urls.js';
 import nockListener from './request-recorder/nock-listener.js';
 import nockMock from './request-recorder/nock-mock.js';
-import healSqsSendMessageBatch from './request-recorder/heal-sqs-send-message-batch.js';
+import healSqs from './request-recorder/heal-sqs.js';
 import applyModifiers from './request-recorder/apply-modifiers.js';
 import requestInjector from './request-recorder/request-injector.js';
 
@@ -223,7 +223,7 @@ export default (opts) => {
 
               if (anyFlagPresent(['magic', 'response'])) {
                 const responseBody = tryParseJson([
-                  healSqsSendMessageBatch
+                  healSqs
                 ].reduce(
                   (respBody, fn) => fn(requestBodyString, respBody, scope, req),
                   interceptor.body

src/modules/request-recorder/heal-sqs.js

@@ -0,0 +1,30 @@
+import crypto from 'crypto';
+import { tryParseJson } from './util.js';
+import migration from './heal-sqs/migration.js';
+
+export default (requestBody, responseBody, scope, req) => {
+  if (scope?.basePath !== 'https://sqs.us-west-2.amazonaws.com:443') {
+    return responseBody;
+  }
+
+  const header = req?.options?.headers?.['x-amz-target'];
+
+  if (typeof responseBody === 'string' && responseBody.startsWith('<?xml')) {
+    return migration({ responseBody, header });
+  }
+
+  const requestJson = tryParseJson(requestBody);
+  const responseJson = tryParseJson(responseBody);
+
+  if (header === 'AmazonSQS.SendMessageBatch') {
+    return {
+      Successful: requestJson.Entries.map(({ Id, MessageBody }, idx) => ({
+        Id,
+        MessageId: responseJson?.Successful?.[idx]?.MessageId || crypto.randomUUID(),
+        MD5OfMessageBody: crypto.createHash('md5').update(MessageBody).digest('hex')
+      }))
+    };
+  }
+
+  return responseBody;
+};

src/modules/request-recorder/heal-sqs/migration.js

@@ -0,0 +1,106 @@
+// This code logic is used to migrate legacy AWS SQS xml to json
+import xml2js from 'xml2js';
+import objectScan from 'object-scan';
+
+const tryParseXML = (body) => {
+  let parsed = body;
+  try {
+    xml2js.parseString(body, (err, result) => {
+      parsed = JSON.parse(JSON.stringify(result));
+    });
+  } catch (e) {
+    return null;
+  }
+  return parsed;
+};
+
+export default ({ responseBody, header }) => {
+  const responseXml = tryParseXML(responseBody);
+  if (responseXml !== null) {
+    if (header === 'AmazonSQS.ListQueueTags') {
+      const scanner = objectScan(['ListQueueTagsResponse.ListQueueTagsResult[0].Tag[*]'], {
+        rtn: ({ value }) => [value.Key, value.Value[0]],
+        afterFn: ({ result }) => Object.fromEntries(result)
+      });
+      const Tags = scanner(responseXml);
+      return { Tags };
+    }
+    if (header === 'AmazonSQS.GetQueueAttributes') {
+      const scanner = objectScan(['GetQueueAttributesResponse.GetQueueAttributesResult[0].Attribute[*]'], {
+        rtn: ({ value }) => [value.Name[0], value.Value[0]],
+        afterFn: ({ result }) => Object.fromEntries(result)
+      });
+      const Attributes = scanner(responseXml);
+      return { Attributes };
+    }
+    if (header === 'AmazonSQS.GetQueueUrl') {
+      if (responseXml?.ErrorResponse?.Error?.[0]?.Code?.[0] === 'AWS.SimpleQueueService.NonExistentQueue') {
+        return {
+          __type: 'com.amazonaws.sqs#QueueDoesNotExist',
+          message: 'The specified queue does not exist.'
+        };
+      }
+      const QueueUrl = responseXml?.GetQueueUrlResponse?.GetQueueUrlResult?.[0]?.QueueUrl?.[0];
+      return { QueueUrl };
+    }
+    if (header === 'AmazonSQS.CreateQueue') {
+      if (responseXml?.ErrorResponse?.Error?.[0]?.Code?.[0] === 'QueueAlreadyExists') {
+        return {
+          __type: 'com.amazonaws.sqs#QueueNameExists',
+          message: 'The specified queue name does exist.'
+        };
+      }
+      const QueueUrl = responseXml?.CreateQueueResponse?.CreateQueueResult?.[0]?.QueueUrl?.[0];
+      return { QueueUrl };
+    }
+    if (header === 'AmazonSQS.ListQueues') {
+      const scannerQueueUrls = objectScan(
+        ['ListQueuesResponse.ListQueuesResult[0].QueueUrl[*]'],
+        { rtn: 'value', reverse: false }
+      );
+      const scannerNextToken = objectScan(
+        ['ListQueuesResponse.ListQueuesResult[0].NextToken[0]'],
+        { rtn: 'value', reverse: false, abort: true }
+      );
+      return {
+        QueueUrls: scannerQueueUrls(responseXml),
+        NextToken: scannerNextToken(responseXml)
+      };
+    }
+    if (header === 'AmazonSQS.TagQueue') {
+      return {};
+    }
+    if (header === 'AmazonSQS.SetQueueAttributes') {
+      return {};
+    }
+    if (header === 'AmazonSQS.SendMessageBatch') {
+      const scannerSuccessful = objectScan(
+        ['SendMessageBatchResponse.SendMessageBatchResult[0].SendMessageBatchResultEntry[*]'],
+        {
+          rtn: ({ value }) => ({
+            Id: value.Id[0],
+            MessageId: value.MessageId[0],
+            MD5OfMessageBody: value.MD5OfMessageBody[0]
+          }),
+          reverse: false
+        }
+      );
+      const scannerFailed = objectScan(
+        ['SendMessageBatchResponse.SendMessageBatchResult[0].BatchResultErrorEntry[*]'],
+        {
+          rtn: ({ value }) => ({
+            Id: value.Id[0],
+            SenderFault: value.SenderFault[0],
+            Code: value.Code[0]
+          }),
+          reverse: false
+        }
+      );
+      return {
+        Successful: scannerSuccessful(responseXml),
+        Failed: scannerFailed(responseXml)
+      };
+    }
+  }
+  return responseBody;
+};

test/modules/request-recorder.spec.js

@@ -3,26 +3,10 @@ import https from 'https';
 import fs from 'smart-fs';
 import get from 'lodash.get';
 import axios from 'axios';
-import { logger } from 'lambda-monitor-logger';
 import { expect } from 'chai';
-import awsSdkWrap from 'aws-sdk-wrap';
-import {
-  SQSClient,
-  SendMessageBatchCommand
-} from '@aws-sdk/client-sqs';
 import { describe } from '../../src/index.js';
 import { NockRecord, spawnServer } from '../server.js';
 
-const aws = awsSdkWrap({
-  logger,
-  services: {
-    SQS: SQSClient,
-    'SQS:CMD': {
-      SendMessageBatchCommand
-    }
-  }
-});
-
 describe('Testing RequestRecorder', { useTmpDir: true, timestamp: 0 }, () => {
   const cassetteFile = 'file1.json';
   let tmpDir;
@@ -383,20 +367,6 @@ describe('Testing RequestRecorder', { useTmpDir: true, timestamp: 0 }, () => {
       await runner('prune,record', { qs: [1], raises: true });
     });
 
-    describe('Testing magic healing', { cryptoSeed: 'd28095c6-19f4-4dc2-a7cc-f7640c032967' }, () => {
-      it('Testing heal SQS response', async ({ fixture }) => {
-        fs.smartWrite(path.join(tmpDir, cassetteFile), fixture('sqs-cassette-bad'));
-        const r = await nockRecord(() => aws.sqs.sendMessageBatch({
-          messages: [{ k: 1 }, { k: 2 }],
-          queueUrl: process.env.QUEUE_URL
-        }), { heal: 'magic' });
-        const expected = fixture('sqs-cassette-expected');
-        expected[0].reqheaders['user-agent'] = r.expectedCassette[0].reqheaders['user-agent'];
-        expect(r.expectedCassette[0].reqheaders['user-agent'].startsWith('aws-sdk-js/3.')).to.equal(true);
-        expect(r.expectedCassette).to.deep.equal(expected);
-      });
-    });
-
     it('Testing record (with headers)', async () => {
       await runner('record', {
         qs: [1, 2, 3],

test/modules/response-healing.spec.js

@@ -0,0 +1,78 @@
+import path from 'path';
+import fs from 'smart-fs';
+import { logger } from 'lambda-monitor-logger';
+import { expect } from 'chai';
+import awsSdkWrap from 'aws-sdk-wrap';
+import {
+  SQSClient,
+  SendMessageBatchCommand,
+  GetQueueUrlCommand,
+  GetQueueAttributesCommand,
+  ListQueueTagsCommand,
+  CreateQueueCommand,
+  ListQueuesCommand,
+  TagQueueCommand,
+  SetQueueAttributesCommand
+} from '@aws-sdk/client-sqs';
+import objectScan from 'object-scan';
+import { describe } from '../../src/index.js';
+import { NockRecord } from '../server.js';
+import reqHeaderOverwrite from '../req-header-overwrite.js';
+
+const aws = awsSdkWrap({
+  logger,
+  services: {
+    SQS: SQSClient,
+    'SQS:CMD': {
+      SendMessageBatchCommand,
+      GetQueueUrlCommand,
+      GetQueueAttributesCommand,
+      ListQueueTagsCommand,
+      CreateQueueCommand,
+      ListQueuesCommand,
+      TagQueueCommand,
+      SetQueueAttributesCommand
+    }
+  }
+});
+
+const fixtureFolder = `${fs.filename(import.meta.url)}__fixtures`;
+const files = fs
+  .walkDir(fixtureFolder)
+  .filter((f) => !f.endsWith('.json__expected.json'));
+
+describe('Testing Response Healing', {
+  useTmpDir: true,
+  timestamp: 0,
+  cryptoSeed: 'd28095c6-19f4-4dc2-a7cc-f7640c032967'
+}, () => {
+  const cassetteFile = 'file1.json';
+  let tmpDir;
+  let nockRecord;
+
+  beforeEach(async ({ dir }) => {
+    tmpDir = dir;
+    nockRecord = NockRecord(tmpDir, cassetteFile);
+  });
+
+  // eslint-disable-next-line mocha/no-setup-in-describe
+  files.forEach((f) => {
+    it(`Testing ${f}`, async ({ fixture }) => {
+      const { fn, params, cassette } = fixture(f);
+      fs.smartWrite(path.join(tmpDir, cassetteFile), cassette);
+      const func = fn.split('.').reduce((p, v) => p[v], aws);
+      objectScan(['**'], {
+        filterFn: ({ value, parent, property }) => {
+          if (value === '$queueUrl') {
+            // eslint-disable-next-line no-param-reassign
+            parent[property] = process.env.QUEUE_URL;
+          }
+        }
+      })(params);
+      const r = await nockRecord(() => func(...params), { heal: 'magic', reqHeaderOverwrite });
+      const outFile = path.join(fixtureFolder, `${f}__expected.json`);
+      const overwritten = fs.smartWrite(outFile, r.expectedCassette);
+      expect(overwritten).to.deep.equal(false);
+    });
+  });
+});

test/modules/response-healing.spec.js.env.yml

@@ -0,0 +1,4 @@
+AWS_REGION: "us-west-2"
+AWS_ACCESS_KEY_ID: "XXXXXXXXXXXXXXXXXXXX"
+AWS_SECRET_ACCESS_KEY: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+QUEUE_URL: "https://sqs.us-west-2.amazonaws.com/123456789101/service-name-data-local-SomeQueue"

test/modules/response-healing.spec.js__fixtures/json_bad-md5.json

@@ -0,0 +1,33 @@
+{
+  "fn": "sqs.sendMessageBatch",
+  "params": [{
+    "messages": [{ "k": 1 }, { "k": 2 }],
+    "queueUrl": "$queueUrl"
+  }],
+  "cassette": [
+    {
+      "scope": "https://sqs.us-west-2.amazonaws.com:443",
+      "method": "POST",
+      "path": "/",
+      "body": {
+        "Entries": [
+          {
+            "Id": "4f494d6b422d1e5fbbe6b6d82584ce7f70c1530f",
+            "MessageBody": "{\"k\":1}"
+          }
+        ],
+        "QueueUrl": "https://sqs.us-west-2.amazonaws.com/123456789101/service-name-data-local-SomeQueue"
+      },
+      "status": 200,
+      "response": {
+        "Successful": [
+          {
+            "Id": "feabf4c420f7eec1fbd3ad1b74e972af9f879cad",
+            "MessageId": "e40625e7-e66f-4c6c-8d9b-7bb8944b0108",
+            "MD5OfMessageBody": "some-bad-hash45be96b2d5b9d22cdc4"
+          }
+        ]
+      }
+    }
+  ]
+}