diff --git a/.flake8 b/.flake8
index fd30fc6ba..ec5ead7b5 100644
--- a/.flake8
+++ b/.flake8
@@ -12,4 +12,4 @@ exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,.venv
 
 # F401 - Unused imports -- this is the only way to have a file-wide rule exception
 per-file-ignores =
-    upgrades/upgrade_testing_framework/steps/__init__.py:F401
\ No newline at end of file
+    experimental/upgrades/upgrade_testing_framework/steps/__init__.py:F401
\ No newline at end of file
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
index 367156858..dfbd65288 100644
--- a/.github/workflows/python-tests.yml
+++ b/.github/workflows/python-tests.yml
@@ -3,10 +3,10 @@ name: python-tests
 on:
   push:
     paths:
-      - 'cluster_migration_core/**.py'
+      - 'experimental/cluster_migration_core/**.py'
   pull_request:
     paths:
-      - 'cluster_migration_core/**.py'
+      - 'experimental/cluster_migration_core/**.py'
 
 jobs:
   test-linux:
@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     defaults:
       run:
-        working-directory: ./cluster_migration_core
+        working-directory: ./experimental/cluster_migration_core
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
@@ -34,4 +34,4 @@ jobs:
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v3
         with:
-          files: cluster_migration_core/coverage.xml
\ No newline at end of file
+          files: cluster_migration_core/coverage.xml
diff --git a/.gitignore b/.gitignore
index 1732dfbe7..c974fa427 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ coverage.xml
 .venv
 __pycache__
 *.egg-info*
-.python-version
\ No newline at end of file
+.python-version
+logs
\ No newline at end of file
diff --git a/index_configuration_tool/Dockerfile b/FetchMigration/Dockerfile
similarity index 91%
rename from index_configuration_tool/Dockerfile
rename to FetchMigration/Dockerfile
index c0d53baaf..db3bd0a47 100644
--- a/index_configuration_tool/Dockerfile
+++ b/FetchMigration/Dockerfile
@@ -1,5 +1,5 @@
 FROM opensearch-data-prepper:2.4.0-SNAPSHOT
-COPY requirements.txt .
+COPY python/requirements.txt .
 
 # Install dependencies to local user directory
 RUN apk update
@@ -9,7 +9,7 @@ RUN pip install --user -r requirements.txt
 ENV ICT_CODE_PATH /code
 WORKDIR $ICT_CODE_PATH
 # Copy only source code
-COPY ./*.py .
+COPY python/*.py .
 
 # update PATH
 ENV PATH=/root/.local:$PATH
diff --git a/index_configuration_tool/README.md b/FetchMigration/README.md
similarity index 55%
rename from index_configuration_tool/README.md
rename to FetchMigration/README.md
index 0a4b5584a..79d5ba0ab 100644
--- a/index_configuration_tool/README.md
+++ b/FetchMigration/README.md
@@ -1,16 +1,30 @@
 # Index Configuration Tool
 
-Python package that automates the creation of indices on a target cluster based on the contents of a source cluster. Index settings and index mappings are correctly copied over.
+Python package that automates the creation of indices on a target cluster based on the contents of a source cluster. 
+Index settings and index mappings are correctly copied over, but no data is transferred. 
+This tool seeks to eliminate the need to [specify index templates](https://github.com/awslabs/logstash-output-amazon_es#optional-parameters) when migrating data from one cluster to another.
+The tool currently supports ElasticSearch or OpenSearch as source and target.
 
-This tool seeks to automate [the steps outlined here](https://github.com/kartg/opensearch-migrations/tree/datastash/datastash#2-configure-index-templates-on-the-target-cluster) and eliminate the need for defining index templates on the target cluster when migrating data from one cluster to another. The tool currently supports ElasticSearch or OpenSearch as source and target.
+## Parameters
 
-If an identical index or an index with the same name but differing settings/mappings is already present on the target cluster, that index is skipped and left as-is on the target cluster. The tool completes by printing a report of all processed indices under 3 buckets:
+The first required input to the tool is a path to a [Data Prepper](https://github.com/opensearch-project/data-prepper) pipeline YAML file, which is parsed to obtain the source and target cluster endpoints.
+The second required input is an output path to which a modified version of the pipeline YAML file is written.
+This version of the pipeline adds an index inclusion configuration to the sink, specifying only those indices that were created by the index configuration tool.
+The tool also supports several optional flags:
+
+| Flag | Purpose |
+| ------------- | ------------- |
+| `-h, --help`    | Prints help text and exits |
+| `--report, -r`  | Prints a report of indices indicating which ones will be created, along with indices that are identical or have conflicting settings/mappings.  |
+| `--dryrun`      | Skips the actual creation of indices on the target cluster |
+
+### Reporting
+
+If `--report` is specified, the tool prints all processed indices organized into 3 buckets:
 * Successfully created on the target cluster
 * Skipped due to a conflict in settings/mappings
 * Skipped since the index configuration is identical on source and target
 
-The input to the tool is a Logstash configuration file that is then parsed to obtain source and target endpoints.
-
 ## Current Limitations
 
 * Only supports ElasticSearch and OpenSearch endpoints for source and target
@@ -41,17 +55,22 @@ python -m pip install -r index_configuration_tool/requirements.txt
 After [setup](#setup), the tool can be executed using:
 
 ```shell
-python index_configuration_tool/main.py <path-to-Logstash-file>
+python index_configuration_tool/pre_migration.py <pipeline_yaml_path> <output_file>
 ```
 
-Usage information can also be printed by supplying the `-h` flag.
-
 ### Docker
 
-Replace `<path-to-Logstash-config-file>` in the command below with the path to your Logstash config file.
+First build the Docker image from the `Dockerfile`:
+
+```shell
+docker build -t fetch-migration .
+```
+
+Then run the `fetch-migration` image.
+Replace `<pipeline_yaml_path>` in the command below with the path to your Logstash config file:
 
 ```shell
-confPath=<path-to-Logstash-config-file>; docker run -v $confPath:/tmp/conf.json -t kartg/index-configuration-tool /tmp/conf.json
+docker run -p 4900:4900 -v <pipeline_yaml_path>:/code/input.yaml ict
 ```
 
 ## Development
@@ -86,9 +105,4 @@ Note that the `--omit` parameter must be specified to avoid tracking code covera
 ```shell
 python -m coverage report --omit "*/tests/*"
 python -m coverage html --omit "*/tests/*"
-```
-
-### Lark
-
-The code uses [Lark](https://github.com/lark-parser/lark) for grammar definition, tree parsing and transformation.
-The Logstash parser grammar is adapted from [node-logstash](https://github.com/bpaquet/node-logstash/blob/master/lib/logstash_config.jison).
\ No newline at end of file
+```
\ No newline at end of file
diff --git a/index_configuration_tool/__init__.py b/FetchMigration/python/__init__.py
similarity index 100%
rename from index_configuration_tool/__init__.py
rename to FetchMigration/python/__init__.py
diff --git a/index_configuration_tool/dev-requirements.txt b/FetchMigration/python/dev-requirements.txt
similarity index 100%
rename from index_configuration_tool/dev-requirements.txt
rename to FetchMigration/python/dev-requirements.txt
diff --git a/FetchMigration/python/endpoint_info.py b/FetchMigration/python/endpoint_info.py
new file mode 100644
index 000000000..b77880dc0
--- /dev/null
+++ b/FetchMigration/python/endpoint_info.py
@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class EndpointInfo:
+    url: str
+    auth: tuple = None
+    verify_ssl: bool = True
diff --git a/FetchMigration/python/index_operations.py b/FetchMigration/python/index_operations.py
new file mode 100644
index 000000000..c653344f7
--- /dev/null
+++ b/FetchMigration/python/index_operations.py
@@ -0,0 +1,45 @@
+import requests
+
+from endpoint_info import EndpointInfo
+
+# Constants
+SETTINGS_KEY = "settings"
+MAPPINGS_KEY = "mappings"
+COUNT_KEY = "count"
+__INDEX_KEY = "index"
+__ALL_INDICES_ENDPOINT = "*"
+__COUNT_ENDPOINT = "/_count"
+__INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"]
+
+
+def fetch_all_indices(endpoint: EndpointInfo) -> dict:
+    actual_endpoint = endpoint.url + __ALL_INDICES_ENDPOINT
+    resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl)
+    # Remove internal settings
+    result = dict(resp.json())
+    for index in result:
+        for setting in __INTERNAL_SETTINGS_KEYS:
+            index_settings = result[index][SETTINGS_KEY]
+            if __INDEX_KEY in index_settings:
+                index_settings[__INDEX_KEY].pop(setting, None)
+    return result
+
+
+def create_indices(indices_data: dict, endpoint: EndpointInfo):
+    for index in indices_data:
+        actual_endpoint = endpoint.url + index
+        data_dict = dict()
+        data_dict[SETTINGS_KEY] = indices_data[index][SETTINGS_KEY]
+        data_dict[MAPPINGS_KEY] = indices_data[index][MAPPINGS_KEY]
+        try:
+            resp = requests.put(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl, json=data_dict)
+            resp.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to create index [{index}] - {e!s}")
+
+
+def doc_count(indices: set, endpoint: EndpointInfo) -> int:
+    actual_endpoint = endpoint.url + ','.join(indices) + __COUNT_ENDPOINT
+    resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl)
+    result = dict(resp.json())
+    return int(result[COUNT_KEY])
diff --git a/FetchMigration/python/migration_monitor.py b/FetchMigration/python/migration_monitor.py
new file mode 100644
index 000000000..3b5be7781
--- /dev/null
+++ b/FetchMigration/python/migration_monitor.py
@@ -0,0 +1,104 @@
+import argparse
+import time
+from typing import Optional, List
+
+import requests
+from prometheus_client import Metric
+from prometheus_client.parser import text_string_to_metric_families
+
+from endpoint_info import EndpointInfo
+from migration_monitor_params import MigrationMonitorParams
+
+__PROMETHEUS_METRICS_ENDPOINT = "/metrics/prometheus"
+__SHUTDOWN_ENDPOINT = "/shutdown"
+__DOC_SUCCESS_METRIC = "_opensearch_documentsSuccess"
+__RECORDS_IN_FLIGHT_METRIC = "_BlockingBuffer_recordsInFlight"
+__NO_PARTITIONS_METRIC = "_noPartitionsAcquired"
+
+
+def shutdown_pipeline(endpoint: EndpointInfo):
+    shutdown_endpoint = endpoint.url + __SHUTDOWN_ENDPOINT
+    requests.post(shutdown_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl)
+
+
+def fetch_prometheus_metrics(endpoint: EndpointInfo) -> Optional[List[Metric]]:
+    metrics_endpoint = endpoint.url + __PROMETHEUS_METRICS_ENDPOINT
+    try:
+        response = requests.get(metrics_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl)
+        response.raise_for_status()
+    except requests.exceptions.RequestException:
+        return None
+    # Based on response headers defined in Data Prepper's PrometheusMetricsHandler.java class
+    metrics = response.content.decode('utf-8')
+    # Collect generator return values into list
+    return list(text_string_to_metric_families(metrics))
+
+
+def get_metric_value(metric_families: List, metric_suffix: str) -> Optional[int]:
+    for metric_family in metric_families:
+        if metric_family.name.endswith(metric_suffix):
+            return int(metric_family.samples[0].value)
+    return None
+
+
+def check_if_complete(doc_count: Optional[int], in_flight: Optional[int], no_part_count: Optional[int],
+                      prev_no_part_count: int, target: int) -> bool:
+    # Check for target doc_count
+    # TODO Add a check for partitionsCompleted = indices
+    if doc_count is not None and doc_count >= target:
+        # Check for idle pipeline
+        if in_flight is not None and in_flight == 0:
+            # No-partitions metrics should steadily tick up
+            if no_part_count is not None and no_part_count > prev_no_part_count > 0:
+                return True
+    return False
+
+
+def run(args: MigrationMonitorParams, wait_seconds: int = 30) -> None:
+    # TODO Remove hardcoded EndpointInfo
+    default_auth = ('admin', 'admin')
+    endpoint = EndpointInfo(args.dp_endpoint, default_auth, False)
+    prev_no_partitions_count = 0
+    terminal = False
+    while not terminal:
+        # If the API call fails, the response is empty
+        metrics = fetch_prometheus_metrics(endpoint)
+        if metrics is not None:
+            success_docs = get_metric_value(metrics, __DOC_SUCCESS_METRIC)
+            rec_in_flight = get_metric_value(metrics, __RECORDS_IN_FLIGHT_METRIC)
+            no_partitions_count = get_metric_value(metrics, __NO_PARTITIONS_METRIC)
+            terminal = check_if_complete(success_docs, rec_in_flight, no_partitions_count,
+                                         prev_no_partitions_count, args.target_count)
+            if not terminal:
+                # Save no_partitions_count
+                prev_no_partitions_count = no_partitions_count
+
+        if not terminal:
+            time.sleep(wait_seconds)
+    # Loop terminated, shut down the Data Prepper pipeline
+    shutdown_pipeline(endpoint)
+
+
+if __name__ == '__main__':  # pragma no cover
+    # Set up parsing for command line arguments
+    arg_parser = argparse.ArgumentParser(
+        prog="python monitor.py",
+        description="""Monitoring process for a running Data Prepper pipeline.
+        The first input is the Data Prepper URL endpoint.
+        The second input is the target doc_count for termination.""",
+        formatter_class=argparse.RawTextHelpFormatter
+    )
+    # Required positional arguments
+    arg_parser.add_argument(
+        "dp_endpoint",
+        help="URL endpoint for the running Data Prepper process"
+    )
+    arg_parser.add_argument(
+        "target_count",
+        type=int,
+        help="Target doc_count to reach, after which the Data Prepper pipeline will be terminated"
+    )
+    namespace = arg_parser.parse_args()
+    print("\n##### Starting monitor tool... #####\n")
+    run(MigrationMonitorParams(namespace.target_count, namespace.dp_endpoint))
+    print("\n##### Ending monitor tool... #####\n")
diff --git a/FetchMigration/python/migration_monitor_params.py b/FetchMigration/python/migration_monitor_params.py
new file mode 100644
index 000000000..b147b978d
--- /dev/null
+++ b/FetchMigration/python/migration_monitor_params.py
@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class MigrationMonitorParams:
+    target_count: int
+    dp_endpoint: str = "https://localhost:4900"
diff --git a/index_configuration_tool/main.py b/FetchMigration/python/pre_migration.py
similarity index 79%
rename from index_configuration_tool/main.py
rename to FetchMigration/python/pre_migration.py
index 2f2d69f47..eda146f06 100644
--- a/index_configuration_tool/main.py
+++ b/FetchMigration/python/pre_migration.py
@@ -6,6 +6,10 @@
 import utils
 
 # Constants
+from endpoint_info import EndpointInfo
+from pre_migration_params import PreMigrationParams
+from pre_migration_result import PreMigrationResult
+
 SUPPORTED_ENDPOINTS = ["opensearch", "elasticsearch"]
 SOURCE_KEY = "source"
 SINK_KEY = "sink"
@@ -36,19 +40,14 @@ def get_auth(input_data: dict) -> Optional[tuple]:
         return input_data[USER_KEY], input_data[PWD_KEY]
 
 
-def get_endpoint_info(plugin_config: dict) -> tuple:
+def get_endpoint_info(plugin_config: dict) -> EndpointInfo:
     # "hosts" can be a simple string, or an array of hosts for Logstash to hit.
     # This tool needs one accessible host, so we pick the first entry in the latter case.
-    endpoint = plugin_config[HOSTS_KEY][0] if type(plugin_config[HOSTS_KEY]) is list else plugin_config[HOSTS_KEY]
-    endpoint += "/"
-    return endpoint, get_auth(plugin_config)
-
-
-def fetch_all_indices_by_plugin(plugin_config: dict) -> dict:
-    endpoint, auth_tuple = get_endpoint_info(plugin_config)
+    url = plugin_config[HOSTS_KEY][0] if type(plugin_config[HOSTS_KEY]) is list else plugin_config[HOSTS_KEY]
+    url += "/"
     # verify boolean will be the inverse of the insecure SSL key, if present
     should_verify = not is_insecure(plugin_config)
-    return index_operations.fetch_all_indices(endpoint, auth_tuple, should_verify)
+    return EndpointInfo(url, get_auth(plugin_config), should_verify)
 
 
 def check_supported_endpoint(config: dict) -> Optional[tuple]:
@@ -112,7 +111,6 @@ def write_output(yaml_data: dict, new_indices: set, output_path: str):
     source_config[INDICES_KEY] = source_indices
     with open(output_path, 'w') as out_file:
         yaml.dump(yaml_data, out_file)
-    print("Wrote output YAML pipeline to: " + output_path)
 
 
 # Computes differences in indices between source and target.
@@ -138,50 +136,64 @@ def get_index_differences(source: dict, target: dict) -> tuple[set, set, set]:
 
 # The order of data in the tuple is:
 # (indices to create), (identical indices), (indices with conflicts)
-def print_report(index_differences: tuple[set, set, set]):  # pragma no cover
+def print_report(index_differences: tuple[set, set, set], count: int):  # pragma no cover
     print("Identical indices in the target cluster (no changes will be made): " +
           utils.string_from_set(index_differences[1]))
     print("Indices in target cluster with conflicting settings/mappings: " +
           utils.string_from_set(index_differences[2]))
     print("Indices to create: " + utils.string_from_set(index_differences[0]))
+    print("Total documents to be moved: " + str(count))
+
+
+def compute_endpoint_and_fetch_indices(config: dict, key: str) -> tuple[EndpointInfo, dict]:
+    endpoint = get_supported_endpoint(config, key)
+    # Endpoint is a tuple of (type, config)
+    endpoint_info = get_endpoint_info(endpoint[1])
+    indices = index_operations.fetch_all_indices(endpoint_info)
+    return endpoint_info, indices
 
 
-def run(args: argparse.Namespace) -> None:
+def run(args: PreMigrationParams) -> PreMigrationResult:
+    # Sanity check
+    if not args.report and len(args.output_file) == 0:
+        raise ValueError("No output file specified")
     # Parse and validate pipelines YAML file
     with open(args.config_file_path, 'r') as pipeline_file:
         dp_config = yaml.safe_load(pipeline_file)
     # We expect the Data Prepper pipeline to only have a single top-level value
     pipeline_config = next(iter(dp_config.values()))
     validate_pipeline_config(pipeline_config)
-    # Endpoint is a tuple of (type, config)
-    endpoint = get_supported_endpoint(pipeline_config, SOURCE_KEY)
-    # Fetch all indices from source cluster
-    source_indices = fetch_all_indices_by_plugin(endpoint[1])
-    # Fetch all indices from target cluster
-    # TODO Refactor this to avoid duplication with fetch_all_indices_by_plugin
-    endpoint = get_supported_endpoint(pipeline_config, SINK_KEY)
-    target_endpoint, target_auth = get_endpoint_info(endpoint[1])
-    target_indices = index_operations.fetch_all_indices(target_endpoint, target_auth)
+    # Fetch EndpointInfo and indices
+    source_endpoint_info, source_indices = compute_endpoint_and_fetch_indices(pipeline_config, SOURCE_KEY)
+    target_endpoint_info, target_indices = compute_endpoint_and_fetch_indices(pipeline_config, SINK_KEY)
     # Compute index differences and print report
     diff = get_index_differences(source_indices, target_indices)
-    if args.report:
-        print_report(diff)
     # The first element in the tuple is the set of indices to create
     indices_to_create = diff[0]
+    result = PreMigrationResult()
+    if indices_to_create:
+        result.created_indices = indices_to_create
+        result.target_doc_count = index_operations.doc_count(indices_to_create, source_endpoint_info)
+    if args.report:
+        print_report(diff, result.target_doc_count)
     if indices_to_create:
         # Write output YAML
-        write_output(dp_config, indices_to_create, args.output_file)
+        if len(args.output_file) > 0:
+            write_output(dp_config, indices_to_create, args.output_file)
+            if args.report:
+                print("Wrote output YAML pipeline to: " + args.output_file)
         if not args.dryrun:
             index_data = dict()
             for index_name in indices_to_create:
                 index_data[index_name] = source_indices[index_name]
-            index_operations.create_indices(index_data, target_endpoint, target_auth)
+            index_operations.create_indices(index_data, target_endpoint_info)
+    return result
 
 
 if __name__ == '__main__':  # pragma no cover
     # Set up parsing for command line arguments
     arg_parser = argparse.ArgumentParser(
-        prog="python main.py",
+        prog="python pre_migration.py",
         description="This tool creates indices on a target cluster based on the contents of a source cluster.\n" +
         "The first input to the tool is a path to a Data Prepper pipeline YAML file, which is parsed to obtain " +
         "the source and target cluster endpoints.\nThe second input is an output path to which a modified version " +
@@ -191,20 +203,21 @@ def run(args: argparse.Namespace) -> None:
         "along with indices that are identical or have conflicting settings/mappings.",
         formatter_class=argparse.RawTextHelpFormatter
     )
-    # Positional, required arguments
+    # Required positional argument
     arg_parser.add_argument(
         "config_file_path",
         help="Path to the Data Prepper pipeline YAML file to parse for source and target endpoint information"
     )
+    # Optional positional argument
     arg_parser.add_argument(
         "output_file",
+        nargs='?', default="",
         help="Output path for the Data Prepper pipeline YAML file that will be generated"
     )
-    # Optional arguments
+    # Flags
     arg_parser.add_argument("--report", "-r", action="store_true",
                             help="Print a report of the index differences")
     arg_parser.add_argument("--dryrun", action="store_true",
                             help="Skips the actual creation of indices on the target cluster")
-    print("\n##### Starting index configuration tool... #####\n")
-    run(arg_parser.parse_args())
-    print("\n##### Index configuration tool has completed! #####\n")
+    namespace = arg_parser.parse_args()
+    run(PreMigrationParams(namespace.config_file_path, namespace.output_file, namespace.report, namespace.dryrun))
diff --git a/FetchMigration/python/pre_migration_params.py b/FetchMigration/python/pre_migration_params.py
new file mode 100644
index 000000000..f6b01b492
--- /dev/null
+++ b/FetchMigration/python/pre_migration_params.py
@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class PreMigrationParams:
+    config_file_path: str
+    output_file: str = ""
+    report: bool = False
+    dryrun: bool = False
diff --git a/FetchMigration/python/pre_migration_result.py b/FetchMigration/python/pre_migration_result.py
new file mode 100644
index 000000000..65cac54fd
--- /dev/null
+++ b/FetchMigration/python/pre_migration_result.py
@@ -0,0 +1,7 @@
+from dataclasses import dataclass, field
+
+
+@dataclass
+class PreMigrationResult:
+    target_doc_count: int = 0
+    created_indices: set = field(default_factory=set)
diff --git a/index_configuration_tool/requirements.txt b/FetchMigration/python/requirements.txt
similarity index 70%
rename from index_configuration_tool/requirements.txt
rename to FetchMigration/python/requirements.txt
index 1a7d17eef..377006ee9 100644
--- a/index_configuration_tool/requirements.txt
+++ b/FetchMigration/python/requirements.txt
@@ -1,4 +1,5 @@
 jsondiff>=2.0.0
+prometheus-client>=0.17.1
 pyyaml>=6.0
 requests>=2.28.2
 responses>=0.23.1
\ No newline at end of file
diff --git a/index_configuration_tool/tests/__init__.py b/FetchMigration/python/tests/__init__.py
similarity index 100%
rename from index_configuration_tool/tests/__init__.py
rename to FetchMigration/python/tests/__init__.py
diff --git a/index_configuration_tool/tests/resources/expected_parse_output.pickle b/FetchMigration/python/tests/resources/expected_parse_output.pickle
similarity index 100%
rename from index_configuration_tool/tests/resources/expected_parse_output.pickle
rename to FetchMigration/python/tests/resources/expected_parse_output.pickle
diff --git a/index_configuration_tool/tests/resources/test_pipeline_input.yaml b/FetchMigration/python/tests/resources/test_pipeline_input.yaml
similarity index 100%
rename from index_configuration_tool/tests/resources/test_pipeline_input.yaml
rename to FetchMigration/python/tests/resources/test_pipeline_input.yaml
diff --git a/index_configuration_tool/tests/test_constants.py b/FetchMigration/python/tests/test_constants.py
similarity index 100%
rename from index_configuration_tool/tests/test_constants.py
rename to FetchMigration/python/tests/test_constants.py
diff --git a/index_configuration_tool/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py
similarity index 53%
rename from index_configuration_tool/tests/test_index_operations.py
rename to FetchMigration/python/tests/test_index_operations.py
index 0a06303f9..98ace7cae 100644
--- a/index_configuration_tool/tests/test_index_operations.py
+++ b/FetchMigration/python/tests/test_index_operations.py
@@ -1,10 +1,12 @@
 import copy
 import unittest
 
+import requests
 import responses
 from responses import matchers
 
 import index_operations
+from endpoint_info import EndpointInfo
 from tests import test_constants
 
 
@@ -14,7 +16,7 @@ def test_fetch_all_indices(self):
         # Set up GET response
         responses.get(test_constants.SOURCE_ENDPOINT + "*", json=test_constants.BASE_INDICES_DATA)
         # Now send request
-        index_data = index_operations.fetch_all_indices(test_constants.SOURCE_ENDPOINT)
+        index_data = index_operations.fetch_all_indices(EndpointInfo(test_constants.SOURCE_ENDPOINT))
         self.assertEqual(3, len(index_data.keys()))
         # Test that internal data has been filtered, but non-internal data is retained
         index_settings = index_data[test_constants.INDEX1_NAME][test_constants.SETTINGS_KEY]
@@ -32,7 +34,28 @@ def test_create_indices(self):
                       match=[matchers.json_params_matcher(test_data[test_constants.INDEX2_NAME])])
         responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX3_NAME,
                       match=[matchers.json_params_matcher(test_data[test_constants.INDEX3_NAME])])
-        index_operations.create_indices(test_data, test_constants.TARGET_ENDPOINT, None)
+        index_operations.create_indices(test_data, EndpointInfo(test_constants.TARGET_ENDPOINT))
+
+    @responses.activate
+    def test_create_indices_exception(self):
+        # Set up expected PUT calls with a mock response status
+        test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA)
+        del test_data[test_constants.INDEX2_NAME]
+        del test_data[test_constants.INDEX3_NAME]
+        responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX1_NAME,
+                      body=requests.Timeout())
+        self.assertRaises(RuntimeError, index_operations.create_indices, test_data,
+                          EndpointInfo(test_constants.TARGET_ENDPOINT))
+
+    @responses.activate
+    def test_doc_count(self):
+        test_indices = {test_constants.INDEX1_NAME, test_constants.INDEX2_NAME}
+        expected_count_endpoint = test_constants.SOURCE_ENDPOINT + ",".join(test_indices) + "/_count"
+        mock_count_response = {"count": "10"}
+        responses.get(expected_count_endpoint, json=mock_count_response)
+        # Now send request
+        count_value = index_operations.doc_count(test_indices, EndpointInfo(test_constants.SOURCE_ENDPOINT))
+        self.assertEqual(10, count_value)
 
 
 if __name__ == '__main__':
diff --git a/FetchMigration/python/tests/test_migration_monitor.py b/FetchMigration/python/tests/test_migration_monitor.py
new file mode 100644
index 000000000..8a7907bf1
--- /dev/null
+++ b/FetchMigration/python/tests/test_migration_monitor.py
@@ -0,0 +1,140 @@
+import unittest
+from unittest.mock import patch, MagicMock, PropertyMock
+
+import requests
+import responses
+from prometheus_client.parser import text_string_to_metric_families
+
+import migration_monitor
+from endpoint_info import EndpointInfo
+
+# Constants
+from migration_monitor_params import MigrationMonitorParams
+
+TEST_ENDPOINT = "test"
+TEST_AUTH = ("user", "pass")
+TEST_FLAG = False
+TEST_METRIC_NAME = "test_metric"
+TEST_METRIC_VALUE = 123.45
+TEST_PROMETHEUS_METRIC_STRING = "# HELP " + TEST_METRIC_NAME + " Unit Test Metric\n"\
+    + "# TYPE " + TEST_METRIC_NAME + " gauge\n" \
+    + TEST_METRIC_NAME + "{serviceName=\"unittest\",} " + str(TEST_METRIC_VALUE)
+
+
+class TestMigrationMonitor(unittest.TestCase):
+    @patch('requests.post')
+    def test_shutdown(self, mock_post: MagicMock):
+        expected_shutdown_url = TEST_ENDPOINT + "/shutdown"
+        test_endpoint = EndpointInfo(TEST_ENDPOINT, TEST_AUTH, TEST_FLAG)
+        migration_monitor.shutdown_pipeline(test_endpoint)
+        mock_post.assert_called_once_with(expected_shutdown_url, auth=TEST_AUTH, verify=TEST_FLAG)
+
+    @patch('requests.get')
+    def test_fetch_prometheus_metrics(self, mock_get: MagicMock):
+        expected_url = TEST_ENDPOINT + "/metrics/prometheus"
+        # Set up GET response
+        mock_response = MagicMock()
+        # content is a property
+        mock_content = PropertyMock(return_value=bytes(TEST_PROMETHEUS_METRIC_STRING, "utf-8"))
+        type(mock_response).content = mock_content
+        mock_get.return_value = mock_response
+        # Test fetch
+        raw_metrics_list = migration_monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT))
+        mock_get.assert_called_once_with(expected_url, auth=None, verify=True)
+        self.assertEqual(1, len(raw_metrics_list))
+        test_metric = raw_metrics_list[0]
+        self.assertEqual(TEST_METRIC_NAME, test_metric.name)
+        self.assertTrue(len(test_metric.type) > 0)
+        self.assertTrue(len(test_metric.documentation) > 0)
+        self.assertEqual(1, len(test_metric.samples))
+        test_sample = test_metric.samples[0]
+        self.assertEqual(TEST_METRIC_NAME, test_sample.name)
+        self.assertEqual(TEST_METRIC_VALUE, test_sample.value)
+        self.assertTrue(len(test_sample.labels) > 0)
+
+    @responses.activate
+    def test_fetch_prometheus_metrics_failure(self):
+        # Set up expected GET call with a mock exception
+        expected_url = TEST_ENDPOINT + "/metrics/prometheus"
+        responses.get(expected_url, body=requests.Timeout())
+        # Test fetch
+        result = migration_monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT))
+        self.assertIsNone(result)
+
+    def test_get_metric_value(self):
+        # Return value is an int
+        expected_val = int(TEST_METRIC_VALUE)
+        test_input = list(text_string_to_metric_families(TEST_PROMETHEUS_METRIC_STRING))
+        # Should fetch by suffix
+        val = migration_monitor.get_metric_value(test_input, "metric")
+        self.assertEqual(expected_val, val)
+        # No matching metric returns None
+        val = migration_monitor.get_metric_value(test_input, "invalid")
+        self.assertEqual(None, val)
+
+    @patch('migration_monitor.shutdown_pipeline')
+    @patch('time.sleep')
+    @patch('migration_monitor.check_if_complete')
+    @patch('migration_monitor.get_metric_value')
+    @patch('migration_monitor.fetch_prometheus_metrics')
+    # Note that mock objects are passed bottom-up from the patch order above
+    def test_run(self, mock_fetch: MagicMock, mock_get: MagicMock, mock_check: MagicMock, mock_sleep: MagicMock,
+                 mock_shut: MagicMock):
+        # The param values don't matter since we've mocked the check method
+        test_input = MigrationMonitorParams(1, "test")
+        mock_get.return_value = None
+        # Check will first fail, then pass
+        mock_check.side_effect = [False, True]
+        # Run test method
+        wait_time = 3
+        migration_monitor.run(test_input, wait_time)
+        # Test that fetch was called with the expected EndpointInfo
+        expected_endpoint_info = EndpointInfo(test_input.dp_endpoint, ('admin', 'admin'), False)
+        self.assertEqual(2, mock_fetch.call_count)
+        mock_fetch.assert_called_with(expected_endpoint_info)
+        # We expect one wait cycle
+        mock_sleep.assert_called_once_with(wait_time)
+        mock_shut.assert_called_once_with(expected_endpoint_info)
+
+    @patch('migration_monitor.shutdown_pipeline')
+    @patch('time.sleep')
+    @patch('migration_monitor.check_if_complete')
+    @patch('migration_monitor.get_metric_value')
+    @patch('migration_monitor.fetch_prometheus_metrics')
+    # Note that mock objects are passed bottom-up from the patch order above
+    def test_run_with_fetch_failure(self, mock_fetch: MagicMock, mock_get: MagicMock, mock_check: MagicMock,
+                                    mock_sleep: MagicMock, mock_shut: MagicMock):
+        # The param values don't matter since we've mocked the check method
+        test_input = MigrationMonitorParams(1, "test")
+        mock_get.return_value = None
+        mock_check.return_value = True
+        # Fetch call will first fail, then succeed
+        mock_fetch.side_effect = [None, MagicMock()]
+        # Run test method
+        wait_time = 3
+        migration_monitor.run(test_input, wait_time)
+        # Test that fetch was called with the expected EndpointInfo
+        expected_endpoint_info = EndpointInfo(test_input.dp_endpoint, ('admin', 'admin'), False)
+        self.assertEqual(2, mock_fetch.call_count)
+        mock_fetch.assert_called_with(expected_endpoint_info)
+        # We expect one wait cycle
+        mock_sleep.assert_called_once_with(wait_time)
+        mock_shut.assert_called_once_with(expected_endpoint_info)
+
+    def test_check_if_complete(self):
+        # If any of the optional values are missing, we are not complete
+        self.assertFalse(migration_monitor.check_if_complete(None, 0, 1, 0, 2))
+        self.assertFalse(migration_monitor.check_if_complete(2, None, 1, 0, 2))
+        self.assertFalse(migration_monitor.check_if_complete(2, 0, None, 0, 2))
+        # Target count not reached
+        self.assertFalse(migration_monitor.check_if_complete(1, None, None, 0, 2))
+        # Target count reached, but has records in flight
+        self.assertFalse(migration_monitor.check_if_complete(2, 1, None, 0, 2))
+        # Target count reached, no records in flight, but no prev no_part_count
+        self.assertFalse(migration_monitor.check_if_complete(2, 0, 1, 0, 2))
+        # Terminal state
+        self.assertTrue(migration_monitor.check_if_complete(2, 0, 2, 1, 2))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/index_configuration_tool/tests/test_main.py b/FetchMigration/python/tests/test_pre_migration.py
similarity index 72%
rename from index_configuration_tool/tests/test_main.py
rename to FetchMigration/python/tests/test_pre_migration.py
index ac72ce43f..9cfe76311 100644
--- a/index_configuration_tool/tests/test_main.py
+++ b/FetchMigration/python/tests/test_pre_migration.py
@@ -1,4 +1,3 @@
-import argparse
 import copy
 import pickle
 import random
@@ -6,7 +5,8 @@
 from typing import Optional
 from unittest.mock import patch, MagicMock, ANY
 
-import main
+import pre_migration
+from pre_migration_params import PreMigrationParams
 from tests import test_constants
 
 # Constants
@@ -37,32 +37,32 @@ def create_plugin_config(host_list: list[str],
 # Utility method to creat a test config section
 def create_config_section(plugin_config: dict) -> dict:
     valid_plugin = dict()
-    valid_plugin[random.choice(main.SUPPORTED_ENDPOINTS)] = plugin_config
+    valid_plugin[random.choice(pre_migration.SUPPORTED_ENDPOINTS)] = plugin_config
     config_section = copy.deepcopy(BASE_CONFIG_SECTION)
     config_section[TEST_KEY].append(valid_plugin)
     return config_section
 
 
-class TestMain(unittest.TestCase):
+class TestPreMigration(unittest.TestCase):
     # Run before each test
     def setUp(self) -> None:
         with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f:
             self.loaded_pipeline_config = pickle.load(f)
 
     def test_is_insecure_default_value(self):
-        self.assertFalse(main.is_insecure({}))
+        self.assertFalse(pre_migration.is_insecure({}))
 
     def test_is_insecure_top_level_key(self):
         test_input = {"key": 123, INSECURE_KEY: True}
-        self.assertTrue(main.is_insecure(test_input))
+        self.assertTrue(pre_migration.is_insecure(test_input))
 
     def test_is_insecure_nested_key(self):
         test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}}
-        self.assertTrue(main.is_insecure(test_input))
+        self.assertTrue(pre_migration.is_insecure(test_input))
 
     def test_is_insecure_missing_nested(self):
         test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}}
-        self.assertFalse(main.is_insecure(test_input))
+        self.assertFalse(pre_migration.is_insecure(test_input))
 
     def test_get_auth_returns_none(self):
         # The following inputs should not return an auth tuple:
@@ -71,11 +71,11 @@ def test_get_auth_returns_none(self):
         # - password without user
         input_list = [{}, {"username": "test"}, {"password": "test"}]
         for test_input in input_list:
-            self.assertIsNone(main.get_auth(test_input))
+            self.assertIsNone(pre_migration.get_auth(test_input))
 
     def test_get_auth_for_valid_input(self):
         # Test valid input
-        result = main.get_auth({"username": "user", "password": "pass"})
+        result = pre_migration.get_auth({"username": "user", "password": "pass"})
         self.assertEqual(tuple, type(result))
         self.assertEqual("user", result[0])
         self.assertEqual("pass", result[1])
@@ -87,30 +87,31 @@ def test_get_endpoint_info(self):
         test_password = "password"
         # Simple base case
         test_config = create_plugin_config([host_input])
-        result = main.get_endpoint_info(test_config)
-        self.assertEqual(expected_endpoint, result[0])
-        self.assertIsNone(result[1])
+        result = pre_migration.get_endpoint_info(test_config)
+        self.assertEqual(expected_endpoint, result.url)
+        self.assertIsNone(result.auth)
+        self.assertTrue(result.verify_ssl)
         # Invalid auth config
         test_config = create_plugin_config([host_input], test_user)
-        result = main.get_endpoint_info(test_config)
-        self.assertEqual(expected_endpoint, result[0])
-        self.assertIsNone(result[1])
+        result = pre_migration.get_endpoint_info(test_config)
+        self.assertEqual(expected_endpoint, result.url)
+        self.assertIsNone(result.auth)
         # Valid auth config
         test_config = create_plugin_config([host_input], user=test_user, password=test_password)
-        result = main.get_endpoint_info(test_config)
-        self.assertEqual(expected_endpoint, result[0])
-        self.assertEqual(test_user, result[1][0])
-        self.assertEqual(test_password, result[1][1])
+        result = pre_migration.get_endpoint_info(test_config)
+        self.assertEqual(expected_endpoint, result.url)
+        self.assertEqual(test_user, result.auth[0])
+        self.assertEqual(test_password, result.auth[1])
         # Array of hosts uses the first entry
         test_config = create_plugin_config([host_input, "other_host"], test_user, test_password)
-        result = main.get_endpoint_info(test_config)
-        self.assertEqual(expected_endpoint, result[0])
-        self.assertEqual(test_user, result[1][0])
-        self.assertEqual(test_password, result[1][1])
+        result = pre_migration.get_endpoint_info(test_config)
+        self.assertEqual(expected_endpoint, result.url)
+        self.assertEqual(test_user, result.auth[0])
+        self.assertEqual(test_password, result.auth[1])
 
     def test_get_index_differences_empty(self):
         # Base case should return an empty list
-        result_tuple = main.get_index_differences(dict(), dict())
+        result_tuple = pre_migration.get_index_differences(dict(), dict())
         # Invariant
         self.assertEqual(3, len(result_tuple))
         # All diffs should be empty
@@ -119,7 +120,7 @@ def test_get_index_differences_empty(self):
         self.assertEqual(set(), result_tuple[2])
 
     def test_get_index_differences_empty_target(self):
-        result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, dict())
+        result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, dict())
         # Invariant
         self.assertEqual(3, len(result_tuple))
         # No conflicts or identical indices
@@ -135,7 +136,7 @@ def test_get_index_differences_identical_index(self):
         test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA)
         del test_data[test_constants.INDEX2_NAME]
         del test_data[test_constants.INDEX3_NAME]
-        result_tuple = main.get_index_differences(test_data, test_data)
+        result_tuple = pre_migration.get_index_differences(test_data, test_data)
         # Invariant
         self.assertEqual(3, len(result_tuple))
         # No indices to move, or conflicts
@@ -150,7 +151,7 @@ def test_get_index_differences_settings_conflict(self):
         # Set up conflict in settings
         index_settings = test_data[test_constants.INDEX2_NAME][test_constants.SETTINGS_KEY]
         index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1
-        result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, test_data)
+        result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data)
         # Invariant
         self.assertEqual(3, len(result_tuple))
         # No indices to move
@@ -167,7 +168,7 @@ def test_get_index_differences_mappings_conflict(self):
         test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA)
         # Set up conflict in mappings
         test_data[test_constants.INDEX3_NAME][test_constants.MAPPINGS_KEY] = {}
-        result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, test_data)
+        result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data)
         # Invariant
         self.assertEqual(3, len(result_tuple))
         # No indices to move
@@ -182,34 +183,34 @@ def test_get_index_differences_mappings_conflict(self):
 
     def test_validate_plugin_config_unsupported_endpoints(self):
         # No supported endpoints
-        self.assertRaises(ValueError, main.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY)
+        self.assertRaises(ValueError, pre_migration.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY)
 
     def test_validate_plugin_config_missing_host(self):
         test_data = create_config_section({})
-        self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY)
+        self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY)
 
     def test_validate_plugin_config_missing_auth(self):
         test_data = create_config_section(create_plugin_config(["host"]))
-        self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY)
+        self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY)
 
     def test_validate_plugin_config_missing_password(self):
         test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=False))
-        self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY)
+        self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY)
 
     def test_validate_plugin_config_missing_user(self):
         test_data = create_config_section(create_plugin_config(["host"], password="test"))
-        self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY)
+        self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY)
 
     def test_validate_plugin_config_auth_disabled(self):
         test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=True))
         # Should complete without errors
-        main.validate_plugin_config(test_data, TEST_KEY)
+        pre_migration.validate_plugin_config(test_data, TEST_KEY)
 
     def test_validate_plugin_config_happy_case(self):
         plugin_config = create_plugin_config(["host"], "user", "password")
         test_data = create_config_section(plugin_config)
         # Should complete without errors
-        main.validate_plugin_config(test_data, TEST_KEY)
+        pre_migration.validate_plugin_config(test_data, TEST_KEY)
 
     def test_validate_pipeline_config_missing_required_keys(self):
         # Test cases:
@@ -218,24 +219,25 @@ def test_validate_pipeline_config_missing_required_keys(self):
         # - missing input
         bad_configs = [{}, {"source": {}}, {"sink": {}}]
         for config in bad_configs:
-            self.assertRaises(ValueError, main.validate_pipeline_config, config)
+            self.assertRaises(ValueError, pre_migration.validate_pipeline_config, config)
 
     def test_validate_pipeline_config_happy_case(self):
         # Get top level value
         test_config = next(iter(self.loaded_pipeline_config.values()))
-        main.validate_pipeline_config(test_config)
+        pre_migration.validate_pipeline_config(test_config)
 
-    @patch('main.write_output')
-    @patch('main.print_report')
+    @patch('index_operations.doc_count')
+    @patch('pre_migration.write_output')
+    @patch('pre_migration.print_report')
     @patch('index_operations.create_indices')
     @patch('index_operations.fetch_all_indices')
     # Note that mock objects are passed bottom-up from the patch order above
     def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: MagicMock,
-                        mock_print_report: MagicMock, mock_write_output: MagicMock):
+                        mock_print_report: MagicMock, mock_write_output: MagicMock, mock_doc_count: MagicMock):
+        mock_doc_count.return_value = 1
         index_to_create = test_constants.INDEX3_NAME
         index_with_conflict = test_constants.INDEX2_NAME
         index_exact_match = test_constants.INDEX1_NAME
-        expected_output_path = "dummy"
         # Set up expected arguments to mocks so we can verify
         expected_create_payload = {index_to_create: test_constants.BASE_INDICES_DATA[index_to_create]}
         # Print report accepts a tuple. The elements of the tuple
@@ -249,38 +251,34 @@ def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: Ma
         index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1
         # Fetch indices is called first for source, then for target
         mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data]
-        # Set up test input
-        test_input = argparse.Namespace()
-        test_input.config_file_path = test_constants.PIPELINE_CONFIG_RAW_FILE_PATH
-        test_input.output_file = expected_output_path
-        test_input.report = True
-        test_input.dryrun = False
-        main.run(test_input)
-        mock_create_indices.assert_called_once_with(expected_create_payload, test_constants.TARGET_ENDPOINT, ANY)
-        mock_print_report.assert_called_once_with(expected_diff)
-        mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_create}, expected_output_path)
-
-    @patch('main.print_report')
-    @patch('main.write_output')
+        test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, report=True)
+        pre_migration.run(test_input)
+        mock_create_indices.assert_called_once_with(expected_create_payload, ANY)
+        mock_doc_count.assert_called()
+        mock_print_report.assert_called_once_with(expected_diff, 1)
+        mock_write_output.assert_not_called()
+
+    @patch('index_operations.doc_count')
+    @patch('pre_migration.print_report')
+    @patch('pre_migration.write_output')
     @patch('index_operations.fetch_all_indices')
     # Note that mock objects are passed bottom-up from the patch order above
     def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: MagicMock,
-                        mock_print_report: MagicMock):
+                        mock_print_report: MagicMock, mock_doc_count: MagicMock):
         index_to_create = test_constants.INDEX1_NAME
+        mock_doc_count.return_value = 1
         expected_output_path = "dummy"
         # Create mock data for indices on target
         target_indices_data = copy.deepcopy(test_constants.BASE_INDICES_DATA)
         del target_indices_data[index_to_create]
         # Fetch indices is called first for source, then for target
         mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data]
-        # Set up test input
-        test_input = argparse.Namespace()
-        test_input.config_file_path = test_constants.PIPELINE_CONFIG_RAW_FILE_PATH
-        test_input.output_file = expected_output_path
-        test_input.dryrun = True
-        test_input.report = False
-        main.run(test_input)
+        test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, expected_output_path, dryrun=True)
+        test_result = pre_migration.run(test_input)
+        self.assertEqual(mock_doc_count.return_value, test_result.target_doc_count)
+        self.assertEqual({index_to_create}, test_result.created_indices)
         mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_create}, expected_output_path)
+        mock_doc_count.assert_called()
         # Report should not be printed
         mock_print_report.assert_not_called()
 
@@ -307,10 +305,14 @@ def test_write_output(self, mock_dump: MagicMock):
         del test_input['test-pipeline-input']['source']['opensearch']['indices']['include']
         # Call method under test
         with patch('builtins.open') as mock_open:
-            main.write_output(test_input, {index_to_create}, expected_output_path)
+            pre_migration.write_output(test_input, {index_to_create}, expected_output_path)
             mock_open.assert_called_once_with(expected_output_path, 'w')
             mock_dump.assert_called_once_with(expected_output_data, ANY)
 
+    def test_missing_output_file_non_report(self):
+        test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH)
+        self.assertRaises(ValueError, pre_migration.run, test_input)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/index_configuration_tool/tests/test_utils.py b/FetchMigration/python/tests/test_utils.py
similarity index 100%
rename from index_configuration_tool/tests/test_utils.py
rename to FetchMigration/python/tests/test_utils.py
diff --git a/index_configuration_tool/utils.py b/FetchMigration/python/utils.py
similarity index 100%
rename from index_configuration_tool/utils.py
rename to FetchMigration/python/utils.py
diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md
index 034a51d7f..bc0f19cec 100644
--- a/TrafficCapture/README.md
+++ b/TrafficCapture/README.md
@@ -34,9 +34,9 @@ server, recording the packet traffic of the new interactions.
 
 Learn more about its functionality and setup here: [Traffic Replayer](trafficReplayer/README.md)
 
-### OpenSearch Benchmark
+### Migration Console
 
-A container with a [script](dockerSolution/src/main/docker/openSearchBenchmark/runTestBenchmarks.sh) to run different [OpenSearch Benchmark](https://github.com/opensearch-project/opensearch-benchmark) workloads
+A container with a [script](dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh) to run different [OpenSearch Benchmark](https://github.com/opensearch-project/opensearch-benchmark) workloads
 is brought up as part of the solution.
 
 The workloads are started with the Traffic Capture Proxy Server set as the target, which will capture the requests sent by OpenSearch Benchmark,
@@ -46,7 +46,7 @@ The Traffic Replayer's logs (Tuples consisting of a request, a pair of responses
 
 Note that the script must be manually started.
 
-Partial example output of OpenSearch Benchmark:
+Partial example output of the OpenSearch Benchmark tool:
 
 ```
 
@@ -73,8 +73,62 @@ Running check-cluster-health                                                   [
 Running index-append                                                           [100% done]
 ```
 
+The `runTestBenchmarks` tool has a few configurable options. It will attempt to guess the correct endpoint to send traffic to,
+and it will automatically attach the basic auth user/password `admin`/`admin`.
+
+To set a custom endpoint, specify it with `--endpoint`, for example `./runTestBenchmarks --endpoint https://capture-proxy-domain.com:9200`.
+
+To set custom basic auth params, use `--auth_user` and `--auth_pass`. To prevent the script from attaching _any_ auth params, use the `--no_auth` flag.
+This flag overrides any other auth params, so if you use both `--auth_user` and `--no_auth`, the end result will be no auth being applied.
+
+As an example of including multiple options:
+```sh
+./runTestBenchmarks --endpoint https://capture-proxy-domain.com:9200 --auth_pass Admin123!
+```
+
+will send requests to `capture-proxy-domain.com`, using the auth combo `admin`/`Admin123!`.
+
+Support for Sigv4 signing and other auth options may be a future option.
+
+#### Understanding Data from the Replayer
+
+The Migration Console can be used to access and help interpret the data from the replayer.
+
+The data generated from the replayer is stored on an Elastic File System volume shared between the Replayer and Migration Console.
+It is mounted to the Migration Console at the path `/shared_replayer_output`. The Replayer generates files named `output_tuples.log`.
+These files are rolled over as they hit 10 MB to a series of `output_tuples-%d{yyyy-MM-dd-HH:mm}.log` files.
+
+The data in these files is in the format of JSON lines, each of which is a log message containing a specific request-response-response tuple.
+The body of the messages is sometimes gzipped which makes it difficult to represent as text in a JSON. Therefore, the body field of all requests
+and responses is base64 encoded before it is logged. This makes the files stable, but not human-readable.
+
+We have provided a utility script that can parse these files and output them to a human-readable format: the bodies are
+base64 decoded, un-gzipped if applicable, and parsed as JSON if applicable. They're then saved back to JSON format on disk.
+
+To use this utility from the Migration Console,
+```sh
+$ ./humanReadableLogs.py --help
+usage: humanReadableLogs.py [-h] [--outfile OUTFILE] infile
+
+positional arguments:
+  infile             Path to input logged tuple file.
+
+options:
+  -h, --help         show this help message and exit
+  --outfile OUTFILE  Path for output human readable tuple file.
+
+# By default, the output file is the same path as the input file, but the file name is prefixed with `readable-`.
+$ ./humanReadableLogs.py /shared_replayer_output/tuples.log
+Input file: /shared_replayer_output/tuples.log; Output file: /shared_replayer_output/readable-tuples.log
+
+# A specific output file can also be specified.
+$ ./humanReadableLogs.py /shared_replayer_output/tuples.log --outfile local-tuples.log
+Input file: /shared_replayer_output/tuples.log; Output file: local-tuples.log
+```
+
 ### Capture Kafka Offloader
 
 The Capture Kafka Offloader will act as a Kafka Producer for offloading captured traffic logs to the configured Kafka cluster.
 
 Learn more about its functionality and setup here: [Capture Kafka Offloader](captureKafkaOffloader/README.md)
+
diff --git a/TrafficCapture/buildSrc/build.gradle b/TrafficCapture/buildSrc/build.gradle
index ef08cc9c1..44eb3eee1 100644
--- a/TrafficCapture/buildSrc/build.gradle
+++ b/TrafficCapture/buildSrc/build.gradle
@@ -17,3 +17,11 @@ repositories {
 dependencies {
     implementation 'com.bmuschko:gradle-docker-plugin:9.3.1'
 }
+
+tasks.withType(Tar){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
+
+tasks.withType(Zip){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
diff --git a/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy b/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy
index 1864adaa4..e7c01f202 100644
--- a/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy
+++ b/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy
@@ -69,13 +69,27 @@ class CommonUtils {
             //defaultCommand('/runJavaWithClasspath.sh', '...')
         }
     }
+
+    static def wasRequestedVersionReleasedBeforeTargetVersion(String requested, String target) {
+        def requestedParts = requested.split('\\.')*.toInteger()
+        def targetParts = target.split('\\.')*.toInteger()
+
+        for (int i = 0; i < 3; i++) {
+            if (requestedParts[i] < targetParts[i]) {
+                return true
+            } else if (requestedParts[i] > targetParts[i]) {
+                return false
+            }
+        }
+        return false // In this case, versions are equal
+    }
 }
 
 class CommonConfigurations {
     static void applyCommonConfigurations(Project project) {
         project.configurations.all {
             resolutionStrategy.dependencySubstitution {
-                substitute module('org.apache.xmlgraphics:batik-codec') using module('org.apache.xmlgraphics:batik-all:1.15')
+                substitute module('org.apache.xmlgraphics:batik-codec') using module('org.apache.xmlgraphics:batik-all:1.17')
             }
         }
     }
diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle
index cd7a20e9f..122c4fb2b 100644
--- a/TrafficCapture/captureKafkaOffloader/build.gradle
+++ b/TrafficCapture/captureKafkaOffloader/build.gradle
@@ -13,10 +13,14 @@ dependencies {
     implementation project(':captureOffloader')
     implementation 'org.projectlombok:lombok:1.18.26'
     implementation 'com.google.protobuf:protobuf-java:3.22.2'
-    implementation 'org.apache.kafka:kafka-clients:3.5.0'
-    implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.7'
+    implementation 'org.apache.kafka:kafka-clients:3.5.1'
+    implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.9'
     implementation 'org.slf4j:slf4j-api:2.0.7'
     testImplementation project(':captureProtobufs')
     testImplementation 'org.mockito:mockito-core:4.6.1'
     testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1'
+    testImplementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0'
+    testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0'
+    testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0'
+    testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7'
 }
\ No newline at end of file
diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java
index b14db8368..1d60c361f 100644
--- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java
+++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java
@@ -14,12 +14,14 @@
 import java.util.Arrays;
 import java.util.WeakHashMap;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicLong;
 
 @Slf4j
 public class KafkaCaptureFactory implements IConnectionCaptureFactory {
 
     private static final String DEFAULT_TOPIC_NAME_FOR_TRAFFIC = "logging-traffic-topic";
+    // This value encapsulates overhead we should reserve for a given Producer record to account for record key bytes and
+    // general Kafka message overhead
+    public static final int KAFKA_MESSAGE_OVERHEAD_BYTES = 500;
 
     private final String nodeId;
     // Potential future optimization here to use a direct buffer (e.g. nio) instead of byte array
@@ -28,22 +30,19 @@ public class KafkaCaptureFactory implements IConnectionCaptureFactory {
     private final int bufferSize;
 
     public KafkaCaptureFactory(String nodeId, Producer<String, byte[]> producer,
-                               String topicNameForTraffic, int bufferSize) {
+                               String topicNameForTraffic, int messageSize) {
         this.nodeId = nodeId;
-        // There is likely some default timeout/retry settings we should configure here to reduce any potential blocking
-        // i.e. the Kafka cluster is unavailable
         this.producer = producer;
         this.topicNameForTraffic = topicNameForTraffic;
-        this.bufferSize = bufferSize;
+        this.bufferSize = messageSize - KAFKA_MESSAGE_OVERHEAD_BYTES;
     }
 
-    public KafkaCaptureFactory(String nodeId, Producer<String, byte[]> producer, int bufferSize) {
-        this(nodeId, producer, DEFAULT_TOPIC_NAME_FOR_TRAFFIC, bufferSize);
+    public KafkaCaptureFactory(String nodeId, Producer<String, byte[]> producer, int messageSize) {
+        this(nodeId, producer, DEFAULT_TOPIC_NAME_FOR_TRAFFIC, messageSize);
     }
 
     @Override
     public IChannelConnectionCaptureSerializer createOffloader(String connectionId) throws IOException {
-        AtomicLong supplierCallCounter = new AtomicLong();
         // This array is only an indirection to work around Java's constraint that lambda values are final
         CompletableFuture[] singleAggregateCfRef = new CompletableFuture[1];
         singleAggregateCfRef[0] = CompletableFuture.completedFuture(null);
@@ -55,15 +54,17 @@ public IChannelConnectionCaptureSerializer createOffloader(String connectionId)
                 codedStreamToByteStreamMap.put(cos, bb);
                 return cos;
             },
-            (codedOutputStream) -> {
+            (captureSerializerResult) -> {
                 try {
+                    CodedOutputStream codedOutputStream = captureSerializerResult.getCodedOutputStream();
                     ByteBuffer byteBuffer = codedStreamToByteStreamMap.get(codedOutputStream);
                     codedStreamToByteStreamMap.remove(codedOutputStream);
-                    String recordId = String.format("%s_%d", connectionId, supplierCallCounter.incrementAndGet());
+                    String recordId = String.format("%s.%d", connectionId, captureSerializerResult.getTrafficStreamIndex());
                     ProducerRecord<String, byte[]> record = new ProducerRecord<>(topicNameForTraffic, recordId,
                         Arrays.copyOfRange(byteBuffer.array(), 0, byteBuffer.position()));
                     // Used to essentially wrap Future returned by Producer to CompletableFuture
                     CompletableFuture cf = new CompletableFuture<>();
+                    log.debug("Sending Kafka producer record: {} for topic: {}", recordId, topicNameForTraffic);
                     // Async request to Kafka cluster
                     producer.send(record, handleProducerRecordSent(cf, recordId));
                     // Note that ordering is not guaranteed to be preserved here
@@ -76,16 +77,26 @@ public IChannelConnectionCaptureSerializer createOffloader(String connectionId)
             });
     }
 
+    /**
+     *  The default KafkaProducer comes with built-in retry and error-handling logic that suits many cases. From the
+     *  documentation here for retry: https://kafka.apache.org/35/javadoc/org/apache/kafka/clients/producer/KafkaProducer.html
+     *  "If the request fails, the producer can automatically retry. The retries setting defaults to Integer.MAX_VALUE,
+     *  and it's recommended to use delivery.timeout.ms to control retry behavior, instead of retries."
+     *
+     *  Apart from this the KafkaProducer has logic for deciding whether an error is transient and should be
+     *  retried or not retried at all: https://kafka.apache.org/35/javadoc/org/apache/kafka/common/errors/RetriableException.html
+     *  as well as basic retry backoff
+     */
     private Callback handleProducerRecordSent(CompletableFuture cf, String recordId) {
         return (metadata, exception) -> {
-            cf.complete(metadata);
-
             if (exception != null) {
-                log.error(String.format("Error sending producer record: %s", recordId), exception);
+                log.error("Error sending producer record: {}", recordId, exception);
+                cf.completeExceptionally(exception);
             }
-            else if (log.isDebugEnabled()) {
-                log.debug(String.format("Kafka producer record: %s has finished sending for topic: %s and partition %d",
-                    recordId, metadata.topic(), metadata.partition()));
+            else {
+                log.debug("Kafka producer record: {} has finished sending for topic: {} and partition {}",
+                    recordId, metadata.topic(), metadata.partition());
+                cf.complete(metadata);
             }
         };
     }
diff --git a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java
index 3e2798109..041654f9e 100644
--- a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java
+++ b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java
@@ -1,9 +1,18 @@
 package org.opensearch.migrations.trafficcapture.kafkaoffloader;
 
 import io.netty.buffer.Unpooled;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.ApiVersions;
 import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.MockProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.record.AbstractRecords;
+import org.apache.kafka.common.record.CompressionType;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -13,23 +22,76 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Clock;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
+@Slf4j
 @ExtendWith(MockitoExtension.class)
 public class KafkaCaptureFactoryTest {
 
     public static final String TEST_NODE_ID_STRING = "test_node_id";
     @Mock
     private Producer<String, byte[]> mockProducer;
+    private String connectionId = "0242c0fffea82008-0000000a-00000003-62993a3207f92af6-9093ce33";
+    private String topic = "test_topic";
 
-    private String connectionId = "test1234";
 
+    @Test
+    public void testLargeRequestIsWithinKafkaMessageSizeLimit() throws IOException, ExecutionException, InterruptedException {
+        final var referenceTimestamp = Instant.now(Clock.systemUTC());
+
+        int maxAllowableMessageSize = 1024*1024;
+        MockProducer<String, byte[]> producer = new MockProducer<>(true, new StringSerializer(), new ByteArraySerializer());
+        KafkaCaptureFactory kafkaCaptureFactory =
+            new KafkaCaptureFactory(TEST_NODE_ID_STRING, producer, maxAllowableMessageSize);
+        IChannelConnectionCaptureSerializer serializer = kafkaCaptureFactory.createOffloader(connectionId);
+
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < 15000; i++) {
+            sb.append("{ \"create\": { \"_index\": \"office-index\" } }\n{ \"title\": \"Malone's Cones\", \"year\": 2013 }\n");
+        }
+        Assertions.assertTrue(sb.toString().getBytes().length > 1024*1024);
+        byte[] fakeDataBytes = sb.toString().getBytes(StandardCharsets.UTF_8);
+        var bb = Unpooled.wrappedBuffer(fakeDataBytes);
+        serializer.addReadEvent(referenceTimestamp, bb);
+        CompletableFuture future = serializer.flushCommitAndResetStream(true);
+        future.get();
+        for (ProducerRecord<String, byte[]> record : producer.history()) {
+            int recordSize = calculateRecordSize(record, null);
+            Assertions.assertTrue(recordSize <= maxAllowableMessageSize);
+            int largeIdRecordSize = calculateRecordSize(record, connectionId + ".9999999999");
+            Assertions.assertTrue(largeIdRecordSize <= maxAllowableMessageSize);
+        }
+        bb.release();
+        producer.close();
+    }
+
+    /**
+     * This size calculation is based off the KafkaProducer client request size validation check done when Producer
+     * records are sent. This validation appears to be consistent for several versions now, here is a reference to
+     * version 3.5 at the time of writing this: https://github.com/apache/kafka/blob/3.5/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java#L1030-L1032.
+     * It is, however, subject to change which may make this test scenario more suited for an integration test where
+     * a KafkaProducer does not need to be mocked.
+     */
+    private int calculateRecordSize(ProducerRecord<String, byte[]> record, String recordKeySubstitute) {
+        StringSerializer stringSerializer = new StringSerializer();
+        ByteArraySerializer byteArraySerializer = new ByteArraySerializer();
+        String recordKey = recordKeySubstitute == null ? record.key() : recordKeySubstitute;
+        byte[] serializedKey = stringSerializer.serialize(record.topic(), record.headers(),  recordKey);
+        byte[] serializedValue = byteArraySerializer.serialize(record.topic(), record.headers(), record.value());
+        ApiVersions apiVersions = new ApiVersions();
+        stringSerializer.close();
+        byteArraySerializer.close();
+        return AbstractRecords.estimateSizeInBytesUpperBound(apiVersions.maxUsableProduceMagic(),
+            CompressionType.NONE, serializedKey, serializedValue, record.headers().toArray());
+    }
 
     @Test
     public void testLinearOffloadingIsSuccessful() throws IOException {
@@ -60,17 +122,17 @@ public void testLinearOffloadingIsSuccessful() throws IOException {
         Assertions.assertEquals(false, cf1.isDone());
         Assertions.assertEquals(false, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(0).onCompletion(null, null);
+        recordSentCallbacks.get(0).onCompletion(generateRecordMetadata(topic, 1), null);
 
         Assertions.assertEquals(true, cf1.isDone());
         Assertions.assertEquals(false, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(1).onCompletion(null, null);
+        recordSentCallbacks.get(1).onCompletion(generateRecordMetadata(topic, 2), null);
 
         Assertions.assertEquals(true, cf1.isDone());
         Assertions.assertEquals(true, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(2).onCompletion(null, null);
+        recordSentCallbacks.get(2).onCompletion(generateRecordMetadata(topic, 1), null);
 
         Assertions.assertEquals(true, cf1.isDone());
         Assertions.assertEquals(true, cf2.isDone());
@@ -108,19 +170,19 @@ public void testOngoingFuturesAreAggregated() throws IOException {
         Assertions.assertEquals(false, cf1.isDone());
         Assertions.assertEquals(false, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(2).onCompletion(null, null);
+        recordSentCallbacks.get(2).onCompletion(generateRecordMetadata(topic, 1), null);
 
         // Assert that even though particular final producer record has finished sending, its predecessors are incomplete
         // and thus this wrapper cf is also incomplete
         Assertions.assertEquals(false, cf1.isDone());
         Assertions.assertEquals(false, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(1).onCompletion(null, null);
+        recordSentCallbacks.get(1).onCompletion(generateRecordMetadata(topic, 2), null);
 
         Assertions.assertEquals(false, cf1.isDone());
         Assertions.assertEquals(false, cf2.isDone());
         Assertions.assertEquals(false, cf3.isDone());
-        recordSentCallbacks.get(0).onCompletion(null, null);
+        recordSentCallbacks.get(0).onCompletion(generateRecordMetadata(topic, 3), null);
 
         Assertions.assertEquals(true, cf1.isDone());
         Assertions.assertEquals(true, cf2.isDone());
@@ -128,4 +190,9 @@ public void testOngoingFuturesAreAggregated() throws IOException {
 
         mockProducer.close();
     }
+
+    private RecordMetadata generateRecordMetadata(String topicName, int partition) {
+        TopicPartition topicPartition = new TopicPartition(topicName, partition);
+        return new RecordMetadata(topicPartition, 0, 0, 0, 0, 0);
+    }
 }
diff --git a/TrafficCapture/captureKafkaOffloader/src/test/resources/log4j2.properties b/TrafficCapture/captureKafkaOffloader/src/test/resources/log4j2.properties
new file mode 100644
index 000000000..ff1fd89e6
--- /dev/null
+++ b/TrafficCapture/captureKafkaOffloader/src/test/resources/log4j2.properties
@@ -0,0 +1,11 @@
+status = error
+
+# Root logger options
+rootLogger.level = debug
+rootLogger.appenderRef.console.ref = Console
+
+# Console appender configuration
+appender.console.type = Console
+appender.console.name = Console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d %p %c{1.} [%t] %m%n
\ No newline at end of file
diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CaptureSerializerResult.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CaptureSerializerResult.java
new file mode 100644
index 000000000..c69966a0f
--- /dev/null
+++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CaptureSerializerResult.java
@@ -0,0 +1,17 @@
+package org.opensearch.migrations.trafficcapture;
+
+import com.google.protobuf.CodedOutputStream;
+import lombok.Getter;
+
+@Getter
+public class CaptureSerializerResult {
+
+    private final CodedOutputStream codedOutputStream;
+    private final int trafficStreamIndex;
+
+    public CaptureSerializerResult(CodedOutputStream codedOutputStream, int trafficStreamIndex) {
+        this.codedOutputStream = codedOutputStream;
+        this.trafficStreamIndex = trafficStreamIndex;
+    }
+
+}
diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CodedOutputStreamSizeUtil.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CodedOutputStreamSizeUtil.java
index 994a99186..01ac565ac 100644
--- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CodedOutputStreamSizeUtil.java
+++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/CodedOutputStreamSizeUtil.java
@@ -2,6 +2,7 @@
 
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.Timestamp;
+import org.opensearch.migrations.trafficcapture.protos.EndOfSegmentsIndication;
 import org.opensearch.migrations.trafficcapture.protos.TrafficObservation;
 import org.opensearch.migrations.trafficcapture.protos.TrafficStream;
 
@@ -22,28 +23,38 @@ public static int getSizeOfTimestamp(Instant t) {
     }
 
     /**
-     * This function calculates the maximum bytes needed to store a message ByteBuffer and its associated
-     * Traffic Stream overhead into a CodedOutputStream. The actual required bytes could be marginally smaller.
+     * This function calculates the maximum bytes that would be needed to store a [Read/Write]SegmentObservation, if constructed
+     * from the given ByteBuffer and associated segment field numbers and values passed in. This estimate is essentially
+     * the max size needed in the CodedOutputStream to store the provided ByteBuffer data and its associated TrafficStream
+     * overhead. The actual required bytes could be marginally smaller.
      */
-    public static int maxBytesNeededForMessage(Instant timestamp, int observationFieldNumber, int dataFieldNumber,
-        int dataCountFieldNumber, int dataCount,  ByteBuffer buffer, int flushes) {
-        // Timestamp closure bytes
+    public static int maxBytesNeededForASegmentedObservation(Instant timestamp, int observationFieldNumber, int dataFieldNumber,
+        int dataCountFieldNumber, int dataCount,  ByteBuffer buffer, int numberOfTrafficStreamsSoFar) {
+        // Timestamp required bytes
         int tsContentSize = getSizeOfTimestamp(timestamp);
-        int tsClosureSize = CodedOutputStream.computeInt32Size(TrafficObservation.TS_FIELD_NUMBER, tsContentSize) + tsContentSize;
+        int tsTagAndContentSize = CodedOutputStream.computeInt32Size(TrafficObservation.TS_FIELD_NUMBER, tsContentSize) + tsContentSize;
 
-        // Capture closure bytes
+        // Capture required bytes
         int dataSize = CodedOutputStream.computeByteBufferSize(dataFieldNumber, buffer);
         int dataCountSize = dataCountFieldNumber > 0 ? CodedOutputStream.computeInt32Size(dataCountFieldNumber, dataCount) : 0;
         int captureContentSize = dataSize + dataCountSize;
-        int captureClosureSize = CodedOutputStream.computeInt32Size(observationFieldNumber, captureContentSize) + captureContentSize;
+        int captureTagAndContentSize = CodedOutputStream.computeInt32Size(observationFieldNumber, captureContentSize) + captureContentSize;
 
-        // Observation tag and closure size needed bytes
-        int observationTagAndClosureSize = CodedOutputStream.computeInt32Size(TrafficStream.SUBSTREAM_FIELD_NUMBER, tsClosureSize + captureClosureSize);
+        // Observation and closing index required bytes
+        return bytesNeededForObservationAndClosingIndex(tsTagAndContentSize + captureTagAndContentSize, numberOfTrafficStreamsSoFar);
+    }
+
+    /**
+     * This function determines the number of bytes needed to store a TrafficObservation and a closing index for a
+     * TrafficStream, from the provided input.
+     */
+    public static int bytesNeededForObservationAndClosingIndex(int observationContentSize, int numberOfTrafficStreamsSoFar) {
+        int observationTagSize = CodedOutputStream.computeUInt32Size(TrafficStream.SUBSTREAM_FIELD_NUMBER, observationContentSize);
 
-        // Size for closing index, use arbitrary field to calculate
-        int indexSize = CodedOutputStream.computeInt32Size(TrafficStream.NUMBER_FIELD_NUMBER, flushes);
+        // Size for TrafficStream index added when flushing, use arbitrary field to calculate
+        int indexSize = CodedOutputStream.computeInt32Size(TrafficStream.NUMBEROFTHISLASTCHUNK_FIELD_NUMBER, numberOfTrafficStreamsSoFar);
 
-        return observationTagAndClosureSize + tsClosureSize + captureClosureSize + indexSize;
+        return observationTagSize + observationContentSize + indexSize;
     }
 
 
diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java
index 5d1525a2b..abd8042c7 100644
--- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java
+++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java
@@ -10,10 +10,8 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.UUID;
 import java.util.WeakHashMap;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 
 /**
@@ -68,7 +66,6 @@ private CompletableFuture closeHandler(String connectionId, ByteBuffer byteBuffe
 
     @Override
     public IChannelConnectionCaptureSerializer createOffloader(String connectionId) throws IOException {
-        AtomicInteger supplierCallCounter = new AtomicInteger();
         // This array is only an indirection to work around Java's constraint that lambda values are final
         CompletableFuture[] singleAggregateCfRef = new CompletableFuture[1];
         singleAggregateCfRef[0] = CompletableFuture.completedFuture(null);
@@ -80,8 +77,9 @@ public IChannelConnectionCaptureSerializer createOffloader(String connectionId)
                 codedStreamToFileStreamMap.put(cos, bb);
                 return cos;
             },
-            (codedOutputStream) -> {
-                CompletableFuture cf = closeHandler(connectionId, codedStreamToFileStreamMap.get(codedOutputStream), codedOutputStream, supplierCallCounter.incrementAndGet());
+            (captureSerializerResult) -> {
+                CodedOutputStream codedOutputStream = captureSerializerResult.getCodedOutputStream();
+                CompletableFuture cf = closeHandler(connectionId, codedStreamToFileStreamMap.get(codedOutputStream), codedOutputStream, captureSerializerResult.getTrafficStreamIndex());
                 singleAggregateCfRef[0] = singleAggregateCfRef[0].isDone() ? cf : CompletableFuture.allOf(singleAggregateCfRef[0], cf);
                 return singleAggregateCfRef[0];
             }
diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java
index bee38741d..b1f36aac5 100644
--- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java
+++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java
@@ -9,6 +9,7 @@
 import org.opensearch.migrations.trafficcapture.protos.CloseObservation;
 import org.opensearch.migrations.trafficcapture.protos.ConnectionExceptionObservation;
 import org.opensearch.migrations.trafficcapture.protos.EndOfMessageIndication;
+import org.opensearch.migrations.trafficcapture.protos.EndOfSegmentsIndication;
 import org.opensearch.migrations.trafficcapture.protos.ReadObservation;
 import org.opensearch.migrations.trafficcapture.protos.ReadSegmentObservation;
 import org.opensearch.migrations.trafficcapture.protos.TrafficObservation;
@@ -31,7 +32,7 @@
  * into the defined Protobuf format {@link org.opensearch.migrations.trafficcapture.protos.TrafficStream}, and then write
  * this formatted data to the provided CodedOutputStream.
  *
- * Commented throughout the class are example markers such as (i.e. 1: "1234ABCD") which line up with the textual
+ * Commented throughout the class are example markers such as (e.g. 1: "1234ABCD") which line up with the textual
  * representation of this Protobuf format to be used as a guide as fields are written. An example TrafficStream can
  * also be visualized below for reference.
  *
@@ -64,7 +65,7 @@ public class StreamChannelConnectionCaptureSerializer implements IChannelConnect
     private final static int MAX_ID_SIZE = 96;
 
     private final Supplier<CodedOutputStream> codedOutputStreamSupplier;
-    private final Function<CodedOutputStream, CompletableFuture> closeHandler;
+    private final Function<CaptureSerializerResult, CompletableFuture> closeHandler;
     private final String nodeIdString;
     private final String connectionIdString;
     private CodedOutputStream currentCodedOutputStreamOrNull;
@@ -74,7 +75,7 @@ public class StreamChannelConnectionCaptureSerializer implements IChannelConnect
 
     public StreamChannelConnectionCaptureSerializer(String nodeId, String connectionId,
                                                     Supplier<CodedOutputStream> codedOutputStreamSupplier,
-                                                    Function<CodedOutputStream, CompletableFuture> closeHandler) throws IOException {
+                                                    Function<CaptureSerializerResult, CompletableFuture> closeHandler) throws IOException {
         this.codedOutputStreamSupplier = codedOutputStreamSupplier;
         this.closeHandler = closeHandler;
         assert (nodeId == null ? 0 : CodedOutputStream.computeStringSize(TrafficStream.NODEID_FIELD_NUMBER, nodeId)) +
@@ -93,9 +94,10 @@ private CodedOutputStream getOrCreateCodedOutputStream() throws IOException {
             return currentCodedOutputStreamOrNull;
         } else {
             currentCodedOutputStreamOrNull = codedOutputStreamSupplier.get();
-            // i.e. 1: "1234ABCD"
+            // e.g. 1: "1234ABCD"
             currentCodedOutputStreamOrNull.writeString(TrafficStream.CONNECTIONID_FIELD_NUMBER, connectionIdString);
             if (nodeIdString != null) {
+                // e.g. 5: "5ae27fca-0ac4-11ee-be56-0242ac120002"
                 currentCodedOutputStreamOrNull.writeString(TrafficStream.NODEID_FIELD_NUMBER, nodeIdString);
             }
             return currentCodedOutputStreamOrNull;
@@ -112,18 +114,27 @@ private void writeObservationTag(int fieldNumber) throws IOException {
                 getWireTypeForFieldIndex(TrafficObservation.getDescriptor(), fieldNumber));
     }
 
-    private void beginSubstreamObservation(Instant timestamp, int captureTag, int captureClosureSize) throws IOException {
-
-        // i.e. 2 {
+    /**
+     * Will write the beginning fields for a TrafficObservation after first checking if sufficient space exists in the
+     * CodedOutputStream and flushing if space does not exist. This should be called before writing any observation to
+     * the TrafficStream.
+     */
+    private void beginSubstreamObservation(Instant timestamp, int captureTagFieldNumber, int captureTagLengthAndContentSize) throws IOException {
+        final var tsContentSize = CodedOutputStreamSizeUtil.getSizeOfTimestamp(timestamp);
+        final var tsTagSize = CodedOutputStream.computeInt32Size(TrafficObservation.TS_FIELD_NUMBER, tsContentSize);
+        final var captureTagNoLengthSize = CodedOutputStream.computeTagSize(captureTagFieldNumber);
+        final var observationContentSize = tsTagSize + tsContentSize + captureTagNoLengthSize + captureTagLengthAndContentSize;
+        // Ensure space is available before starting an observation
+        if (getOrCreateCodedOutputStream().spaceLeft() <
+            CodedOutputStreamSizeUtil.bytesNeededForObservationAndClosingIndex(observationContentSize, numFlushesSoFar + 1))
+        {
+            flushCommitAndResetStream(false);
+        }
+        // e.g. 2 {
         writeTrafficStreamTag(TrafficStream.SUBSTREAM_FIELD_NUMBER);
-        final var tsSize = CodedOutputStreamSizeUtil.getSizeOfTimestamp(timestamp);
-        final var captureTagSize = CodedOutputStream.computeTagSize(captureTag);
-        // Writing total size of substream closure [ts size + ts tag + capture tag + capture size]
-        getOrCreateCodedOutputStream().writeUInt32NoTag(tsSize +
-            CodedOutputStream.computeInt32Size(TrafficObservation.TS_FIELD_NUMBER, tsSize) +
-            captureTagSize +
-            captureClosureSize);
-        // i.e. 1 { 1: 1234 2: 1234 }
+        // Write observation content length
+        getOrCreateCodedOutputStream().writeUInt32NoTag(observationContentSize);
+        // e.g. 1 { 1: 1234 2: 1234 }
         writeTimestampForNowToCurrentStream(timestamp);
     }
 
@@ -161,10 +172,11 @@ public CompletableFuture<Object> flushCommitAndResetStream(boolean isFinal) thro
         }
         CodedOutputStream currentStream = getOrCreateCodedOutputStream();
         var fieldNum = isFinal ? TrafficStream.NUMBEROFTHISLASTCHUNK_FIELD_NUMBER : TrafficStream.NUMBER_FIELD_NUMBER;
-        // i.e. 3: 1
+        // e.g. 3: 1
         currentStream.writeInt32(fieldNum, ++numFlushesSoFar);
+        log.debug("Flushing the current CodedOutputStream for {}.{}", connectionIdString, numFlushesSoFar);
         currentStream.flush();
-        var future = closeHandler.apply(currentStream);
+        var future = closeHandler.apply(new CaptureSerializerResult(currentStream, numFlushesSoFar));
         //future.whenComplete((r,t)->{}); // do more cleanup stuff here once the future is complete
         currentCodedOutputStreamOrNull = null;
         return future;
@@ -230,7 +242,7 @@ private void addStringMessage(int captureFieldNumber, int dataFieldNumber,
             lengthSize = getOrCreateCodedOutputStream().computeInt32SizeNoTag(dataSize);
         }
         beginSubstreamObservation(timestamp, captureFieldNumber, dataSize + lengthSize);
-        // i.e. 4 {
+        // e.g. 4 {
         writeObservationTag(captureFieldNumber);
         if (dataSize > 0) {
             getOrCreateCodedOutputStream().writeInt32NoTag(dataSize);
@@ -254,9 +266,9 @@ private void addDataMessage(int captureFieldNumber, int dataFieldNumber, Instant
 
         // The message bytes here are not optimizing for space and instead are calculated on the worst case estimate of
         // the potentially required bytes for simplicity. This could leave ~5 bytes of unused space in the CodedOutputStream
-        // when considering the case of a message that does not need segments or the case of a smaller segment created
+        // when considering the case of a message that does not need segments or for the case of a smaller segment created
         // from a much larger message
-        int messageAndOverheadBytesLeft = CodedOutputStreamSizeUtil.maxBytesNeededForMessage(timestamp,
+        int messageAndOverheadBytesLeft = CodedOutputStreamSizeUtil.maxBytesNeededForASegmentedObservation(timestamp,
             segmentFieldNumber, segmentDataFieldNumber, segmentCountFieldNumber, 2, byteBuffer, numFlushesSoFar + 1);
         int trafficStreamOverhead = messageAndOverheadBytesLeft - byteBuffer.capacity();
 
@@ -267,7 +279,9 @@ private void addDataMessage(int captureFieldNumber, int dataFieldNumber, Instant
 
         // If our message is empty or can fit in the current CodedOutputStream no chunking is needed, and we can continue
         if (byteBuffer.limit() == 0 || messageAndOverheadBytesLeft <= getOrCreateCodedOutputStream().spaceLeft()) {
+            int minExpectedSpaceAfterObservation = getOrCreateCodedOutputStream().spaceLeft() - messageAndOverheadBytesLeft;
             addSubstreamMessage(captureFieldNumber, dataFieldNumber, timestamp, byteBuffer);
+            observationSizeSanityCheck(minExpectedSpaceAfterObservation, captureFieldNumber);
             return;
         }
 
@@ -280,12 +294,15 @@ private void addDataMessage(int captureFieldNumber, int dataFieldNumber, Instant
             bb = bb.slice();
             byteBuffer.position(byteBuffer.position() + chunkBytes);
             addSubstreamMessage(segmentFieldNumber, segmentDataFieldNumber, segmentCountFieldNumber, ++dataCount, timestamp, bb);
+            int minExpectedSpaceAfterObservation = availableCOSSpace - chunkBytes - trafficStreamOverhead;
+            observationSizeSanityCheck(minExpectedSpaceAfterObservation, segmentFieldNumber);
             // 1 to N-1 chunked messages
             if (byteBuffer.position() < byteBuffer.limit()) {
                 flushCommitAndResetStream(false);
                 messageAndOverheadBytesLeft = messageAndOverheadBytesLeft - chunkBytes;
             }
         }
+        writeEndOfSegmentMessage(timestamp);
 
     }
 
@@ -303,7 +320,7 @@ private void addSubstreamMessage(int captureFieldNumber, int dataFieldNumber, in
             captureClosureLength = CodedOutputStream.computeInt32SizeNoTag(dataSize + segmentCountSize);
         }
         beginSubstreamObservation(timestamp, captureFieldNumber, captureClosureLength + dataSize + segmentCountSize);
-        // i.e. 4 {
+        // e.g. 4 {
         writeObservationTag(captureFieldNumber);
         if (dataSize > 0) {
             // Write size of data after capture tag
@@ -407,10 +424,24 @@ private void writeEndOfHttpMessage(Instant timestamp) throws IOException {
                 CodedOutputStream.computeInt32Size(EndOfMessageIndication.HEADERSBYTELENGTH_FIELD_NUMBER, headersByteLength);
         int eomDataSize = eomPairSize + CodedOutputStream.computeInt32SizeNoTag(eomPairSize);
         beginSubstreamObservation(timestamp, TrafficObservation.ENDOFMESSAGEINDICATOR_FIELD_NUMBER, eomDataSize);
-        // i.e. 15 {
+        // e.g. 15 {
         writeObservationTag(TrafficObservation.ENDOFMESSAGEINDICATOR_FIELD_NUMBER);
         getOrCreateCodedOutputStream().writeUInt32NoTag(eomPairSize);
         getOrCreateCodedOutputStream().writeInt32(EndOfMessageIndication.FIRSTLINEBYTELENGTH_FIELD_NUMBER, firstLineByteLength);
         getOrCreateCodedOutputStream().writeInt32(EndOfMessageIndication.HEADERSBYTELENGTH_FIELD_NUMBER, headersByteLength);
     }
+
+    private void writeEndOfSegmentMessage(Instant timestamp) throws IOException {
+        beginSubstreamObservation(timestamp, TrafficObservation.SEGMENTEND_FIELD_NUMBER, 1);
+        getOrCreateCodedOutputStream().writeMessage(TrafficObservation.SEGMENTEND_FIELD_NUMBER, EndOfSegmentsIndication.getDefaultInstance());
+    }
+
+    private void observationSizeSanityCheck(int minExpectedSpaceAfterObservation, int fieldNumber) throws IOException {
+        int actualRemainingSpace = getOrCreateCodedOutputStream().spaceLeft();
+        if (actualRemainingSpace < minExpectedSpaceAfterObservation || minExpectedSpaceAfterObservation < 0) {
+            log.warn("Writing a substream (capture type: {}) for Traffic Stream: {} left {} bytes in the CodedOutputStream but we calculated " +
+                    "at least {} bytes remaining, this should be investigated", fieldNumber, connectionIdString + "." + (numFlushesSoFar + 1),
+                actualRemainingSpace, minExpectedSpaceAfterObservation);
+        }
+    }
 }
diff --git a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java
index ca16f04df..38a9c0393 100644
--- a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java
+++ b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java
@@ -10,9 +10,11 @@
 import org.opensearch.migrations.trafficcapture.protos.CloseObservation;
 import org.opensearch.migrations.trafficcapture.protos.ConnectionExceptionObservation;
 import org.opensearch.migrations.trafficcapture.protos.EndOfMessageIndication;
+import org.opensearch.migrations.trafficcapture.protos.EndOfSegmentsIndication;
 import org.opensearch.migrations.trafficcapture.protos.ReadObservation;
 import org.opensearch.migrations.trafficcapture.protos.TrafficObservation;
 import org.opensearch.migrations.trafficcapture.protos.TrafficStream;
+import org.opensearch.migrations.trafficcapture.protos.WriteObservation;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -99,7 +101,7 @@ public void testLargeReadPacketIsSplit() throws IOException, ExecutionException,
         future.get();
         bb.release();
 
-        var outputBuffersList = new ArrayList<ByteBuffer>(outputBuffersCreated);
+        var outputBuffersList = new ArrayList<>(outputBuffersCreated);
 
         var reconstitutedTrafficStreamsList = new ArrayList<TrafficStream>();
         for (int i=0; i<2; ++i) {
@@ -154,6 +156,33 @@ public void testBasicDataConsistencyWhenChunking() throws IOException, Execution
         Assertions.assertEquals(packetData, reconstructedData);
     }
 
+    @Test
+    public void testCloseObservationAfterWriteWillFlushWhenSpaceNeeded() throws IOException, ExecutionException, InterruptedException {
+        final var referenceTimestamp = Instant.ofEpochMilli(1686593191*1000);
+        String packetData = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        byte[] packetBytes = packetData.getBytes(StandardCharsets.UTF_8);
+        var outputBuffersCreated = new ConcurrentLinkedQueue<ByteBuffer>();
+        // Arbitrarily picking small buffer that can only hold one write observation and no other observations
+        var serializer = createSerializerWithTestHandler(outputBuffersCreated, 85);
+
+        var bb = Unpooled.wrappedBuffer(packetBytes);
+        serializer.addWriteEvent(referenceTimestamp, bb);
+        serializer.addCloseEvent(referenceTimestamp);
+        CompletableFuture future = serializer.flushCommitAndResetStream(true);
+        future.get();
+        bb.release();
+
+        Assertions.assertEquals(2, outputBuffersCreated.size());
+        List<TrafficObservation> observations = new ArrayList<>();
+        for (ByteBuffer buffer : outputBuffersCreated) {
+            var trafficStream = TrafficStream.parseFrom(buffer);
+            observations.addAll(trafficStream.getSubStreamList());
+        }
+        Assertions.assertEquals(2, observations.size());
+        Assertions.assertTrue(observations.get(0).hasWrite());
+        Assertions.assertTrue(observations.get(1).hasClose());
+    }
+
     @Test
     public void testEmptyPacketIsHandledForSmallCodedOutputStream()
         throws IOException, ExecutionException, InterruptedException {
@@ -214,6 +243,75 @@ public void testThatReadCanBeDeserialized() throws IOException, ExecutionExcepti
         Assertions.assertEquals(groundTruth, reconstitutedTrafficStream);
     }
 
+    @Test
+    public void testEndOfSegmentsIndicationAddedWhenChunking() throws IOException, ExecutionException, InterruptedException {
+        final var referenceTimestamp = Instant.ofEpochMilli(1686593191*1000);
+        String packetData = "";
+        for (int i = 0; i < 500; i++) {
+            packetData += "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        }
+        byte[] packetBytes = packetData.getBytes(StandardCharsets.UTF_8);
+        var outputBuffersCreated = new ConcurrentLinkedQueue<ByteBuffer>();
+        // Arbitrarily picking small buffer that can hold the overhead TrafficStream bytes as well as some
+        // data bytes but not all the data bytes and require chunking
+        var serializer = createSerializerWithTestHandler(outputBuffersCreated, 85);
+
+        var bb = Unpooled.wrappedBuffer(packetBytes);
+        serializer.addWriteEvent(referenceTimestamp, bb);
+        CompletableFuture future = serializer.flushCommitAndResetStream(true);
+        future.get();
+        bb.release();
+
+        List<TrafficObservation> observations = new ArrayList<>();
+        for (ByteBuffer buffer : outputBuffersCreated) {
+            var trafficStream = TrafficStream.parseFrom(buffer);
+            observations.addAll(trafficStream.getSubStreamList());
+        }
+
+        int foundEndOfSegments = 0;
+        for (TrafficObservation observation : observations) {
+            if (observation.hasSegmentEnd()) {
+                foundEndOfSegments++;
+                EndOfSegmentsIndication endOfSegment = observation.getSegmentEnd();
+                Assertions.assertEquals(EndOfSegmentsIndication.getDefaultInstance(), endOfSegment);
+            }
+        }
+        Assertions.assertEquals(1, foundEndOfSegments);
+    }
+
+    @Test
+    public void testEndOfSegmentsIndicationNotAddedWhenNotChunking() throws IOException, ExecutionException, InterruptedException {
+        final var referenceTimestamp = Instant.ofEpochMilli(1686593191*1000);
+        String packetData = "";
+        for (int i = 0; i < 10; i++) {
+            packetData += "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        }
+        byte[] packetBytes = packetData.getBytes(StandardCharsets.UTF_8);
+        var outputBuffersCreated = new ConcurrentLinkedQueue<ByteBuffer>();
+        // Buffer size should be large enough to hold all packetData and overhead
+        var serializer = createSerializerWithTestHandler(outputBuffersCreated, 500);
+
+        var bb = Unpooled.wrappedBuffer(packetBytes);
+        serializer.addWriteEvent(referenceTimestamp, bb);
+        CompletableFuture future = serializer.flushCommitAndResetStream(true);
+        future.get();
+        bb.release();
+
+        List<TrafficObservation> observations = new ArrayList<>();
+        for (ByteBuffer buffer : outputBuffersCreated) {
+            var trafficStream = TrafficStream.parseFrom(buffer);
+            observations.addAll(trafficStream.getSubStreamList());
+        }
+
+        int foundEndOfSegments = 0;
+        for (TrafficObservation observation : observations) {
+            if (observation.hasSegmentEnd()) {
+                foundEndOfSegments++;
+            }
+        }
+        Assertions.assertEquals(0, foundEndOfSegments);
+    }
+
     private StreamChannelConnectionCaptureSerializer
     createSerializerWithTestHandler(ConcurrentLinkedQueue<ByteBuffer> outputBuffers, int bufferSize)
             throws IOException {
@@ -228,7 +326,8 @@ public void testThatReadCanBeDeserialized() throws IOException, ExecutionExcepti
                 log.trace("Put COS: " + rval + " into map (keys="+ mapToKeyStrings(codedStreamToByteBuffersMap) +") with bytes=" + bytes);
                 return rval;
             },
-            (codedOutputStream) -> {
+            (captureSerializerResult) -> {
+            CodedOutputStream codedOutputStream = captureSerializerResult.getCodedOutputStream();
             log.trace("Getting ready to flush for " + codedOutputStream);
             log.trace("Bytes written so far... " +
                     StandardCharsets.UTF_8.decode(codedStreamToByteBuffersMap.get(codedOutputStream).duplicate()));
diff --git a/TrafficCapture/captureOffloader/src/test/resources/log4j2.properties b/TrafficCapture/captureOffloader/src/test/resources/log4j2.properties
index 620cfc1fa..ff1fd89e6 100644
--- a/TrafficCapture/captureOffloader/src/test/resources/log4j2.properties
+++ b/TrafficCapture/captureOffloader/src/test/resources/log4j2.properties
@@ -1,4 +1,4 @@
-status = error 
+status = error
 
 # Root logger options
 rootLogger.level = debug
diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle
index bc1d9378b..ecc2197b4 100644
--- a/TrafficCapture/dockerSolution/build.gradle
+++ b/TrafficCapture/dockerSolution/build.gradle
@@ -1,6 +1,6 @@
 plugins {
     id 'org.opensearch.migrations.java-library-conventions'
-    id "com.avast.gradle.docker-compose" version "0.16.12"
+    id "com.avast.gradle.docker-compose" version "0.17.4"
     id 'com.bmuschko.docker-remote-api'
 }
 
@@ -44,7 +44,7 @@ task cloneComparatorRepoIfNeeded(type: Exec) {
 
 def dockerFilesForExternalServices = [
         "elasticsearchWithSearchGuard": "elasticsearch_searchguard",
-        "openSearchBenchmark": "open_search_benchmark"
+        "migrationConsole": "migration_console"
 ]
 // Create the static docker files that aren't hosting migrations java code from this repo
 dockerFilesForExternalServices.each { projectName, dockerImageName ->
@@ -122,7 +122,7 @@ dockerCompose {
 
 task buildDockerImages {
     dependsOn buildDockerImage_elasticsearchWithSearchGuard
-    dependsOn buildDockerImage_openSearchBenchmark
+    dependsOn buildDockerImage_migrationConsole
 
     dependsOn buildDockerImage_trafficCaptureProxyServer
     dependsOn buildDockerImage_trafficReplayer
diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml
index 408fce92c..558403335 100644
--- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml
+++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml
@@ -1,23 +1,40 @@
 version: '3.7'
 services:
 
-  captureproxy:
+
+  # Run combined instance of Capture Proxy and Elasticsearch
+  capture-proxy-es:
     image: 'migrations/capture_proxy:latest'
     networks:
       - migrations
     ports:
       - "9200:9200"
-    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection kafka:9092 --destinationUri  https://elasticsearch:9200  --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
+      - "19200:19200"
+    environment:
+      - http.port=19200
+    # Run processes for elasticsearch and capture proxy, and exit if either one ends
+    command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection kafka:9092 --destinationUri  https://localhost:19200  --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml & wait -n 1'
     depends_on:
       - kafka
-      - elasticsearch
 
-  elasticsearch:
-    image: 'migrations/elasticsearch_searchguard:latest'
-    networks:
-      - migrations
-    ports:
-      - '19200:9200'
+# Run separate instances of Capture Proxy and Elasticsearch
+#  capture-proxy:
+#    image: 'migrations/capture_proxy:latest'
+#    networks:
+#      - migrations
+#    ports:
+#      - "9200:9200"
+#    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection kafka:9092 --destinationUri  https://elasticsearch:9200  --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
+#    depends_on:
+#      - kafka
+#      - elasticsearch
+#
+#  elasticsearch:
+#    image: 'migrations/elasticsearch_searchguard:latest'
+#    networks:
+#      - migrations
+#    ports:
+#      - '19200:9200'
 
   zookeeper:
     image: docker.io/bitnami/zookeeper:3.8
@@ -52,6 +69,10 @@ services:
     image: 'migrations/traffic_replayer:latest'
     networks:
       - migrations
+    volumes:
+      - sharedReplayerOutput:/shared-replayer-output
+    environment:
+      - TUPLE_DIR_PATH=/shared-replayer-output/traffic-replayer-default
     depends_on:
       kafka:
         condition: service_started
@@ -83,7 +104,7 @@ services:
       - sharedComparatorSqlResults:/shared
     command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db"
 
-  jupyter_notebook:
+  jupyter-notebook:
     image: 'migrations/jupyter_notebook:latest'
     networks:
       - migrations
@@ -97,12 +118,12 @@ services:
       - COMPARISONS_DB_LOCATION=/shared/comparisons.db
     command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root'
 
-  opensearch-benchmarks:
-    image: 'migrations/open_search_benchmark:latest'
+  migration-console:
+    image: 'migrations/migration_console:latest'
     networks:
       - migrations
-    depends_on:
-      - captureproxy
+    volumes:
+      - sharedReplayerOutput:/shared-replayer-output
 
 volumes:
   zookeeper_data:
@@ -111,6 +132,8 @@ volumes:
     driver: local
   sharedComparatorSqlResults:
     driver: local
+  sharedReplayerOutput:
+    driver: local
 
 networks:
   migrations:
diff --git a/TrafficCapture/dockerSolution/src/main/docker/elasticsearchWithSearchGuard/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/elasticsearchWithSearchGuard/Dockerfile
index e31c51864..06a8b5f80 100644
--- a/TrafficCapture/dockerSolution/src/main/docker/elasticsearchWithSearchGuard/Dockerfile
+++ b/TrafficCapture/dockerSolution/src/main/docker/elasticsearchWithSearchGuard/Dockerfile
@@ -16,7 +16,7 @@ RUN sed 's/searchguard/plugins.security/g' $ELASTIC_SEARCH_CONFIG_FILE |  \
 # The following two commands are more convenient for development purposes,
 # but maybe not for a demo to show individual steps
 RUN /root/enableTlsConfig.sh $ELASTIC_SEARCH_CONFIG_FILE
-# Do this to disable HTTP auth
+# Alter this config line to either enable(searchguard.disabled: false) or disable(searchguard.disabled: true) HTTP auth
 RUN echo "searchguard.disabled: false" >> $ELASTIC_SEARCH_CONFIG_FILE
 
 #CMD tail -f /dev/null
diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile
new file mode 100644
index 000000000..1c205efe7
--- /dev/null
+++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile
@@ -0,0 +1,17 @@
+FROM ubuntu:jammy
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev gcc libc-dev git curl vim && \
+    pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm
+
+COPY runTestBenchmarks.sh /root/
+COPY humanReadableLogs.py /root/
+COPY catIndices.sh /root/
+RUN chmod ug+x /root/runTestBenchmarks.sh
+RUN chmod ug+x /root/humanReadableLogs.py
+RUN chmod ug+x /root/catIndices.sh
+WORKDIR /root
+
+CMD tail -f /dev/null
\ No newline at end of file
diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh
new file mode 100644
index 000000000..21189e476
--- /dev/null
+++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Default values
+source_endpoint="https://capture-proxy-es:9200"
+source_auth_user_and_pass="admin:admin"
+source_no_auth=false
+target_no_auth=false
+
+# Check for the presence of COPILOT_SERVICE_NAME environment variable
+if [ -n "$COPILOT_SERVICE_NAME" ]; then
+    target_endpoint="https://${MIGRATION_DOMAIN_ENDPOINT}:443"
+    target_auth_user_and_pass="admin:Admin123!"
+else
+    target_endpoint="https://opensearchtarget:9200"
+    target_auth_user_and_pass="admin:admin"
+fi
+
+# Override default values with optional command-line arguments
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --target_endpoint)
+            target_endpoint="$2"
+            shift
+            shift
+            ;;
+        --target_auth_user_and_pass)
+            target_auth_user_and_pass="$2"
+            shift
+            shift
+            ;;
+        --target_no_auth)
+            target_no_auth=true
+            shift
+            ;;
+        --source_endpoint)
+            source_endpoint="$2"
+            shift
+            shift
+            ;;
+        --source_auth_user_and_pass)
+            source_auth_user_and_pass="$2"
+            shift
+            shift
+            ;;
+        --source_no_auth)
+            source_no_auth=true
+            shift
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+source_auth_string="-u $source_auth_user_and_pass"
+target_auth_string="-u $target_auth_user_and_pass"
+
+if [ "$source_no_auth" = true ]; then
+    source_auth_string=""
+fi
+if [ "$target_no_auth" = true ]; then
+    target_auth_string=""
+fi
+
+echo "SOURCE CLUSTER"
+echo "curl $source_endpoint/_cat/indices?v"
+curl $source_endpoint/_cat/indices?v --insecure $source_auth_string
+echo ""
+echo "TARGET CLUSTER"
+echo "curl $target_endpoint/_cat/indices?v"
+curl $target_endpoint/_cat/indices?v --insecure $target_auth_string
+echo ""
\ No newline at end of file
diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py
new file mode 100755
index 000000000..c6427d018
--- /dev/null
+++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+
+import argparse
+import base64
+import gzip
+import json
+import pathlib
+from typing import Optional
+import logging
+
+from tqdm import tqdm
+from tqdm.contrib.logging import logging_redirect_tqdm
+
+logger = logging.getLogger(__name__)
+
+LOG_JSON_TUPLE_FIELD = "message"
+BASE64_ENCODED_TUPLE_PATHS = ["sourceRequest.body", "targetRequest.body", "sourceResponse.body", "targetResponse.body"]
+# TODO: I'm not positive about the capitalization of the Content-Encoding and Content-Type headers.
+# This version worked on my test cases, but not guaranteed to work in all cases.
+CONTENT_ENCODING_PATH = {
+    BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[3]: "targetResponse.Content-Encoding"
+}
+CONTENT_TYPE_PATH = {
+    BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Type",
+    BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Type",
+    BASE64_ENCODED_TUPLE_PATHS[3]: "targetResponse.Content-Type"
+}
+TRANSFER_ENCODING_PATH = {
+    BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Transfer-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Transfer-Encoding",
+    BASE64_ENCODED_TUPLE_PATHS[3]: "targetResponse.Transfer-Encoding"
+}
+
+CONTENT_TYPE_JSON = "application/json"
+CONTENT_ENCODING_GZIP = "gzip"
+TRANSFER_ENCODING_CHUNKED = "chunked"
+URI_PATH = "sourceRequest.Request-URI"
+BULK_URI_PATH = "_bulk"
+
+
+class DictionaryPathException(Exception):
+    pass
+
+
+def get_element(element: str, dict_: dict, raise_on_error=False, try_lowercase_keys=False) -> Optional[any]:
+    """This has a limited version of case-insensitivity. It specifically only checks the provided key
+    and an all lower-case version of the key (if `try_lowercase_keys` is True)."""
+    keys = element.split('.')
+    rv = dict_
+    for key in keys:
+        try:
+            if key in rv:
+                rv = rv[key]
+                continue
+            if try_lowercase_keys and key.lower() in rv:
+                rv = rv[key.lower()]
+        except KeyError:
+            if raise_on_error:
+                raise DictionaryPathException(f"Key {key} was not present.")
+            else:
+                return None
+    return rv
+
+
+def set_element(element: str, dict_: dict, value: any) -> None:
+    keys = element.split('.')
+    rv = dict_
+    for key in keys[:-1]:
+        rv = rv[key]
+    rv[keys[-1]] = value
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("infile", type=pathlib.Path, help="Path to input logged tuple file.")
+    parser.add_argument("--outfile", type=pathlib.Path, help="Path for output human readable tuple file.")
+    return parser.parse_args()
+
+
+def decode_chunked(data: bytes) -> bytes:
+    newdata = []
+    next_newline = data.index(b'\r\n')
+    chunk = data[next_newline + 2:]
+    while len(chunk) > 7:  # the final EOM chunk is 7 bytes
+        next_newline = chunk.index(b'\r\n')
+        newdata.append(chunk[:next_newline])
+        chunk = chunk[next_newline + 2:]
+    return b''.join(newdata)
+
+
+def parse_body_value(raw_value: str, content_encoding: Optional[str],
+                     content_type: Optional[str], is_bulk: bool, is_chunked_transfer: bool, line_no: int):
+    # Body is base64 decoded
+    try:
+        b64decoded = base64.b64decode(raw_value)
+    except Exception as e:
+        logger.error(f"Body value on line {line_no} could not be decoded: {e}. Skipping parsing body value.")
+        return None
+
+    # Decoded data is un-chunked, if applicable
+    if is_chunked_transfer:
+        contiguous_data = decode_chunked(b64decoded)
+    else:
+        contiguous_data = b64decoded
+
+    # Data is un-gzipped, if applicable
+    is_gzipped = content_encoding is not None and content_encoding == CONTENT_ENCODING_GZIP
+    if is_gzipped:
+        try:
+            unzipped = gzip.decompress(contiguous_data)
+        except Exception as e:
+            logger.error(f"Body value on line {line_no} should be gzipped but could not be unzipped: {e}. "
+                         "Skipping parsing body value.")
+            return contiguous_data
+    else:
+        unzipped = contiguous_data
+
+    # Data is decoded to utf-8 string
+    try:
+        decoded = unzipped.decode("utf-8")
+    except Exception as e:
+        logger.error(f"Body value on line {line_no} could not be decoded to utf-8: {e}. "
+                     "Skipping parsing body value.")
+        return unzipped
+
+    # Data is parsed as json, if applicable
+    is_json = content_type is not None and CONTENT_TYPE_JSON in content_type
+    if is_json and len(decoded) > 0:
+        # Data is parsed as a bulk json, if applicable
+        if is_bulk:
+            try:
+                return [json.loads(line) for line in decoded.splitlines()]
+            except Exception as e:
+                logger.error("Body value on line {line_no} should be a bulk json (list of json lines) but "
+                             f"could not be parsed: {e}. Skipping parsing body value.")
+                return decoded
+        try:
+            return json.loads(decoded)
+        except Exception as e:
+            logger.error(f"Body value on line {line_no} should be a json but could not be parsed: {e}. "
+                         "Skipping parsing body value.")
+            return decoded
+    return decoded
+
+
+def parse_tuple(line: str, line_no: int) -> dict:
+    item = json.loads(line)
+    message = item[LOG_JSON_TUPLE_FIELD]
+    tuple = json.loads(message)
+    try:
+        is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, tuple, raise_on_error=True)
+    except DictionaryPathException as e:
+        logger.error(f"`{URI_PATH}` on line {line_no} could not be loaded: {e} "
+                     f"Skipping parsing tuple.")
+        return tuple
+    for body_path in BASE64_ENCODED_TUPLE_PATHS:
+        base64value = get_element(body_path, tuple)
+        if base64value is None:
+            # This component has no body element, which is potentially valid.
+            continue
+        content_encoding = get_element(CONTENT_ENCODING_PATH[body_path], tuple, try_lowercase_keys=True)
+        content_type = get_element(CONTENT_TYPE_PATH[body_path], tuple, try_lowercase_keys=True)
+        is_chunked_transfer = get_element(TRANSFER_ENCODING_PATH[body_path],
+                                          tuple, try_lowercase_keys=True) == TRANSFER_ENCODING_CHUNKED
+        value = parse_body_value(base64value, content_encoding, content_type, is_bulk_path,
+                                 is_chunked_transfer, line_no)
+        if value and type(value) is not bytes:
+            set_element(body_path, tuple, value)
+    return tuple
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.outfile:
+        outfile = args.outfile
+    else:
+        outfile = args.infile.parent / f"readable-{args.infile.name}"
+    print(f"Input file: {args.infile}; Output file: {outfile}")
+
+    logging.basicConfig(level=logging.INFO)
+    with logging_redirect_tqdm():
+        with open(args.infile, 'r') as in_f:
+            with open(outfile, 'w') as out_f:
+                for i, line in tqdm(enumerate(in_f)):
+                    print(json.dumps(parse_tuple(line, i + 1)), file=out_f)
diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh
new file mode 100644
index 000000000..37581919b
--- /dev/null
+++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Default values
+endpoint="https://capture-proxy-es:9200"
+auth_user="admin"
+auth_pass="admin"
+no_auth=false
+
+# Override default values with optional command-line arguments
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --endpoint)
+            endpoint="$2"
+            shift
+            shift
+            ;;
+        --auth_user)
+            auth_user="$2"
+            shift
+            shift
+            ;;
+        --auth_pass)
+            auth_pass="$2"
+            shift
+            shift
+            ;;
+        --no-auth)
+            no_auth=true
+            shift
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+# Populate auth string
+if [ "$no_auth" = true ]; then
+    auth_string=""
+else
+    auth_string=",basic_auth_user:${auth_user},basic_auth_password:${auth_pass}"
+fi
+
+# Construct the final client options string
+base_options_string="use_ssl:true,verify_certs:false"
+client_options="${base_options_string}${auth_string}"
+
+echo "Running opensearch-benchmark workloads against ${endpoint}"
+echo "Running opensearch-benchmark w/ 'geonames' workload..." &&
+opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options=$client_options &&
+echo "Running opensearch-benchmark w/ 'http_logs' workload..." &&
+opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=http_logs --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options &&
+echo "Running opensearch-benchmark w/ 'nested' workload..." &&
+opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=nested --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options=$client_options &&
+echo "Running opensearch-benchmark w/ 'nyc_taxis' workload..." &&
+opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options=$client_options
\ No newline at end of file
diff --git a/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/Dockerfile
deleted file mode 100644
index aa2ef84e8..000000000
--- a/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/Dockerfile
+++ /dev/null
@@ -1,13 +0,0 @@
-FROM ubuntu:focal
-
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev gcc libc-dev git curl && \
-    pip3 install opensearch-benchmark
-
-COPY runTestBenchmarks.sh /root/
-RUN chmod ugo+x /root/runTestBenchmarks.sh
-WORKDIR /root
-
-CMD tail -f /dev/null
\ No newline at end of file
diff --git a/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/runTestBenchmarks.sh b/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/runTestBenchmarks.sh
deleted file mode 100644
index 5f3396744..000000000
--- a/TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/runTestBenchmarks.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-
-echo "Running opensearch-benchmark w/ 'geonames' workload..." &&
-opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://captureproxy:9200 --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" &&
-echo "Running opensearch-benchmark w/ 'http_logs' workload..." &&
-opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://captureproxy:9200 --workload=http_logs --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" &&
-echo "Running opensearch-benchmark w/ 'nested' workload..." &&
-opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://captureproxy:9200 --workload=nested --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" &&
-echo "Running opensearch-benchmark w/ 'nyc_taxis' workload..." &&
-opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://captureproxy:9200 --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin"
diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java
index 372505c3a..ed360db17 100644
--- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java
+++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java
@@ -1,7 +1,6 @@
 package org.opensearch.migrations.trafficcapture.netty;
 
 import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.DefaultHttpRequest;
 import io.netty.handler.codec.http.HttpRequest;
 import lombok.extern.slf4j.Slf4j;
 import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer;
@@ -23,14 +22,15 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob
         if (shouldBlockPredicate.test(httpRequest)) {
             trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> {
                 if (t != null) {
+                    // This is a spot where we would benefit from having a behavioral policy that different users
+                    // could set as needed. Some users may be fine with just logging a failed offloading of a request
+                    // where other users may want to stop entirely. JIRA here: https://opensearch.atlassian.net/browse/MIGRATIONS-1276
                     log.warn("Got error: " + t.getMessage());
-                    ctx.close();
-                } else {
-                    try {
-                        super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest);
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
+                }
+                try {
+                    super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
                 }
             });
         } else {
diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java
index 3f60e6f98..972928c58 100644
--- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java
+++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java
@@ -39,7 +39,8 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumer<Embe
         AtomicInteger flushCount = new AtomicInteger();
         var offloader = new StreamChannelConnectionCaptureSerializer("Test", "connection",
                 () -> CodedOutputStream.newInstance(scratchBytes),
-                cos -> {
+                captureSerializerResult -> {
+                    CodedOutputStream cos = captureSerializerResult.getCodedOutputStream();
                     outputByteBuffer.set(ByteBuffer.wrap(scratchBytes, 0, cos.getTotalBytesWritten()));
                     try {
                         cos.flush();
diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle
index 645a13750..9ac7eb908 100644
--- a/TrafficCapture/testUtilities/build.gradle
+++ b/TrafficCapture/testUtilities/build.gradle
@@ -1,3 +1,5 @@
+import org.opensearch.migrations.common.CommonUtils
+
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -29,7 +31,7 @@ spotbugs {
 }
 
 checkstyle {
-    toolVersion = '10.9.3'
+    toolVersion = '10.12.3'
     configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml')
     System.setProperty('checkstyle.cache.file', String.format('%s/%s',
             buildDir, 'checkstyle.cachefile'))
@@ -40,12 +42,14 @@ repositories {
 }
 
 dependencies {
+    spotbugs 'com.github.spotbugs:spotbugs:4.7.3'
+
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0'
     implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre'
     implementation group: 'io.netty', name: 'netty-all', version: '4.1.89.Final'
     implementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1'
-    implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.68'
-    implementation group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version: '1.68'
+    implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.74'
+    implementation group: 'org.bouncycastle', name: 'bcpkix-jdk18on', version: '1.74'
     implementation group: 'org.projectlombok', name: 'lombok', version: '1.18.22'
     implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7'
 
@@ -54,6 +58,17 @@ dependencies {
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0'
 }
 
+configurations.all {
+    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
+        if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') {
+            def targetVersion = '6.7.0'
+            if (CommonUtils.wasRequestedVersionReleasedBeforeTargetVersion(details.requested.version, targetVersion)) {
+                details.useVersion targetVersion
+            }
+        }
+    }
+}
+
 tasks.named('test') {
     useJUnitPlatform()
 }
diff --git a/TrafficCapture/testUtilities/src/main/java/org/opensearch/migrations/testutils/SimpleHttpClientForTesting.java b/TrafficCapture/testUtilities/src/main/java/org/opensearch/migrations/testutils/SimpleHttpClientForTesting.java
index 9f9c809b3..0af722b93 100644
--- a/TrafficCapture/testUtilities/src/main/java/org/opensearch/migrations/testutils/SimpleHttpClientForTesting.java
+++ b/TrafficCapture/testUtilities/src/main/java/org/opensearch/migrations/testutils/SimpleHttpClientForTesting.java
@@ -1,6 +1,10 @@
 package org.opensearch.migrations.testutils;
 
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import lombok.AllArgsConstructor;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPut;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.impl.classic.HttpClients;
 import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager;
@@ -9,8 +13,13 @@
 import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
 import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
 import org.apache.hc.client5.http.ssl.TrustAllStrategy;
+import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.config.RegistryBuilder;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.InputStreamEntity;
+import org.apache.hc.core5.http.io.entity.StringEntity;
 import org.apache.hc.core5.ssl.SSLContexts;
+import java.nio.charset.Charset;
 
 import java.io.IOException;
 import java.net.URI;
@@ -58,6 +67,13 @@ private SimpleHttpClientForTesting(BasicHttpClientConnectionManager connectionMa
         httpClient = HttpClients.custom().setConnectionManager(connectionManager).build();
     }
 
+    @AllArgsConstructor
+    public static class PayloadAndContentType{
+        public final InputStream contents;
+        public final String contentType;
+        //public final Charset charset;
+    }
+
     public SimpleHttpResponse makeGetRequest(URI endpoint, Stream<Map.Entry<String,String>> requestHeaders)
             throws IOException {
         var request = new HttpGet(endpoint);
@@ -69,8 +85,25 @@ public SimpleHttpResponse makeGetRequest(URI endpoint, Stream<Map.Entry<String,S
                 responseBodyBytes, response.getReasonPhrase(), response.getCode());
     }
 
+    public SimpleHttpResponse makePutRequest(URI endpoint, Stream<Map.Entry<String,String>> requestHeaders, PayloadAndContentType payloadAndContentType)
+            throws IOException {
+        var request = new HttpPut(endpoint);
+        if (payloadAndContentType != null) {
+            request.setEntity(new InputStreamEntity(payloadAndContentType.contents,
+                    ContentType.create(payloadAndContentType.contentType)));
+        }
+        requestHeaders.forEach(kvp->request.setHeader(kvp.getKey(), kvp.getValue()));
+        var response = httpClient.execute(request);
+        var responseBodyBytes = response.getEntity().getContent().readAllBytes();
+        return new SimpleHttpResponse(
+                Arrays.stream(response.getHeaders()).collect(Collectors.toMap(h->h.getName(), h->h.getValue())),
+                responseBodyBytes, response.getReasonPhrase(), response.getCode());
+    }
+
     @Override
     public void close() throws IOException {
         httpClient.close();
     }
 }
+
+
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/Main.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/Main.java
index 74e84d309..b99da13f9 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/Main.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/Main.java
@@ -10,13 +10,17 @@
 import lombok.NonNull;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.CommonClientConfigs;
 import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.SaslConfigs;
 import org.apache.logging.log4j.core.util.NullOutputStream;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.migrations.trafficcapture.FileConnectionCaptureFactory;
 import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory;
 import org.opensearch.migrations.trafficcapture.StreamChannelConnectionCaptureSerializer;
 import org.opensearch.migrations.trafficcapture.kafkaoffloader.KafkaCaptureFactory;
+import org.opensearch.migrations.trafficcapture.proxyserver.netty.BacksideConnectionPool;
 import org.opensearch.migrations.trafficcapture.proxyserver.netty.NettyScanningHttpProxy;
 import org.opensearch.security.ssl.DefaultSecurityKeyStore;
 import org.opensearch.security.ssl.util.SSLConfigConstants;
@@ -28,6 +32,7 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.UUID;
@@ -39,6 +44,7 @@
 public class Main {
 
     private final static String HTTPS_CONFIG_PREFIX = "plugins.security.ssl.http.";
+    public final static String DEFAULT_KAFKA_CLIENT_ID = "HttpCaptureProxyProducer";
 
     static class Parameters {
         @Parameter(required = false,
@@ -54,13 +60,13 @@ static class Parameters {
         @Parameter(required = false,
                 names = {"--kafkaConfigFile"},
                 arity = 1,
-                description = "Kafka properties file")
+                description = "Kafka properties file for additional client customization.")
         String kafkaPropertiesFile;
         @Parameter(required = false,
                 names = {"--kafkaClientId"},
                 arity = 1,
                 description = "clientId to use for interfacing with Kafka.")
-        String kafkaClientId = "KafkaLoggingProducer";
+        String kafkaClientId = DEFAULT_KAFKA_CLIENT_ID;
         @Parameter(required = false,
                 names = {"--kafkaConnection"},
                 arity = 1,
@@ -96,6 +102,25 @@ static class Parameters {
                 arity = 1,
                 description = "Exposed port for clients to connect to this proxy.")
         int frontsidePort = 0;
+        @Parameter(required = false,
+                names = {"--numThreads"},
+                arity = 1,
+                description = "How many threads netty should create in its event loop group")
+        int numThreads = 1;
+        @Parameter(required = false,
+        names = {"--destinationConnectionPoolSize"},
+        arity = 1,
+        description = "Number of socket connections that should be maintained to the destination server " +
+                "to reduce the perceived latency to clients.  Each thread will have its own cache, so the " +
+                "total number of outstanding warm connections will be multiplied by numThreads.")
+        int destinationConnectionPoolSize = 0;
+        @Parameter(required = false,
+                names = {"--destinationConnectionPoolTimeout"},
+                arity = 1,
+                description = "Of the socket connections maintained by the destination connection pool, " +
+                        "how long after connection should the be recycled " +
+                        "(closed with a new connection taking its place)")
+        String destinationConnectionPoolTimeout = "PT30S";
     }
 
     public static Parameters parseArgs(String[] args) {
@@ -103,11 +128,11 @@ public static Parameters parseArgs(String[] args) {
         JCommander jCommander = new JCommander(p);
         try {
             jCommander.parse(args);
-            // Exactly one these 4 options are required.  See that exactly one is set by summing up their presence
-            if (Stream.of(p.traceDirectory, p.kafkaPropertiesFile, p.kafkaConnection, (p.noCapture?"":null))
+            // Exactly one these 3 options are required.  See that exactly one is set by summing up their presence
+            if (Stream.of(p.traceDirectory, p.kafkaConnection, (p.noCapture?"":null))
                     .mapToInt(s->s!=null?1:0).sum() != 1) {
-                throw new ParameterException("Expected exactly one of '--traceDirectory', '--kafkaConfigFile', " +
-                        "'--kafkaConnection', or '--noCapture' to be set");
+                throw new ParameterException("Expected exactly one of '--traceDirectory', '--kafkaConnection', or " +
+                    "'--noCapture' to be set");
             }
             return p;
         } catch (ParameterException e) {
@@ -140,55 +165,50 @@ private static IConnectionCaptureFactory getNullConnectionCaptureFactory() {
         System.err.println("No trace log directory specified.  Logging to /dev/null");
         return connectionId -> new StreamChannelConnectionCaptureSerializer(null, connectionId, () ->
                 CodedOutputStream.newInstance(NullOutputStream.getInstance()),
-                cos -> CompletableFuture.completedFuture(null));
+                captureSerializerResult -> CompletableFuture.completedFuture(null));
+    }
+
+    private static String getNodeId() {
+        return UUID.randomUUID().toString();
     }
 
+    static Properties buildKafkaProperties(Parameters params) throws IOException {
+        var kafkaProps = new Properties();
+        kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+        kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer");
+        // Property details: https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#delivery-timeout-ms
+        kafkaProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10000);
+        kafkaProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000);
+        kafkaProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000);
 
-    private static IConnectionCaptureFactory getKafkaConnectionFactory(String nodeId,
-                                                                       String kafkaPropsPath,
-                                                                       int bufferSize)
-            throws IOException {
-        Properties producerProps = new Properties();
-        if (kafkaPropsPath != null) {
+        if (params.kafkaPropertiesFile != null) {
             try {
-                producerProps.load(new FileReader(kafkaPropsPath));
+                kafkaProps.load(new FileReader(params.kafkaPropertiesFile));
             } catch (IOException e) {
-                log.error("Unable to locate provided Kafka producer properties file path: " + kafkaPropsPath);
+                log.error("Unable to locate provided Kafka producer properties file path: " + params.kafkaPropertiesFile);
                 throw e;
             }
         }
-        return new KafkaCaptureFactory(nodeId, new KafkaProducer<>(producerProps), bufferSize);
-    }
 
-    private static String getNodeId(Parameters params) {
-        return UUID.randomUUID().toString();
+        kafkaProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, params.kafkaConnection);
+        kafkaProps.put(ProducerConfig.CLIENT_ID_CONFIG, params.kafkaClientId);
+        if (params.mskAuthEnabled) {
+            kafkaProps.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL");
+            kafkaProps.setProperty(SaslConfigs.SASL_MECHANISM, "AWS_MSK_IAM");
+            kafkaProps.setProperty(SaslConfigs.SASL_JAAS_CONFIG, "software.amazon.msk.auth.iam.IAMLoginModule required;");
+            kafkaProps.setProperty(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, "software.amazon.msk.auth.iam.IAMClientCallbackHandler");
+        }
+        return kafkaProps;
     }
 
     private static IConnectionCaptureFactory getConnectionCaptureFactory(Parameters params) throws IOException {
-        var nodeId = getNodeId(params);
+        var nodeId = getNodeId();
         // TODO - it might eventually be a requirement to do multiple types of offloading.
         // Resist the urge for now though until it comes in as a request/need.
         if (params.traceDirectory != null) {
             return new FileConnectionCaptureFactory(nodeId, params.traceDirectory, params.maximumTrafficStreamSize);
-        } else if (params.kafkaPropertiesFile != null) {
-            if (params.kafkaConnection != null || params.mskAuthEnabled) {
-                log.warn("--kafkaConnection and --enableMSKAuth options are ignored when providing a Kafka properties file (--kafkaConfigFile) ");
-            }
-            return getKafkaConnectionFactory(nodeId, params.kafkaPropertiesFile, params.maximumTrafficStreamSize);
         } else if (params.kafkaConnection != null) {
-            var kafkaProps = new Properties();
-            kafkaProps.put("bootstrap.servers", params.kafkaConnection);
-            kafkaProps.put("client.id", params.kafkaClientId);
-            kafkaProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
-            kafkaProps.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
-            if (params.mskAuthEnabled) {
-                kafkaProps.setProperty("security.protocol", "SASL_SSL");
-                kafkaProps.setProperty("sasl.mechanism", "AWS_MSK_IAM");
-                kafkaProps.setProperty("sasl.jaas.config", "software.amazon.msk.auth.iam.IAMLoginModule required;");
-                kafkaProps.setProperty("sasl.client.callback.handler.class", "software.amazon.msk.auth.iam.IAMClientCallbackHandler");
-            }
-
-            return new KafkaCaptureFactory(nodeId, new KafkaProducer<>(kafkaProps), params.maximumTrafficStreamSize);
+            return new KafkaCaptureFactory(nodeId, new KafkaProducer<>(buildKafkaProperties(params)), params.maximumTrafficStreamSize);
         } else if (params.noCapture) {
             return getNullConnectionCaptureFactory();
         } else {
@@ -244,10 +264,14 @@ public static void main(String[] args) throws InterruptedException, IOException
 
         sksOp.ifPresent(x->x.initHttpSSLConfig());
         var proxy = new NettyScanningHttpProxy(params.frontsidePort);
-
         try {
-            proxy.start(backsideUri, loadBacksideSslContext(backsideUri, params.allowInsecureConnectionsToBackside),
-                    sksOp.map(sks-> (Supplier<SSLEngine>) () -> {
+            var pooledConnectionTimeout = params.destinationConnectionPoolSize == 0 ? Duration.ZERO :
+                    Duration.parse(params.destinationConnectionPoolTimeout);
+            var backsideConnectionPool = new BacksideConnectionPool(backsideUri,
+                    loadBacksideSslContext(backsideUri, params.allowInsecureConnectionsToBackside),
+                    params.destinationConnectionPoolSize, pooledConnectionTimeout);
+            proxy.start(backsideConnectionPool, params.numThreads,
+                    sksOp.map(sks -> (Supplier<SSLEngine>) () -> {
                         try {
                             var sslEngine = sks.createHTTPSSLEngine();
                             return sslEngine;
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java
new file mode 100644
index 000000000..aeb298c05
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java
@@ -0,0 +1,110 @@
+package org.opensearch.migrations.trafficcapture.proxyserver.netty;
+
+import io.netty.channel.socket.nio.NioSocketChannel;
+import org.slf4j.event.Level;
+import io.netty.bootstrap.Bootstrap;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.DefaultChannelPromise;
+import io.netty.channel.EventLoop;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslHandler;
+import io.netty.util.concurrent.FastThreadLocal;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.net.ssl.SSLEngine;
+import java.net.URI;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+@Slf4j
+public class BacksideConnectionPool {
+    private final URI backsideUri;
+    private final SslContext backsideSslContext;
+    private final FastThreadLocal connectionCacheForEachThread;
+    private final Duration inactivityTimeout;
+    private final int poolSize;
+
+    public BacksideConnectionPool(URI backsideUri, SslContext backsideSslContext,
+                                  int poolSize, Duration inactivityTimeout) {
+        this.backsideUri = backsideUri;
+        this.backsideSslContext = backsideSslContext;
+        this.connectionCacheForEachThread = new FastThreadLocal();
+        this.inactivityTimeout = inactivityTimeout;
+        this.poolSize = poolSize;
+    }
+
+    public ChannelFuture getOutboundConnectionFuture(EventLoop eventLoop) {
+        if (poolSize == 0) {
+            return buildConnectionFuture(eventLoop);
+        }
+        return getExpiringWarmChannelPool(eventLoop).getAvailableOrNewItem();
+    }
+
+    private ExpiringSubstitutableItemPool<ChannelFuture, Void>
+    getExpiringWarmChannelPool(EventLoop eventLoop) {
+        var thisContextsConnectionCache = (ExpiringSubstitutableItemPool<ChannelFuture, Void>)
+                connectionCacheForEachThread.get();
+        if (thisContextsConnectionCache == null) {
+            thisContextsConnectionCache =
+                    new ExpiringSubstitutableItemPool<ChannelFuture, Void>(inactivityTimeout,
+                            eventLoop,
+                            () -> buildConnectionFuture(eventLoop),
+                            x->x.channel().close(), poolSize, Duration.ZERO);
+            if (log.isInfoEnabled()) {
+                logProgressAtInterval(Level.INFO, eventLoop,
+                        thisContextsConnectionCache, Duration.ofSeconds(30));
+            }
+            connectionCacheForEachThread.set(thisContextsConnectionCache);
+        }
+
+        return thisContextsConnectionCache;
+    }
+
+    private void logProgressAtInterval(Level logLevel, EventLoop eventLoop,
+                                       ExpiringSubstitutableItemPool<ChannelFuture, Void> channelPoolMap,
+                                       Duration frequency) {
+        eventLoop.schedule(() -> {
+            log.atLevel(logLevel).log(channelPoolMap.getStats().toString());
+            logProgressAtInterval(logLevel, eventLoop, channelPoolMap, frequency);
+        }, frequency.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    private ChannelFuture buildConnectionFuture(EventLoop eventLoop) {
+        // Start the connection attempt.
+        Bootstrap b = new Bootstrap();
+        b.group(eventLoop)
+                .channel(NioSocketChannel.class)
+                .handler(new ChannelDuplexHandler())
+                .option(ChannelOption.AUTO_READ, false);
+        var f = b.connect(backsideUri.getHost(), backsideUri.getPort());
+        var rval = new DefaultChannelPromise(f.channel());
+        f.addListener((ChannelFutureListener) connectFuture -> {
+            if (connectFuture.isSuccess()) {
+                // connection complete start to read first data
+                log.debug("Done setting up backend channel & it was successful (" + connectFuture.channel() + ")");
+                if (backsideSslContext != null) {
+                    var pipeline = connectFuture.channel().pipeline();
+                    SSLEngine sslEngine = backsideSslContext.newEngine(connectFuture.channel().alloc());
+                    sslEngine.setUseClientMode(true);
+                    var sslHandler =  new SslHandler(sslEngine);
+                    pipeline.addFirst("ssl", sslHandler);
+                    sslHandler.handshakeFuture().addListener(handshakeFuture -> {
+                        if (handshakeFuture.isSuccess()) {
+                            rval.setSuccess();
+                        } else {
+                            rval.setFailure(handshakeFuture.cause());
+                        }
+                    });
+                } else {
+                    rval.setSuccess();
+                }
+            } else {
+                rval.setFailure(connectFuture.cause());
+            }
+        });
+        return rval;
+    }
+}
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideHandler.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideHandler.java
index 6abe5ec09..094e30028 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideHandler.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideHandler.java
@@ -33,7 +33,7 @@ public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
 
     @Override
     public void channelInactive(ChannelHandlerContext ctx) {
-        log.debug("inactive channel - closing");
+        log.debug("inactive channel - closing (" + ctx.channel() + ")");
         FrontsideHandler.closeAndFlush(writeBackChannel);
     }
 
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java
new file mode 100644
index 000000000..b5e7ddd19
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java
@@ -0,0 +1,306 @@
+package org.opensearch.migrations.trafficcapture.proxyserver.netty;
+
+import io.netty.channel.EventLoop;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.LinkedHashSet;
+import java.util.Queue;
+import java.util.StringJoiner;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+
+/**
+ * This class maintains N items for S seconds.  After S seconds, items are expired as per the
+ * specified expiration callback.  Callers can retrieve items from the cache or built on-demand
+ * if no items are available within the cache that are ready to go.
+ *
+ * This class does not use locking.  Instead, it is assumed that one of these will be created for
+ * each netty event loop.
+ */
+@Slf4j
+public class ExpiringSubstitutableItemPool<F extends Future<U>, U> {
+
+    private static class Entry<F> {
+        Instant timestamp;
+        F future;
+        public Entry(F future) {
+            timestamp = Instant.now();
+            this.future = future;
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder sb = new StringBuilder("Entry{");
+            sb.append("timestamp=").append(timestamp);
+            sb.append(", value=").append(future);
+            sb.append('}');
+            return sb.toString();
+        }
+    }
+
+    public static class PoolClosedException extends RuntimeException { }
+
+    public static class Stats {
+        @Getter
+        private long nItemsCreated;
+        @Getter
+        private long nItemsExpired;
+        @Getter
+        private long nHotGets; // cache hits
+        @Getter
+        private long nColdGets; // cache misses
+        @Getter
+        Duration totalDurationBuildingItems = Duration.ZERO;
+        @Getter
+        Duration totalWaitTimeForCallers = Duration.ZERO;
+
+        @Override
+        public String toString() {
+            return new StringJoiner(", ", Stats.class.getSimpleName() + "[", "]")
+                    .add("nItemsCreated=" + nItemsCreated)
+                    .add("nHotGets=" + nHotGets)
+                    .add("nColdGets=" + nColdGets)
+                    .add("nExpiredItems=" + nItemsExpired)
+                    .add("avgDurationBuildingItems=" + averageBuildTime())
+                    .add("avgWaitTimeForCallers=" + averageWaitTime())
+                    .toString();
+        }
+
+        public long getTotalGets() {
+            return nHotGets+nColdGets;
+        }
+
+        public Duration averageWaitTime() {
+            if (getTotalGets() == 0) {
+                return Duration.ZERO;
+            }
+            return totalWaitTimeForCallers.dividedBy(getTotalGets());
+        }
+
+        public Duration averageBuildTime() {
+            if (totalItemsCreated() == 0) {
+                return Duration.ZERO;
+            }
+            return totalDurationBuildingItems.dividedBy(totalItemsCreated());
+        }
+
+        private void itemBuilt(Duration delta) {
+            totalDurationBuildingItems = totalDurationBuildingItems.plus(delta);
+            nItemsCreated++;
+        }
+        private  void addWaitTime(Duration delta) {
+            totalWaitTimeForCallers = totalWaitTimeForCallers.plus(delta);
+        }
+
+        private void addHotGet() {
+            nHotGets++;
+        }
+
+        private void addColdGet() {
+            nColdGets++;
+        }
+
+        private long totalItemsCreated() {
+            return nItemsCreated;
+        }
+
+        private void addExpiredItem() {
+            nItemsExpired++;
+        }
+    }
+
+    // Store in-progress futures that were the result of item builds in their "in-order"
+    // creation time so that if the readyItems is empty, we can return a future that is
+    // more likely to complete.
+    private final LinkedHashSet<F> inProgressItems;
+    private final Queue<Entry<F>> readyItems;
+    private final Supplier<F> itemSupplier;
+    private final Consumer<F> onExpirationConsumer;
+    @Getter
+    private final EventLoop eventLoop;
+    private Duration inactivityTimeout;
+    private GenericFutureListener<F> shuffleInProgressToReady;
+    @Getter
+    private Stats stats = new Stats();
+    private int poolSize;
+
+    public ExpiringSubstitutableItemPool(@NonNull Duration inactivityTimeout,
+                                         @NonNull EventLoop eventLoop,
+                                         @NonNull Supplier<F> itemSupplier,
+                                         @NonNull Consumer<F> onExpirationConsumer,
+                                         int numItemsToLoad, @NonNull Duration initialItemLoadInterval) {
+        this(inactivityTimeout, eventLoop, itemSupplier, onExpirationConsumer);
+        increaseCapacityWithSchedule(numItemsToLoad, initialItemLoadInterval);
+    }
+
+    public ExpiringSubstitutableItemPool(@NonNull Duration inactivityTimeout,
+                                         @NonNull EventLoop eventLoop,
+                                         @NonNull Supplier<F> itemSupplier,
+                                         @NonNull Consumer<F> onExpirationConsumer) {
+        assert inactivityTimeout.multipliedBy(-1).isNegative() : "inactivityTimeout must be > 0";
+        this.inProgressItems = new LinkedHashSet<>();
+        this.readyItems = new ArrayDeque<>();
+        this.eventLoop = eventLoop;
+        this.inactivityTimeout = inactivityTimeout;
+        this.onExpirationConsumer = onExpirationConsumer;
+        this.itemSupplier = () -> {
+            var startTime = Instant.now();
+            var rval = itemSupplier.get();
+            rval.addListener(v->{
+                stats.itemBuilt(Duration.between(startTime, Instant.now()));
+            });
+            return rval;
+        };
+        // store this as a field so that we can remove the listener once the inProgress item has been
+        // shifted to the readyItems
+        this.shuffleInProgressToReady =
+        f -> {
+            inProgressItems.remove(f);
+            if (f.isSuccess()) {
+                readyItems.add(new Entry(f));
+                scheduleNextExpirationSweep(inactivityTimeout);
+            } else {
+                // the calling context should track failures too - no reason to log
+                // TODO - add some backoff here
+                beginLoadingNewItemIfNecessary();
+            }
+        };
+    }
+
+    public int reduceCapacity(int delta) {
+        assert delta >= 0 : "expected capacity delta to be >= 0";
+        poolSize -= delta;
+        assert poolSize >= 0 : "expected pool size to remain >= 0";
+        return poolSize;
+    }
+
+    public int increaseCapacity(int itemsToLoad) {
+        return increaseCapacityWithSchedule(itemsToLoad, Duration.ZERO);
+    }
+
+    public int increaseCapacityWithSchedule(int itemsToLoad, Duration gapBetweenLoads) {
+        poolSize += itemsToLoad;
+        scheduleItemLoadsRecurse(itemsToLoad, gapBetweenLoads);
+        return poolSize;
+    }
+
+    public F getAvailableOrNewItem() {
+        if (inactivityTimeout.isZero()) {
+            throw new PoolClosedException();
+        }
+        var startTime = Instant.now();
+        {
+            log.trace("getAvailableOrNewItem: readyItems.size()="+readyItems.size());
+            var item = readyItems.poll();
+            log.trace("getAvailableOrNewItem: item="+item + " remaining readyItems.size()="+readyItems.size());
+            if (item != null) {
+                stats.addHotGet();
+                beginLoadingNewItemIfNecessary();
+                stats.addWaitTime(Duration.between(startTime, Instant.now()));
+                return item.future;
+            }
+        }
+
+        BiFunction<F,String,F> durationTrackingDecoratedItem =
+                (itemsFuture, label) -> (F) itemsFuture.addListener(f->{
+                    stats.addWaitTime(Duration.between(startTime, Instant.now()));
+                    log.trace(label + "returning value="+ f.get()+" from future " + itemsFuture);
+                });
+        stats.addColdGet();
+        var inProgressIt = inProgressItems.iterator();
+
+        if (inProgressIt.hasNext()) {
+            var firstItem = inProgressIt.next();
+            inProgressIt.remove();
+            firstItem.removeListeners(shuffleInProgressToReady);
+            beginLoadingNewItemIfNecessary();
+            return durationTrackingDecoratedItem.apply(firstItem, "IN_PROGRESS: ");
+        }
+        return durationTrackingDecoratedItem.apply(itemSupplier.get(), "FRESH: ");
+    }
+
+    public void close() {
+        inactivityTimeout = Duration.ZERO;
+        expireItems();
+    }
+
+    private void scheduleItemLoadsRecurse(int itemsToLoad, Duration gapBetweenLoads) {
+        eventLoop.schedule(()-> {
+            beginLoadingNewItemIfNecessary();
+            if (itemsToLoad >= 0) {
+                scheduleItemLoadsRecurse(itemsToLoad-1, gapBetweenLoads);
+            }
+        }, gapBetweenLoads.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    private void scheduleNextExpirationSweep(Duration d) {
+        eventLoop.schedule(this::expireItems, d.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    private void expireItems() {
+        var thresholdTimestamp = Instant.now().minus(this.inactivityTimeout);
+        log.debug("expiration threshold = " + thresholdTimestamp);
+        while (!readyItems.isEmpty()) {
+            var oldestItem = readyItems.peek();
+            var gap = Duration.between(thresholdTimestamp, oldestItem.timestamp);
+            if (!gap.isNegative()) {
+                log.debug("scheduling next sweep for " + gap);
+                scheduleNextExpirationSweep(gap);
+                return;
+            } else {
+                stats.addExpiredItem();
+                var removedItem = readyItems.poll();
+                assert removedItem == oldestItem : "expected the set of readyItems to be ordered chronologically, " +
+                        "so with a fixed item timeout, nothing should ever be able to cut back in time.  " +
+                        "Secondly, a concurrent mutation of any sort while in this function " +
+                        "should have been impossible since we're only modifying this object through a shared eventloop";
+                log.debug("Removing " + removedItem);
+                onExpirationConsumer.accept(removedItem.future);
+                beginLoadingNewItemIfNecessary();
+            }
+        }
+    }
+
+    private void beginLoadingNewItemIfNecessary() {
+        if (inactivityTimeout.isZero()) {
+            throw new PoolClosedException();
+        } else if (poolSize > (inProgressItems.size() + readyItems.size())) {
+            var futureItem = itemSupplier.get();
+            inProgressItems.add(futureItem);
+            futureItem.addListener(shuffleInProgressToReady);
+        }
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("ExpiringSubstitutableItemPool{");
+        sb.append("poolSize=").append(poolSize);
+        if (eventLoop.inEventLoop()) {
+            // these two lines are dangerous if toString() is run from a concurrent environment
+            sb.append(", inProgressItems=").append(inProgressItems);
+            sb.append(", readyItems=").append(readyItems);
+        } else {
+            sb.append(", numInProgressItems=").append(inProgressItems.size());
+            sb.append(", numReadyItems=").append(readyItems.size());
+        }
+        sb.append(", inProgressItems=").append(inProgressItems);
+        sb.append(", readyItems=").append(readyItems);
+        sb.append(", itemSupplier=").append(itemSupplier);
+        sb.append(", onExpirationConsumer=").append(onExpirationConsumer);
+        sb.append(", eventLoop=").append(eventLoop);
+        sb.append(", inactivityTimeout=").append(inactivityTimeout);
+        sb.append(", stats=").append(stats);
+        sb.append('}');
+        return sb.toString();
+    }
+}
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java
index 7344af3c2..8f2aa3030 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java
@@ -1,77 +1,53 @@
 package org.opensearch.migrations.trafficcapture.proxyserver.netty;
 
-import io.netty.bootstrap.Bootstrap;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.channel.ChannelOption;
-import io.netty.handler.ssl.SslContext;
-import io.netty.handler.ssl.SslHandler;
+import io.netty.handler.logging.LogLevel;
 import lombok.extern.slf4j.Slf4j;
 
-import javax.net.ssl.SSLEngine;
 import java.net.URI;
 
 @Slf4j
 public class FrontsideHandler extends ChannelInboundHandlerAdapter {
 
     private Channel outboundChannel;
+    private BacksideConnectionPool backsideConnectionPool;
 
-    private final URI backsideUri;
-    private final SslContext backsideSslContext;
 
     /**
-     * Create a handler that sets the autoreleases flag
-     * @param backsideUri
-     * @param backsideSslContext
+     * Create a handler that sets the autorelease flag
      */
-    public FrontsideHandler(URI backsideUri, SslContext backsideSslContext) {
-        this.backsideUri = backsideUri;
-        this.backsideSslContext = backsideSslContext;
+    public FrontsideHandler(BacksideConnectionPool backsideConnectionPool) {
+        this.backsideConnectionPool = backsideConnectionPool;
     }
 
     @Override
     public void channelActive(ChannelHandlerContext ctx) {
         final Channel inboundChannel = ctx.channel();
-        // Start the connection attempt.
-        Bootstrap b = new Bootstrap();
-        b.group(inboundChannel.eventLoop())
-                .channel(ctx.channel().getClass())
-                .handler(new BacksideHandler(inboundChannel))
-                .option(ChannelOption.AUTO_READ, false);
-        log.debug("Active - setting up backend connection");
-        var f = b.connect(backsideUri.getHost(), backsideUri.getPort());
-        outboundChannel = f.channel();
-        f.addListener(new ChannelFutureListener() {
-            @Override
-            public void operationComplete(ChannelFuture future) {
-                if (future.isSuccess()) {
-                    // connection complete start to read first data
-                    log.debug("Done setting up backend channel & it was successful");
-                    if (backsideSslContext != null) {
-                        var pipeline = future.channel().pipeline();
-                        SSLEngine sslEngine = backsideSslContext.newEngine(future.channel().alloc());
-                        sslEngine.setUseClientMode(true);
-                        pipeline.addFirst("ssl", new SslHandler(sslEngine));
-                    }
-                    inboundChannel.read();
-                } else {
-                    // Close the connection if the connection attempt has failed.
-                    log.debug("closing outbound channel because CONNECT future was not successful");
-                    inboundChannel.close();
-                }
+        var outboundChannelFuture = backsideConnectionPool.getOutboundConnectionFuture(inboundChannel.eventLoop());
+        log.debug("Active - setting up backend connection with channel " + outboundChannelFuture.channel());
+        outboundChannelFuture.addListener((ChannelFutureListener) (future -> {
+            if (future.isSuccess()) {
+                var pipeline = future.channel().pipeline();
+                pipeline.addLast(new BacksideHandler(inboundChannel));
+                inboundChannel.read();
+            } else {
+                // Close the connection if the connection attempt has failed.
+                log.debug("closing outbound channel because CONNECT future was not successful");
+                inboundChannel.close();
             }
-        });
+        }));
+        outboundChannel = outboundChannelFuture.channel();
     }
 
     @Override
     public void channelRead(final ChannelHandlerContext ctx, Object msg) {
-        log.debug("frontend handler read: "+msg);
+        log.debug("frontend handler[" + this.outboundChannel + "] read: "+msg);
         if (outboundChannel.isActive()) {
-            log.debug("Writing data to backside handler");
+            log.debug("Writing data to backside handler " + outboundChannel);
             outboundChannel.writeAndFlush(msg)
                     .addListener((ChannelFutureListener) future -> {
                         if (future.isSuccess()) {
@@ -82,7 +58,10 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) {
                             future.channel().close(); // close the backside
                         }
                     });
-        } // if the outbound channel has died, so be it... let this frontside finish with it's caller naturally
+            outboundChannel.config().setAutoRead(true);
+        } else { // if the outbound channel has died, so be it... let this frontside finish with it's caller naturally
+            log.warn("Output channel (" + outboundChannel + ") is NOT active");
+        }
     }
 
     public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java
index 4f2afde0c..6a7e90133 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java
@@ -6,13 +6,11 @@
 import io.netty.channel.EventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.ssl.SslContext;
 import io.netty.util.internal.logging.InternalLoggerFactory;
 import io.netty.util.internal.logging.JdkLoggerFactory;
 import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory;
 
 import javax.net.ssl.SSLEngine;
-import java.net.URI;
 import java.util.function.Supplier;
 
 public class NettyScanningHttpProxy {
@@ -29,16 +27,17 @@ public int getProxyPort() {
         return proxyPort;
     }
 
-    public void start(URI backsideUri, SslContext backsideSslContext, Supplier<SSLEngine> sslEngineSupplier,
-        IConnectionCaptureFactory connectionCaptureFactory) throws InterruptedException {
-        InternalLoggerFactory.setDefaultFactory(JdkLoggerFactory.INSTANCE);
+    public void start(BacksideConnectionPool backsideConnectionPool,
+                      int numThreads,
+                      Supplier<SSLEngine> sslEngineSupplier,
+                      IConnectionCaptureFactory connectionCaptureFactory) throws InterruptedException {
         bossGroup = new NioEventLoopGroup(1);
-        workerGroup = new NioEventLoopGroup();
+        workerGroup = new NioEventLoopGroup(numThreads);
         ServerBootstrap serverBootstrap = new ServerBootstrap();
         try {
             mainChannel = serverBootstrap.group(bossGroup, workerGroup)
                     .channel(NioServerSocketChannel.class)
-                    .childHandler(new ProxyChannelInitializer(backsideUri, backsideSslContext, sslEngineSupplier,
+                    .childHandler(new ProxyChannelInitializer(backsideConnectionPool, sslEngineSupplier,
                             connectionCaptureFactory))
                     .childOption(ChannelOption.AUTO_READ, false)
                     .bind(proxyPort).sync().channel();
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java
index 2687261f2..98e207c5b 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java
@@ -2,35 +2,26 @@
 
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.http.DefaultHttpRequest;
 import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpObjectDecoder;
 import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
 import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory;
 import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler;
-import org.opensearch.migrations.trafficcapture.netty.LoggingHttpRequestHandler;
 import org.opensearch.migrations.trafficcapture.netty.LoggingHttpResponseHandler;
 
 import javax.net.ssl.SSLEngine;
 import java.io.IOException;
-import java.net.URI;
 import java.util.function.Supplier;
 
 public class ProxyChannelInitializer extends ChannelInitializer<SocketChannel> {
 
     private final IConnectionCaptureFactory connectionCaptureFactory;
     private final Supplier<SSLEngine> sslEngineProvider;
-    private final URI backsideUri;
-    private final SslContext backsideSslContext;
+    private final BacksideConnectionPool backsideConnectionPool;
 
-    public ProxyChannelInitializer(URI backsideUri, SslContext backsideSslContext, Supplier<SSLEngine> sslEngineSupplier,
+    public ProxyChannelInitializer(BacksideConnectionPool backsideConnectionPool, Supplier<SSLEngine> sslEngineSupplier,
                                    IConnectionCaptureFactory connectionCaptureFactory) {
-        this.backsideUri = backsideUri;
-        this.backsideSslContext = backsideSslContext;
+        this.backsideConnectionPool = backsideConnectionPool;
         this.sslEngineProvider = sslEngineSupplier;
         this.connectionCaptureFactory = connectionCaptureFactory;
     }
@@ -53,6 +44,6 @@ protected void initChannel(SocketChannel ch) throws IOException {
         ch.pipeline().addLast(new LoggingHttpResponseHandler(offloader));
         ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(offloader,
                 this::shouldGuaranteeMessageOffloading));
-        ch.pipeline().addLast(new FrontsideHandler(backsideUri, backsideSslContext));
+        ch.pipeline().addLast(new FrontsideHandler(backsideConnectionPool));
     }
 }
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java
index ebc62696b..93c9cdadf 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java
@@ -48,7 +48,8 @@ public IChannelConnectionCaptureSerializer createOffloader(String connectionId)
             var cos = CodedOutputStream.newInstance(bb);
             codedStreamToByteBufferMap.put(cos, bb);
             return cos;
-        }, (codedOutputStream) -> {
+        }, (captureSerializerResult) -> {
+            CodedOutputStream codedOutputStream = captureSerializerResult.getCodedOutputStream();
             CompletableFuture cf = closeHandler(codedStreamToByteBufferMap.get(codedOutputStream));
             codedStreamToByteBufferMap.remove(codedOutputStream);
             singleAggregateCfRef[0] = singleAggregateCfRef[0].isDone() ? cf : CompletableFuture.allOf(singleAggregateCfRef[0], cf);
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/MainSetupTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/MainSetupTest.java
new file mode 100644
index 000000000..ac081a5c3
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/MainSetupTest.java
@@ -0,0 +1,62 @@
+package org.opensearch.migrations.trafficcapture.proxyserver;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.SaslConfigs;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Properties;
+
+public class MainSetupTest {
+
+    public final static String kafkaBrokerString = "invalid:9092";
+
+    @Test
+    public void testBuildKafkaPropertiesBaseCase() throws IOException {
+        Main.Parameters parameters = Main.parseArgs(new String[]{"--destinationUri", "invalid:9200", "--listenPort", "80", "--kafkaConnection", kafkaBrokerString});
+        Properties props = Main.buildKafkaProperties(parameters);
+
+        Assertions.assertEquals(Main.DEFAULT_KAFKA_CLIENT_ID, props.get(ProducerConfig.CLIENT_ID_CONFIG));
+        Assertions.assertEquals(kafkaBrokerString, props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG));
+        Assertions.assertEquals("org.apache.kafka.common.serialization.StringSerializer", props.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG));
+        Assertions.assertEquals("org.apache.kafka.common.serialization.ByteArraySerializer", props.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG));
+
+    }
+
+    @Test
+    public void testBuildKafkaPropertiesWithMSKAuth() throws IOException {
+        Main.Parameters parameters = Main.parseArgs(new String[]{"--destinationUri", "invalid:9200", "--listenPort", "80", "--kafkaConnection", kafkaBrokerString, "--enableMSKAuth"});
+        Properties props = Main.buildKafkaProperties(parameters);
+
+        Assertions.assertEquals(Main.DEFAULT_KAFKA_CLIENT_ID, props.get(ProducerConfig.CLIENT_ID_CONFIG));
+        Assertions.assertEquals(kafkaBrokerString, props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG));
+        Assertions.assertEquals("org.apache.kafka.common.serialization.StringSerializer", props.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG));
+        Assertions.assertEquals("org.apache.kafka.common.serialization.ByteArraySerializer", props.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG));
+        Assertions.assertEquals("SASL_SSL", props.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG));
+        Assertions.assertEquals("AWS_MSK_IAM", props.get(SaslConfigs.SASL_MECHANISM));
+        Assertions.assertEquals("software.amazon.msk.auth.iam.IAMLoginModule required;", props.get(SaslConfigs.SASL_JAAS_CONFIG));
+        Assertions.assertEquals("software.amazon.msk.auth.iam.IAMClientCallbackHandler", props.get(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS));
+    }
+
+    @Test
+    public void testBuildKafkaPropertiesWithPropertyFile() throws IOException {
+        Main.Parameters parameters = Main.parseArgs(new String[]{"--destinationUri", "invalid:9200", "--listenPort", "80", "--kafkaConnection", kafkaBrokerString, "--enableMSKAuth", "--kafkaConfigFile", "src/test/resources/simple-kafka.properties"});
+        Properties props = Main.buildKafkaProperties(parameters);
+
+        // Default settings which property file does not provide remain intact
+        Assertions.assertEquals(Main.DEFAULT_KAFKA_CLIENT_ID, props.get(ProducerConfig.CLIENT_ID_CONFIG));
+        Assertions.assertEquals(kafkaBrokerString, props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG));
+        Assertions.assertEquals("org.apache.kafka.common.serialization.ByteArraySerializer", props.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG));
+
+        // Default settings which property file provides are overriden
+        Assertions.assertEquals("org.apache.kafka.common.serialization.ByteArraySerializer", props.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG));
+
+        // Additional settings from property file are added
+        Assertions.assertEquals("212", props.get("reconnect.backoff.max.ms"));
+
+        // Settings needed for other passed arguments (i.e. --enableMSKAuth) are ignored by property file
+        Assertions.assertEquals("SASL_SSL", props.get(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG));
+    }
+}
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java
new file mode 100644
index 000000000..e4dfe3a16
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java
@@ -0,0 +1,147 @@
+package org.opensearch.migrations.trafficcapture.proxyserver.netty;
+
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.util.concurrent.DefaultPromise;
+import io.netty.util.concurrent.Future;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+@Slf4j
+class ExpiringSubstitutableItemPoolTest {
+
+    public static final int NUM_POOLED_ITEMS = 5;
+    private static final Duration SUPPLY_WORK_TIME = Duration.ofMillis(100);
+    private static final Duration TIME_BETWEEN_INITIAL_ITEMS = Duration.ofMillis(10);
+    public static final Duration INACTIVITY_TIMEOUT = Duration.ofMillis(1000);
+    public static final int NUM_ITEMS_TO_PULL = 1;
+
+    /**
+     * This test uses concurrency primitives to govern when fully-built items can be returned.
+     * That removes timing inconsistencies for the first half or so of this test.  However,
+     * expirations aren't as easy to control.  Built items aren't added until AFTER the callback,
+     * giving the callback an opportunity to drive sequencing.  However, expirations are driven
+     * by netty's clock and internal values of the ExpiringSubstitutablePool, so it's much harder
+     * to mitigate temporal inconsistencies for the expiration-related checks.
+     *
+     * Still, there's been enormous value in finding issues with the assertions in the latter
+     * part of this test.  If there are future issues, putting more conservative duration values
+     * in place may further mitigate inconsistencies, though I haven't had any tests fail yet
+     * unless I've stopped threads within the debugger.
+     */
+    @Test
+    void get() throws Exception {
+        var firstWaveBuildCountdownLatch = new CountDownLatch(NUM_POOLED_ITEMS);
+        var expireCountdownLatch = new CountDownLatch(NUM_POOLED_ITEMS-NUM_ITEMS_TO_PULL);
+        var secondWaveBuildCountdownLatch = new CountDownLatch(NUM_POOLED_ITEMS);
+        var expirationsAreDoneFuture = new CompletableFuture<Boolean>();
+        var builtItemCursor = new AtomicInteger();
+        var expiredItems = new ArrayList<Integer>();
+        var eventLoop = new NioEventLoopGroup(1);
+        var lastCreation = new AtomicReference<Instant>();
+        var pool = new ExpiringSubstitutableItemPool<Future<Integer>,Integer>(
+                INACTIVITY_TIMEOUT, eventLoop.next(),
+                () -> {
+                    var rval = new DefaultPromise<Integer>(eventLoop.next());
+                    eventLoop.schedule(() -> {
+                                if (firstWaveBuildCountdownLatch.getCount() <= 0) {
+                                    expirationsAreDoneFuture.whenComplete((v,t)->
+                                            rval.setSuccess(getIntegerItem(builtItemCursor, lastCreation, secondWaveBuildCountdownLatch)));
+                                } else {
+                                    rval.setSuccess(getIntegerItem(builtItemCursor, lastCreation, firstWaveBuildCountdownLatch));
+                                }
+                            },
+                            SUPPLY_WORK_TIME.toMillis(), TimeUnit.MILLISECONDS);
+                    return rval;
+                },
+                item->{
+                    expireCountdownLatch.countDown();
+                    log.info("Expiring item: "+item);
+                    try {
+                        expiredItems.add(item.get());
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    } catch (ExecutionException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+        for (int i = 0; i<NUM_POOLED_ITEMS; ++i) {
+            Thread.sleep(TIME_BETWEEN_INITIAL_ITEMS.toMillis());
+            log.info("instructing builder to add item now");
+            pool.increaseCapacity(1);
+        }
+
+        log.debug("waiting for firstWaveBuildCountdownLatch");
+        log.info("Pool=" + pool);
+        firstWaveBuildCountdownLatch.await();
+        log.info("Done waiting for firstWaveBuildCountdownLatch");
+        log.info("Pool=" + pool);
+        for (int i=1; i<=NUM_ITEMS_TO_PULL; ++i) {
+            Assertions.assertEquals(i, getNextItem(pool));
+        }
+        log.trace("Awaiting the last items to expire (lastCreationTime=" + lastCreation.get() +
+                " timeout=" + INACTIVITY_TIMEOUT);
+        expireCountdownLatch.await(); // wait for the 4 other original items to expire
+        log.trace("Done waiting");
+        {
+            var nowInstant = Instant.now();
+            var limitInstant = lastCreation.get().plus(INACTIVITY_TIMEOUT);
+            log.debug("nowInstant=" + nowInstant + " limitInstant=" + limitInstant +
+                    " gap = " + Duration.between(limitInstant,nowInstant) +
+                    " limit is before now = "  + limitInstant.isBefore(nowInstant));
+            Assertions.assertTrue(limitInstant.isBefore(nowInstant));
+        }
+        var initialExpiredItemsStr = IntStream.range(NUM_ITEMS_TO_PULL+1, NUM_POOLED_ITEMS+1)
+                .mapToObj(i->Integer.toString(i))
+                .collect(Collectors.joining(","));
+        Assertions.assertEquals(initialExpiredItemsStr,
+                expiredItems.stream().limit(NUM_POOLED_ITEMS-NUM_ITEMS_TO_PULL)
+                        .map(i->i.toString()).collect(Collectors.joining(",")));
+
+        expirationsAreDoneFuture.complete(true);
+        Assertions.assertTrue(pool.getStats().getNItemsCreated() >= 5);
+        Assertions.assertEquals(0, pool.getStats().getNColdGets());
+        Assertions.assertEquals(1, pool.getStats().getNHotGets());
+        Assertions.assertTrue(pool.getStats().getNItemsExpired() >= 4);
+
+        for (int i=1; i<=NUM_POOLED_ITEMS*2; ++i) {
+            log.info("Pool=" + pool);
+            Assertions.assertEquals(NUM_POOLED_ITEMS+i, getNextItem(pool));
+        }
+
+        Assertions.assertTrue(pool.getStats().getNItemsCreated() >= 15);
+        Assertions.assertEquals(11, pool.getStats().getNHotGets()+pool.getStats().getNColdGets());
+        Assertions.assertTrue(pool.getStats().getNItemsExpired() >= 4);
+
+        Assertions.assertTrue(pool.getStats().averageBuildTime().toMillis() > 0);
+        Assertions.assertTrue(pool.getStats().averageWaitTime().toMillis() <
+                pool.getStats().averageBuildTime().toMillis());
+    }
+
+    private static Integer getNextItem(ExpiringSubstitutableItemPool<Future<Integer>,Integer> pool)
+            throws InterruptedException, ExecutionException {
+        return pool.getEventLoop().next().schedule(()->pool.getAvailableOrNewItem(),
+                0, TimeUnit.MILLISECONDS).get().get();
+    }
+
+    private static Integer getIntegerItem(AtomicInteger builtItemCursor,
+                                          AtomicReference<Instant> lastCreation,
+                                          CountDownLatch countdownLatchToUse) {
+        log.info("Building item (" +builtItemCursor.hashCode() + ") " + (builtItemCursor.get()+1));
+        countdownLatchToUse.countDown();
+        lastCreation.set(Instant.now());
+        return Integer.valueOf(builtItemCursor.incrementAndGet());
+    }
+}
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxyTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxyTest.java
index d8b8006f4..e47dd745a 100644
--- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxyTest.java
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxyTest.java
@@ -20,6 +20,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -197,7 +198,9 @@ private static String makeTestRequestViaClient(SimpleHttpClientForTesting client
             try {
                 URI testServerUri = new URI("http", null, SimpleHttpServer.LOCALHOST, underlyingPort,
                     null, null, null);
-                nshp.get().start(testServerUri,null, null, connectionCaptureFactory);
+                var connectionPool = new BacksideConnectionPool(testServerUri, null,
+                        10, Duration.ofSeconds(10));
+                nshp.get().start(connectionPool,1, null, connectionCaptureFactory);
                 System.out.println("proxy port = "+port.intValue());
             } catch (InterruptedException | URISyntaxException e) {
                 throw new RuntimeException(e);
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/resources/log4j2.properties b/TrafficCapture/trafficCaptureProxyServer/src/test/resources/log4j2.properties
new file mode 100644
index 000000000..9d01fa124
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/resources/log4j2.properties
@@ -0,0 +1,11 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = STDERR
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n
+
+rootLogger.level = trace
+rootLogger.appenderRefs = stderr
+rootLogger.appenderRef.stderr.ref = STDERR
\ No newline at end of file
diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/resources/simple-kafka.properties b/TrafficCapture/trafficCaptureProxyServer/src/test/resources/simple-kafka.properties
new file mode 100644
index 000000000..973406253
--- /dev/null
+++ b/TrafficCapture/trafficCaptureProxyServer/src/test/resources/simple-kafka.properties
@@ -0,0 +1,3 @@
+security.protocol = SASL_SSL2
+key.serializer = org.apache.kafka.common.serialization.ByteArraySerializer
+reconnect.backoff.max.ms = 212
\ No newline at end of file
diff --git a/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/docker-compose.yml b/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/docker-compose.yml
index 75c194289..ff86ec9bc 100644
--- a/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/docker-compose.yml
+++ b/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/docker-compose.yml
@@ -2,6 +2,8 @@ version: '3.7'
 services:
   webserver:
     image: 'migrations/nginx-perf-test-webserver:latest'
+    networks:
+      - testing
     ports:
       - "8080:80"
 
@@ -11,7 +13,7 @@ services:
       - testing
     ports:
       - "9201:9201"
-    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main --destinationUri  http://webserver:80 --listenPort 9201 --noCapture
+    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main --destinationUri  http://webserver:80 --listenPort 9201 --noCapture --destinationConnectionPoolSize 0 --numThreads 1
     depends_on:
       - webserver
 
@@ -19,7 +21,7 @@ services:
     image: 'migrations/jmeter:latest'
     networks:
       - testing
-    command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.JMeterLoadTest -p 9201 -s captureproxy -r HTTPS; tail -f /dev/null"
+    command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.JMeterLoadTest -p 9201 -s captureproxy -r HTTP; tail -f /dev/null"
     depends_on:
       - captureproxy
 
diff --git a/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/nginx/Dockerfile b/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/nginx/Dockerfile
index ed8c9bcae..982dfb43f 100644
--- a/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/nginx/Dockerfile
+++ b/TrafficCapture/trafficCaptureProxyServerTest/src/main/docker/nginx/Dockerfile
@@ -1,47 +1,4 @@
-ARG NGINX_VERSION=1.25.1
-FROM nginx:${NGINX_VERSION}  as build
-
-RUN apt-get update
-RUN apt-get install -y \
-            openssh-client \
-            git \
-            wget \
-            libxml2 \
-            libxslt1-dev \
-            libpcre3 \
-            libpcre3-dev \
-            zlib1g \
-            zlib1g-dev \
-            openssl \
-            libssl-dev \
-            libtool \
-            automake \
-            gcc \
-            g++ \
-            make && \
-        rm -rf /var/cache/apt
-
-RUN wget "http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" && \
-    tar -C /usr/src -xzvf nginx-${NGINX_VERSION}.tar.gz
-
-#RUN mkdir -p -m 0600 ~/.ssh && \
-#    ssh-keyscan github.com >> ~/.ssh/known_hosts
-
-WORKDIR /src/ngx_devel_kit
-#RUN --mount=type=ssh git clone git@github.com:simpl/ngx_devel_kit .
-RUN git clone https://github.com/vision5/ngx_devel_kit .
-
-WORKDIR /src/echo-nginx-module
-#RUN --mount=type=ssh git clone git@github.com:openresty/set-misc-nginx-module.git .
-RUN git clone https://github.com/openresty/echo-nginx-module.git .
-
-WORKDIR /usr/src/nginx-${NGINX_VERSION}
-RUN NGINX_ARGS=$(nginx -V 2>&1 | sed -n -e 's/^.*arguments: //p') \
-    ./configure --with-compat --with-http_ssl_module --add-dynamic-module=/src/ngx_devel_kit --add-dynamic-module=/src/echo-nginx-module ${NGINX_ARGS} && \
-    make modules && \
-    make install
-
-FROM nginx:${NGINX_VERSION}
+FROM nginx:1.25.1
 
 # Installing VIM for the sake of users who would like to exec shells in the webserver container.
 RUN apt-get update
@@ -51,10 +8,10 @@ RUN /bin/bash -c  '\
     export HTMLDIR=/usr/share/nginx/html ; \
     for i in {1..100}; do echo -n t; done > ${HTMLDIR}/100.txt && \
     for i in {1..1000}; do echo -n s; done > ${HTMLDIR}/1K.txt && \
-    for i in {1..10000}; do echo -n m; done > ${HTMLDIR}/10K.txt && \
-    for i in {1..100000}; do echo -n L; done > ${HTMLDIR}/100K.txt && \
-    for i in {1..1000000}; do echo -n X; done > ${HTMLDIR}/1M.txt && \
-    for i in {1..10000000}; do echo -n H; done > ${HTMLDIR}/10M.txt'
+    for i in {1..10}; do cat ${HTMLDIR}/1K.txt | tr s m ; done > ${HTMLDIR}/10K.txt && \
+    for i in {1..10}; do cat ${HTMLDIR}/10K.txt | tr m L; done > ${HTMLDIR}/100K.txt && \
+    for i in {1..10}; do cat ${HTMLDIR}/100K.txt | tr L X; done > ${HTMLDIR}/1M.txt && \
+    for i in {1..10}; do cat ${HTMLDIR}/1M.txt | tr X H; done > ${HTMLDIR}/10M.txt'
 
 COPY nginx.conf /etc/nginx/nginx.conf
 #COPY --from=build /usr/src/nginx-${NGINX_VERSION}/objs/ngx_http_echo_module.so /usr/src/nginx-${NGINX_VERSION}/objs/ndk_http_module.so /usr/lib/nginx/modules/
diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md
index 334c0e6d7..3b7b2ab17 100644
--- a/TrafficCapture/trafficReplayer/README.md
+++ b/TrafficCapture/trafficReplayer/README.md
@@ -83,3 +83,12 @@ this class uses [JOLT](https://github.com/bazaarvoice/jolt) to perform transform
 operations that are defined in the [resources](../trafficReplayer/src/main/resources/jolt/operations) associated with
 the package. Future work will include adding more JSON transformations and other potential JSON transformation tools
 (like [JMESPath](https://jmespath.org/)).
+
+
+## Authorization Header for Replayed Requests
+
+There is a level of precedence that will determine which or if any Auth header should be added to outgoing Replayer requests, which is listed below.
+1. If the user provides an explicit auth header option to the Replayer, such as providing a static value auth header or using a user and secret arn pair, this mechanism will be used for the auth header of outgoing requests. The options can be found as Parameters [here](src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java)
+2. If the devDeploy [script](../../deployment/copilot/devDeploy.sh) is used and its Replayer command has not been altered (as in the case of 1.) and CDK deploys a target cluster with a configured FGAC user (see `fineGrainedManagerUserName` and `fineGrainedManagerUserSecretManagerKeyARN` CDK context options [here](../../deployment/cdk/opensearch-service-migration/README.md#configuration-options)) or is running in demo mode (see `enableDemoAdmin` CDK context option), this user and secret arn pair will be provided in the Replay command for the Auth header
+3. If the user provides no auth header option and incoming captured requests have an auth header, this auth header will be reused for outgoing requests
+4. If the user provides no auth header option and incoming captured requests have no auth header, then no auth header will be used for outgoing requests
diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle
index ca2dd998a..e054a0d38 100644
--- a/TrafficCapture/trafficReplayer/build.gradle
+++ b/TrafficCapture/trafficReplayer/build.gradle
@@ -19,29 +19,37 @@ buildscript {
 plugins {
     id 'org.opensearch.migrations.java-application-conventions'
     id "com.github.spotbugs" version "4.7.3"
-//    id 'checkstyle'
+    id 'checkstyle'
     id 'org.owasp.dependencycheck' version '8.2.1'
     id "io.freefair.lombok" version "8.0.1"
 }
 
+import org.opensearch.migrations.common.CommonUtils
+
 spotbugs {
     includeFilter = new File(rootDir, 'config/spotbugs/spotbugs-include.xml')
 }
 
-//checkstyle {
-//    toolVersion = '10.9.3'
-//    configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml')
-//    System.setProperty('checkstyle.cache.file', String.format('%s/%s',
-//            buildDir, 'checkstyle.cachefile'))
-//}
+checkstyle {
+    toolVersion = '10.12.3'
+    configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml')
+    System.setProperty('checkstyle.cache.file', String.format('%s/%s',
+            buildDir, 'checkstyle.cachefile'))
+}
 
 repositories {
     mavenCentral()
 }
 
 dependencies {
+    spotbugs 'com.github.spotbugs:spotbugs:4.7.3'
+
     implementation project(':captureProtobufs')
 
+    implementation 'software.amazon.awssdk:sdk-core:2.20.102'
+    implementation 'software.amazon.awssdk:auth:2.20.102'
+    implementation group: 'software.amazon.awssdk', name: 'secretsmanager', version: '2.20.127'
+
     implementation group: 'com.beust', name: 'jcommander', version: '1.82'
     implementation group: 'com.bazaarvoice.jolt', name: 'jolt-core', version: '0.1.7'
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0'
@@ -51,15 +59,36 @@ dependencies {
     implementation group: 'org.json', name: 'json', version: '20230227'
     implementation group: 'org.projectlombok', name: 'lombok', version: '1.18.22'
 
-    implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.0'
+    implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.1'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0'
     implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7'
-    implementation group: 'software.amazon.msk', name: 'aws-msk-iam-auth', version: '1.1.7'
+    implementation group: 'software.amazon.msk', name: 'aws-msk-iam-auth', version: '1.1.9'
 
     testImplementation project(':testUtilities')
     testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1'
+    testImplementation 'org.mockito:mockito-core:4.6.1'
+    testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1'
+}
+
+
+
+configurations.all {
+    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
+        if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-text') {
+            def targetVersion = '1.10.0'
+            if (CommonUtils.wasRequestedVersionReleasedBeforeTargetVersion(details.requested.version, targetVersion)) {
+                details.useVersion targetVersion
+            }
+        }
+        if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') {
+            def targetVersion = '6.7.0'
+            if (CommonUtils.wasRequestedVersionReleasedBeforeTargetVersion(details.requested.version, targetVersion)) {
+                details.useVersion targetVersion
+            }
+        }
+    }
 }
 
 application {
@@ -72,18 +101,6 @@ jar {
     }
 }
 
-task uberJar(type: Jar) {
-    manifest {
-        attributes 'Main-Class': application.mainClass
-    }
-    from {
-        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
-    }
-    archiveBaseName = project.name + "-uber"
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-    with jar
-}
-
 tasks.named('test') {
     useJUnitPlatform()
 }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AWSAuthService.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AWSAuthService.java
new file mode 100644
index 000000000..b80c4585e
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AWSAuthService.java
@@ -0,0 +1,44 @@
+package org.opensearch.migrations.replay;
+
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+
+import java.nio.charset.Charset;
+import java.util.Base64;
+
+@Slf4j
+public class AWSAuthService implements AutoCloseable {
+
+    private final SecretsManagerClient secretsManagerClient;
+
+    public AWSAuthService(SecretsManagerClient secretsManagerClient) {
+        this.secretsManagerClient = secretsManagerClient;
+    }
+
+    public AWSAuthService() {
+        this(SecretsManagerClient.builder().build());
+    }
+
+    // SecretId here can be either the unique name of the secret or the secret ARN
+    public String getSecret(String secretId) {
+        return secretsManagerClient.getSecretValue(builder -> builder.secretId(secretId)).secretString();
+    }
+
+    /**
+     * This method synchronously returns a Basic Auth header string, with the username:password Base64 encoded
+     * @param username The plaintext username
+     * @param secretId The unique name of the secret or the secret ARN from AWS Secrets Manager. Its retrieved value
+     *                 will fill the password part of the Basic Auth header
+     * @return Basic Auth header string
+     */
+    public String getBasicAuthHeaderFromSecret(String username, String secretId) {
+        String secretValue = getSecret(secretId);
+        String authHeaderString = username + ":" + secretValue;
+        return "Basic " + Base64.getEncoder().encodeToString(authHeaderString.getBytes(Charset.defaultCharset()));
+    }
+
+    @Override
+    public void close() {
+        secretsManagerClient.close();
+    }
+}
\ No newline at end of file
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java
index 9cd6dcfda..fa712ac0b 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java
@@ -15,15 +15,12 @@ public class HttpMessageAndTimestamp {
     @Setter
     private Instant lastPacketTimestamp;
 
-    /**
-     * TODO - handle out-of-order inserts by making this a radix map
-     */
-    public final ArrayList<byte[]> packetBytes;
+    public final RawPackets packetBytes;
     ByteArrayOutputStream currentSegmentBytes;
 
     public HttpMessageAndTimestamp(Instant firstPacketTimestamp) {
         this.firstPacketTimestamp = firstPacketTimestamp;
-        this.packetBytes = new ArrayList<>();
+        this.packetBytes = new RawPackets();
     }
 
     public boolean hasInProgressSegment() {
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java
index 886da913f..31223a7d8 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java
@@ -29,7 +29,7 @@
  * 1. Capturing a particular set of Kafka traffic records to a file which can then be repeatedly passed as an input param
  * to the Replayer for testing.
  * 2. Printing records for a topic with a different group id then is used normally to compare
- * 3. Clearing records for a  topic with the same group id
+ * 3. Clearing records for a topic with the same group id
  */
 public class KafkaPrinter {
     private static final Logger log = LoggerFactory.getLogger(KafkaPrinter.class);
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java
index 3bc2aafff..f2be84437 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java
@@ -5,22 +5,27 @@
 import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer;
 import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer;
 import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.IAuthTransformerFactory;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.net.URI;
 
 public class PacketToTransformingHttpHandlerFactory implements PacketConsumerFactory<AggregatedTransformedResponse> {
     private final NioEventLoopGroup eventLoopGroup;
     private final URI serverUri;
-    private final JsonTransformer jsonTransformer;
+    private final IJsonTransformer jsonTransformer;
+    private final IAuthTransformerFactory authTransformerFactory;
     private final SslContext sslContext;
 
-    public PacketToTransformingHttpHandlerFactory(URI serverUri, JsonTransformer jsonTransformer,
+    public PacketToTransformingHttpHandlerFactory(URI serverUri,
+                                                  IJsonTransformer jsonTransformer,
+                                                  IAuthTransformerFactory authTransformerFactory,
                                                   SslContext sslContext) {
-        this.jsonTransformer = jsonTransformer;
-        this.eventLoopGroup = new NioEventLoopGroup();
         this.serverUri = serverUri;
+        this.jsonTransformer = jsonTransformer;
+        this.authTransformerFactory = authTransformerFactory;
         this.sslContext = sslContext;
+        this.eventLoopGroup = new NioEventLoopGroup();
     }
 
     NettyPacketToHttpConsumer createNettyHandler(String diagnosticLabel) {
@@ -29,7 +34,8 @@ NettyPacketToHttpConsumer createNettyHandler(String diagnosticLabel) {
 
     @Override
     public IPacketFinalizingConsumer<AggregatedTransformedResponse> create(String diagnosticLabel) {
-        return new HttpJsonTransformingConsumer(jsonTransformer, createNettyHandler(diagnosticLabel), diagnosticLabel);
+        return new HttpJsonTransformingConsumer(jsonTransformer, authTransformerFactory, createNettyHandler(diagnosticLabel),
+                diagnosticLabel);
     }
 
     public void stopGroup() {
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RawPackets.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RawPackets.java
new file mode 100644
index 000000000..035ef80d9
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RawPackets.java
@@ -0,0 +1,6 @@
+package org.opensearch.migrations.replay;
+
+import java.util.ArrayList;
+
+public class RawPackets extends ArrayList<byte[]> {
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayUtils.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayUtils.java
index 5596330f6..4e887de9a 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayUtils.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayUtils.java
@@ -1,5 +1,7 @@
 package org.opensearch.migrations.replay;
 
+import io.netty.buffer.ByteBuf;
+
 import java.io.ByteArrayInputStream;
 import java.io.SequenceInputStream;
 import java.util.Collections;
@@ -11,6 +13,15 @@ public class ReplayUtils {
     public static SequenceInputStream byteArraysToInputStream(List<byte[]> data) {
         return byteArraysToInputStream(data.stream());
     }
+
+    public static SequenceInputStream byteBufsToInputStream(Stream<ByteBuf> byteBufStream) {
+        return byteArraysToInputStream(byteBufStream.map(bb->{
+            byte[] asBytes = new byte[bb.readableBytes()];
+            bb.duplicate().readBytes(asBytes);
+            return asBytes;
+        }));
+    }
+
     public static SequenceInputStream byteArraysToInputStream(Stream<byte[]> data) {
         return new SequenceInputStream(Collections.enumeration(
                 data.map(b -> new ByteArrayInputStream(b)).collect(Collectors.toList())));
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java
new file mode 100644
index 000000000..f4c37b98d
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java
@@ -0,0 +1,138 @@
+package org.opensearch.migrations.replay;
+
+
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Clock;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import lombok.extern.slf4j.Slf4j;
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
+import org.opensearch.migrations.transform.IAuthTransformer;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.signer.Aws4Signer;
+import software.amazon.awssdk.auth.signer.internal.BaseAws4Signer;
+import software.amazon.awssdk.auth.signer.params.Aws4SignerParams;
+import software.amazon.awssdk.core.checksums.SdkChecksum;
+import software.amazon.awssdk.http.SdkHttpFullRequest;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.utils.BinaryUtils;
+
+
+@Slf4j
+public class SigV4Signer extends IAuthTransformer.StreamingFullMessageTransformer {
+    private final static HashSet<String> AUTH_HEADERS_TO_PULL_WITH_PAYLOAD;
+    private final static HashSet<String> AUTH_HEADERS_TO_PULL_NO_PAYLOAD;
+
+    public static final String AMZ_CONTENT_SHA_256 = "x-amz-content-sha256";
+
+    static {
+        AUTH_HEADERS_TO_PULL_NO_PAYLOAD = new HashSet<String>() {{
+            add("authorization");
+            add("x-amz-date");
+            add("x-amz-security-token");
+        }};
+        AUTH_HEADERS_TO_PULL_WITH_PAYLOAD = new HashSet<String>(AUTH_HEADERS_TO_PULL_NO_PAYLOAD) {{
+            add(AMZ_CONTENT_SHA_256);
+        }};
+    }
+
+    private MessageDigest messageDigest;
+    private AwsCredentialsProvider credentialsProvider;
+    private String service;
+    private String region;
+    private String protocol;
+    private Supplier<Clock> timestampSupplier; // for unit testing
+
+    public SigV4Signer(AwsCredentialsProvider credentialsProvider, String service, String region, String protocol,
+                       Supplier<Clock> timestampSupplier) {
+        this.credentialsProvider = credentialsProvider;
+        this.service = service;
+        this.region = region;
+        this.protocol = protocol;
+        this.timestampSupplier = timestampSupplier;
+    }
+
+    @Override
+    public ContextForAuthHeader transformType() {
+        return ContextForAuthHeader.HEADERS_AND_CONTENT_PAYLOAD;
+    }
+
+    @Override
+    public void consumeNextPayloadPart(ByteBuffer payloadChunk) {
+        if (payloadChunk.remaining() <= 0) {
+            return;
+        }
+        if (messageDigest == null) {
+            try {
+                this.messageDigest = MessageDigest.getInstance("SHA-256");
+            } catch (NoSuchAlgorithmException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        messageDigest.update(payloadChunk);
+    }
+
+    @Override
+    public void finalize(HttpJsonMessageWithFaultingPayload msg) {
+        getSignatureHeadersViaSdk(msg).forEach(kvp -> msg.headers().put(kvp.getKey(), kvp.getValue()));
+    }
+
+    private static class AwsSignerWithPrecomputedContentHash extends BaseAws4Signer {
+        public AwsSignerWithPrecomputedContentHash() {}
+
+        protected String calculateContentHash(SdkHttpFullRequest.Builder mutableRequest,
+                                              Aws4SignerParams signerParams,
+                                              SdkChecksum contentFlexibleChecksum) {
+            var contentChecksum = mutableRequest.headers().get(AMZ_CONTENT_SHA_256);
+            return contentChecksum != null ? contentChecksum.get(0) :
+                    super.calculateContentHash(mutableRequest, signerParams, contentFlexibleChecksum);
+        }
+    }
+
+    public Stream<Map.Entry<String, List<String>>> getSignatureHeadersViaSdk(IHttpMessage msg) {
+        var signer = new AwsSignerWithPrecomputedContentHash();
+        var httpRequestBuilder = SdkHttpFullRequest.builder();
+        httpRequestBuilder.method(SdkHttpMethod.fromValue(msg.method()))
+                .uri(URI.create(msg.path()))
+                .protocol(protocol)
+                .host(msg.getFirstHeader("host"));
+
+        var contentType = msg.getFirstHeader(IHttpMessage.CONTENT_TYPE);
+        if (contentType != null) {
+            httpRequestBuilder.appendHeader("Content-Type", contentType);
+        }
+        if (messageDigest != null) {
+            byte[] bytesToEncode = messageDigest.digest();
+            String payloadHash = BinaryUtils.toHex(bytesToEncode);
+            httpRequestBuilder.appendHeader(AMZ_CONTENT_SHA_256, payloadHash);
+        }
+
+        SdkHttpFullRequest request = httpRequestBuilder.build();
+
+        var signingParamsBuilder = Aws4SignerParams.builder()
+                .signingName(service)
+                .signingRegion(Region.of(region))
+                .awsCredentials(credentialsProvider.resolveCredentials());
+        if (timestampSupplier != null) {
+            signingParamsBuilder.signingClockOverride(timestampSupplier.get());
+        }
+        var signedRequest = signer.sign(request, signingParamsBuilder.build());
+
+        var headersToReturn = messageDigest == null ?
+                AUTH_HEADERS_TO_PULL_NO_PAYLOAD : AUTH_HEADERS_TO_PULL_WITH_PAYLOAD;
+        return signedRequest.headers().entrySet().stream().filter(kvp->
+                headersToReturn.contains(kvp.getKey().toLowerCase()));
+    }
+}
+
+
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java
index bd288f15b..7baa15b2e 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java
@@ -1,6 +1,8 @@
 package org.opensearch.migrations.replay;
 
 import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.json.HTTP;
 import org.json.JSONObject;
 
@@ -17,35 +19,35 @@
 @Slf4j
 public class SourceTargetCaptureTuple {
     private RequestResponsePacketPair sourcePair;
-    private final List<byte[]> shadowRequestData;
-    private final List<byte[]> shadowResponseData;
+    private final List<byte[]> targetRequestData;
+    private final List<byte[]> targetResponseData;
     private final AggregatedTransformedResponse.HttpRequestTransformationStatus transformationStatus;
     private final Throwable errorCause;
-    Duration shadowResponseDuration;
+    Duration targetResponseDuration;
 
     public SourceTargetCaptureTuple(RequestResponsePacketPair sourcePair,
-                                    List<byte[]> shadowRequestData,
-                                    List<byte[]> shadowResponseData,
+                                    List<byte[]> targetRequestData,
+                                    List<byte[]> targetResponseData,
                                     AggregatedTransformedResponse.HttpRequestTransformationStatus transformationStatus,
                                     Throwable errorCause,
-                                    Duration shadowResponseDuration) {
+                                    Duration targetResponseDuration) {
         this.sourcePair = sourcePair;
-        this.shadowRequestData = shadowRequestData;
-        this.shadowResponseData = shadowResponseData;
+        this.targetRequestData = targetRequestData;
+        this.targetResponseData = targetResponseData;
         this.transformationStatus = transformationStatus;
         this.errorCause = errorCause;
-        this.shadowResponseDuration = shadowResponseDuration;
+        this.targetResponseDuration = targetResponseDuration;
     }
 
     public static class TupleToFileWriter {
         OutputStream outputStream;
+        Logger tupleLogger = LogManager.getLogger("OutputTupleJsonLogger");
 
         public TupleToFileWriter(OutputStream outputStream){
             this.outputStream = outputStream;
         }
 
         private JSONObject jsonFromHttpDataUnsafe(List<byte[]> data) throws IOException {
-
             SequenceInputStream collatedStream = ReplayUtils.byteArraysToInputStream(data);
             Scanner scanner = new Scanner(collatedStream, StandardCharsets.UTF_8);
             scanner.useDelimiter("\r\n\r\n");  // The headers are seperated from the body with two newlines.
@@ -85,23 +87,26 @@ private JSONObject jsonFromHttpData(List<byte[]> data, Duration latency) throws
 
         private JSONObject toJSONObject(SourceTargetCaptureTuple triple) throws IOException {
             JSONObject meta = new JSONObject();
-            meta.put("request", jsonFromHttpData(triple.sourcePair.requestData.packetBytes));
+            meta.put("sourceRequest", jsonFromHttpData(triple.sourcePair.requestData.packetBytes));
+            meta.put("targetRequest", jsonFromHttpData(triple.targetRequestData));
             //log.warn("TODO: These durations are not measuring the same values!");
-            meta.put("primaryResponse", jsonFromHttpData(triple.sourcePair.responseData.packetBytes,
+            meta.put("sourceResponse", jsonFromHttpData(triple.sourcePair.responseData.packetBytes,
                 Duration.between(triple.sourcePair.requestData.getLastPacketTimestamp(), triple.sourcePair.responseData.getLastPacketTimestamp())));
-            meta.put("shadowResponse", jsonFromHttpData(triple.shadowResponseData,
-                    triple.shadowResponseDuration));
+            meta.put("targetResponse", jsonFromHttpData(triple.targetResponseData,
+                    triple.targetResponseDuration));
+            meta.put("connectionId", triple.sourcePair.connectionId);
 
             return meta;
         }
 
         /**
-         * Writes a "triple" object to an output stream as a JSON object.
-         * The JSON triple is output on one line, and has three objects: "request", "primaryResponse",
-         * and "shadowResponse". An example of the format is below.
+         * Writes a tuple object to an output stream as a JSON object.
+         * The JSON tuple is output on one line, and has several objects: "sourceRequest", "sourceResponse",
+         * "targetRequest", and "targetResponse". The "connectionId" is also included to aid in debugging.
+         * An example of the format is below.
          * <p>
          * {
-         *   "request": {
+         *   "sourceRequest": {
          *     "Request-URI": XYZ,
          *     "Method": XYZ,
          *     "HTTP-Version": XYZ
@@ -109,7 +114,15 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple triple) throws IOExcept
          *     "header-1": XYZ,
          *     "header-2": XYZ
          *   },
-         *   "primaryResponse": {
+         *   "targetRequest": {
+         *     "Request-URI": XYZ,
+         *     "Method": XYZ,
+         *     "HTTP-Version": XYZ
+         *     "body": XYZ,
+         *     "header-1": XYZ,
+         *     "header-2": XYZ
+         *   },
+         *   "sourceResponse": {
          *     "HTTP-Version": ABC,
          *     "Status-Code": ABC,
          *     "Reason-Phrase": ABC,
@@ -117,14 +130,15 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple triple) throws IOExcept
          *     "body": ABC,
          *     "header-1": ABC
          *   },
-         *   "shadowResponse": {
+         *   "targetResponse": {
          *     "HTTP-Version": ABC,
          *     "Status-Code": ABC,
          *     "Reason-Phrase": ABC,
          *     "response_time_ms": 123,
          *     "body": ABC,
          *     "header-2": ABC
-         *   }
+         *   },
+         *   "connectionId": "0242acfffe1d0008-0000000c-00000003-0745a19f7c3c5fc9-121001ff.0"
          * }
          *
          * @param  triple  the RequestResponseResponseTriple object to be converted into json and written to the stream.
@@ -132,7 +146,7 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple triple) throws IOExcept
         public void writeJSON(SourceTargetCaptureTuple triple) throws IOException {
             JSONObject jsonObject = toJSONObject(triple);
 
-            log.trace("Writing json tuple to output stream for " + triple);
+            tupleLogger.info(jsonObject.toString());
             outputStream.write((jsonObject.toString()+"\n").getBytes(StandardCharsets.UTF_8));
             outputStream.flush();
         }
@@ -143,9 +157,9 @@ public String toString() {
         final StringBuilder sb = new StringBuilder("SourceTargetCaptureTuple{");
         sb.append("\n diagnosticLabel=").append(sourcePair.connectionId);
         sb.append("\n sourcePair=").append(sourcePair);
-        sb.append("\n shadowResponseDuration=").append(shadowResponseDuration);
-        sb.append("\n shadowRequestData=").append(Utils.packetsToStringTruncated(shadowRequestData));
-        sb.append("\n shadowResponseData=").append(Utils.packetsToStringTruncated(shadowResponseData));
+        sb.append("\n targetResponseDuration=").append(targetResponseDuration);
+        sb.append("\n targetRequestData=").append(Utils.packetsToStringTruncated(targetRequestData));
+        sb.append("\n targetResponseData=").append(Utils.packetsToStringTruncated(targetResponseData));
         sb.append("\n transformStatus=").append(transformationStatus);
         sb.append("\n errorCause=").append(errorCause==null ? "null" : errorCause.toString());
         sb.append('}');
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java
index 0e90f7324..363fc0305 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java
@@ -11,11 +11,15 @@
 import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
 import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
 import org.opensearch.migrations.trafficcapture.protos.TrafficStream;
-import org.opensearch.migrations.transform.CompositeJsonTransformer;
-import org.opensearch.migrations.transform.JoltJsonTransformer;
-import org.opensearch.migrations.transform.JsonTransformer;
-import org.opensearch.migrations.transform.TypeMappingJsonTransformer;
+import org.opensearch.migrations.transform.IAuthTransformerFactory;
+import org.opensearch.migrations.transform.JsonCompositeTransformer;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
+import org.opensearch.migrations.transform.IJsonTransformer;
+import org.opensearch.migrations.transform.JsonTypeMappingTransformer;
+import org.opensearch.migrations.transform.RemovingAuthTransformerFactory;
+import org.opensearch.migrations.transform.StaticAuthTransformerFactory;
 import org.slf4j.event.Level;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
 
 import javax.net.ssl.SSLException;
 import java.io.BufferedOutputStream;
@@ -26,6 +30,7 @@
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
@@ -36,30 +41,36 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+
 @Slf4j
 public class TrafficReplayer {
 
+    public static final String SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG = "--sigv4-auth-header-service-region";
+    public static final String AUTH_HEADER_VALUE_ARG = "--auth-header-value";
+    public static final String REMOVE_AUTH_HEADER_VALUE_ARG = "--remove-auth-header";
+    public static final String AWS_AUTH_HEADER_USER_AND_SECRET_ARG = "--auth-header-user-and-secret";
     private final PacketToTransformingHttpHandlerFactory packetHandlerFactory;
 
-    public static JsonTransformer buildDefaultJsonTransformer(String newHostName,
-                                                              String authorizationHeader) {
-        var joltJsonTransformerBuilder = JoltJsonTransformer.newBuilder()
+    public static IJsonTransformer buildDefaultJsonTransformer(String newHostName) {
+        var joltJsonTransformerBuilder = JsonJoltTransformer.newBuilder()
                 .addHostSwitchOperation(newHostName);
-        if (authorizationHeader != null) {
-            joltJsonTransformerBuilder = joltJsonTransformerBuilder.addAuthorizationOperation(authorizationHeader);
-        }
         var joltJsonTransformer = joltJsonTransformerBuilder.build();
-        return new CompositeJsonTransformer(joltJsonTransformer, new TypeMappingJsonTransformer());
+        return new JsonCompositeTransformer(joltJsonTransformer, new JsonTypeMappingTransformer());
     }
 
-    public TrafficReplayer(URI serverUri, String authorizationHeader, boolean allowInsecureConnections)
+    public TrafficReplayer(URI serverUri,
+                           IAuthTransformerFactory authTransformerFactory,
+                           boolean allowInsecureConnections)
             throws SSLException {
         this(serverUri, allowInsecureConnections,
-                buildDefaultJsonTransformer(serverUri.getHost(), authorizationHeader));
+                buildDefaultJsonTransformer(serverUri.getHost()),
+                authTransformerFactory);
     }
     
-    public TrafficReplayer(URI serverUri, boolean allowInsecureConnections,
-                           JsonTransformer jsonTransformer)
+    public TrafficReplayer(URI serverUri,
+                           boolean allowInsecureConnections,
+                           IJsonTransformer jsonTransformer,
+                           IAuthTransformerFactory authTransformer)
             throws SSLException
     {
         if (serverUri.getPort() < 0) {
@@ -71,7 +82,7 @@ public TrafficReplayer(URI serverUri, boolean allowInsecureConnections,
         if (serverUri.getScheme() == null) {
             throw new RuntimeException("Scheme (http|https) is not present for URI: "+serverUri);
         }
-        packetHandlerFactory = new PacketToTransformingHttpHandlerFactory(serverUri, jsonTransformer,
+        packetHandlerFactory = new PacketToTransformingHttpHandlerFactory(serverUri, jsonTransformer, authTransformer,
                 loadSslContext(serverUri, allowInsecureConnections));
     }
 
@@ -108,27 +119,57 @@ static class Parameters {
                 arity = 0,
                 description = "Do not check the server's certificate")
         boolean allowInsecureConnections;
+
+
+        @Parameter(required = false,
+                names = {REMOVE_AUTH_HEADER_VALUE_ARG},
+                arity = 0,
+                description = "Remove the authorization header if present and do not replace it with anything.  " +
+                        "(cannot be used with other auth arguments)")
+        boolean removeAuthHeader;
         @Parameter(required = false,
-                names = {"--auth-header-value"},
+                names = {AUTH_HEADER_VALUE_ARG},
                 arity = 1,
-                description = "Value to use for the \"authorization\" header of each request")
+                description = "Static value to use for the \"authorization\" header of each request " +
+                        "(cannot be used with other auth arguments)")
         String authHeaderValue;
         @Parameter(required = false,
-                names = {"-o", "--output"},
-                arity=1,
-                description = "output file to hold the request/response traces for the source and target cluster")
-        String outputFilename;
+                names = {AWS_AUTH_HEADER_USER_AND_SECRET_ARG},
+                arity = 2,
+                description = "<USERNAME> <SECRET_ARN> pair to specify " +
+                        "\"authorization\" header value for each request.  " +
+                        "The USERNAME specifies the plaintext user and the SECRET_ARN specifies the ARN or " +
+                        "Secret name from AWS Secrets Manager to retrieve the password from for the password section" +
+                        "(cannot be used with other auth arguments)")
+        List<String> awsAuthHeaderUserAndSecret;
+        @Parameter(required = false,
+                names = {SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG},
+                arity = 1,
+                description = "Use AWS SigV4 to sign each request with the specified service name and region.  " +
+                        "(e.g. es,us-east-1)  " +
+                        "DefaultCredentialsProvider is used to resolve credentials.  " +
+                        "(cannot be used with other auth arguments)")
+        String useSigV4ServiceAndRegion;
+
+
         @Parameter(required = false,
                 names = {"-i", "--input"},
                 arity=1,
                 description = "input file to read the request/response traces for the source cluster")
         String inputFilename;
+        @Parameter(required = false,
+                names = {"-o", "--output"},
+                arity=1,
+                description = "output file to hold the request/response traces for the source and target cluster")
+        String outputFilename;
+
         @Parameter(required = false,
                 names = {"-t", "--packet-timeout-seconds"},
                 arity = 1,
                 description = "assume that connections were terminated after this many " +
                         "seconds of inactivity observed in the captured stream")
         int observedPacketConnectionTimeout = 30;
+
         @Parameter(required = false,
             names = {"--kafka-traffic-brokers"},
             arity=1,
@@ -185,7 +226,7 @@ public static void main(String[] args) throws IOException, InterruptedException,
             return;
         }
 
-        var tr = new TrafficReplayer(uri, params.authHeaderValue, params.allowInsecureConnections);
+        var tr = new TrafficReplayer(uri, buildAuthTransformerFactory(params), params.allowInsecureConnections);
         try (OutputStream outputStream = params.outputFilename == null ? System.out :
                 new FileOutputStream(params.outputFilename, true)) {
             try (var bufferedOutputStream = new BufferedOutputStream(outputStream)) {
@@ -198,6 +239,60 @@ public static void main(String[] args) throws IOException, InterruptedException,
         }
     }
 
+    /**
+     * Java doesn't have a notion of constexpr like C++ does, so this cannot be used within the
+     * parameters' annotation descriptions, but it's still useful to break the documentation
+     * aspect out from the core logic below.
+     */
+    private static String formatAuthArgFlagsAsString() {
+        return List.of(REMOVE_AUTH_HEADER_VALUE_ARG,
+                        AUTH_HEADER_VALUE_ARG,
+                        AWS_AUTH_HEADER_USER_AND_SECRET_ARG,
+                        SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG).stream()
+                .collect(Collectors.joining(", "));
+    }
+
+    private static IAuthTransformerFactory buildAuthTransformerFactory(Parameters params) {
+        if (params.removeAuthHeader &&
+                params.authHeaderValue != null &&
+                params.useSigV4ServiceAndRegion != null &&
+                params.awsAuthHeaderUserAndSecret != null) {
+            throw new RuntimeException("Cannot specify more than one auth option: " +
+                    formatAuthArgFlagsAsString());
+        }
+
+        var authHeaderValue = params.authHeaderValue;
+        if (params.awsAuthHeaderUserAndSecret != null) {
+            if (params.awsAuthHeaderUserAndSecret.size() != 2) {
+                throw new ParameterException(AWS_AUTH_HEADER_USER_AND_SECRET_ARG +
+                        " must specify two arguments, <USERNAME> <SECRET_ARN>");
+            }
+            try (AWSAuthService awsAuthService = new AWSAuthService()) {
+                authHeaderValue = awsAuthService.getBasicAuthHeaderFromSecret(params.awsAuthHeaderUserAndSecret.get(0),
+                        params.awsAuthHeaderUserAndSecret.get(1));
+            }
+        }
+
+        if (authHeaderValue != null) {
+            return new StaticAuthTransformerFactory(authHeaderValue);
+        } else if (params.useSigV4ServiceAndRegion != null) {
+            var serviceAndRegion = params.useSigV4ServiceAndRegion.split(",");
+            if (serviceAndRegion.length != 2) {
+                throw new RuntimeException("Format for " + SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG + " must be " +
+                        "'SERVICE_NAME,REGION', such as 'es,us-east-1'");
+            }
+            String serviceName = serviceAndRegion[0];
+            String region = serviceAndRegion[1];
+            DefaultCredentialsProvider defaultCredentialsProvider = DefaultCredentialsProvider.create();
+
+            return httpMessage -> new SigV4Signer(defaultCredentialsProvider, serviceName, region, "https", null);
+        } else if (params.removeAuthHeader) {
+            return RemovingAuthTransformerFactory.instance;
+        } else {
+            return null; // default is to do nothing to auth headers
+        }
+    }
+
     private void runReplayWithIOStreams(Duration observedPacketConnectionTimeout,
                                         Stream<TrafficStream> trafficChunkStream,
                                         BufferedOutputStream bufferedOutputStream)
@@ -213,7 +308,8 @@ private void runReplayWithIOStreams(Duration observedPacketConnectionTimeout,
                 new CapturedTrafficToHttpTransactionAccumulator(observedPacketConnectionTimeout,
                         getRecordedRequestReconstructCompleteHandler(requestFutureMap),
                         getRecordedRequestAndResponseReconstructCompleteHandler(successCount, exceptionCount,
-                                tupleWriter, requestFutureMap, requestToFinalWorkFuturesMap));
+                                tupleWriter, requestFutureMap, requestToFinalWorkFuturesMap)
+        );
         try {
             runReplay(trafficChunkStream, trafficToHttpTransactionAccumulator);
         } catch (Exception e) {
@@ -392,7 +488,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture<String
         }
 
         try {
-            log.info("Source/Shadow Request/Response tuple: " + requestResponseTriple);
+            log.info("Source/Target Request/Response tuple: " + requestResponseTriple);
             tripleWriter.writeJSON(requestResponseTriple);
         } catch (IOException e) {
             log.error("Caught an IOException while writing triples output.");
@@ -409,8 +505,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture<String
     }
 
     private DiagnosticTrackableCompletableFuture<String,AggregatedTransformedResponse>
-    writeToSocketAndClose(HttpMessageAndTimestamp request, String diagnosticLabel)
-    {
+    writeToSocketAndClose(HttpMessageAndTimestamp request, String diagnosticLabel) {
         try {
             log.debug("Assembled request/response - starting to write to socket");
             var packetHandler = packetHandlerFactory.create(diagnosticLabel);
@@ -431,7 +526,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture<String
 
     public void runReplay(Stream<TrafficStream> trafficChunkStream,
                           CapturedTrafficToHttpTransactionAccumulator trafficToHttpTransactionAccumulator) {
-        trafficChunkStream
-                .forEach(ts-> trafficToHttpTransactionAccumulator.accept(ts));
+        trafficChunkStream.forEach(ts-> trafficToHttpTransactionAccumulator.accept(ts));
     }
+
 }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java
index 3854ee96e..b78aa5d27 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java
@@ -20,7 +20,7 @@
 import java.util.zip.GZIPOutputStream;
 
 public class Utils {
-    public static final int MAX_BYTES_SHOWN_FOR_TO_STRING = 128;
+    public static final int MAX_BYTES_SHOWN_FOR_TO_STRING = 4096;
     /**
      * See https://en.wikipedia.org/wiki/Fold_(higher-order_function)
      */
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java
index 39c332037..fb6bc704a 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java
@@ -152,6 +152,7 @@ private static void terminateChannel(ChannelPipeline pipeline) {
 
     @Override
     public DiagnosticTrackableCompletableFuture<String,Void> consumeBytes(ByteBuf packetData) {
+        responseBuilder.addRequestPacket(packetData.duplicate());
         log.debug("Scheduling write of packetData["+packetData+"]" +
                 " hash=" + System.identityHashCode(packetData));
         final var completableFuture = new DiagnosticTrackableCompletableFuture<String, Void>(new CompletableFuture<>(),
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java
index e9688cd4c..7e50e7252 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java
@@ -1,13 +1,12 @@
 package org.opensearch.migrations.replay.datahandlers;
 
-import com.fasterxml.jackson.core.type.TypeReference;
 import lombok.extern.slf4j.Slf4j;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
 import org.opensearch.migrations.replay.datahandlers.http.StrictCaseInsensitiveHttpHeadersMap;
 
 import java.util.AbstractMap;
 import java.util.AbstractSet;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
@@ -22,17 +21,15 @@
 @Slf4j
 public class PayloadAccessFaultingMap extends AbstractMap<String, Object> {
 
-    public static final String CONTENT_TYPE = "content-type";
-    public static final String APPLICATION_JSON = "application/json";
     public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody";
 
     private final boolean isJson;
     private Object onlyValue;
 
     public PayloadAccessFaultingMap(StrictCaseInsensitiveHttpHeadersMap headers) {
-        isJson = Optional.ofNullable(headers.get(CONTENT_TYPE))
+        isJson = Optional.ofNullable(headers.get(IHttpMessage.CONTENT_TYPE))
                 .map(list->list.stream()
-                        .anyMatch(s->s.startsWith(APPLICATION_JSON))).orElse(false);
+                        .anyMatch(s->s.startsWith(IHttpMessage.APPLICATION_JSON))).orElse(false);
     }
 
     @Override
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java
index dad4520dd..d39722fcf 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java
@@ -2,10 +2,12 @@
 
 import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap;
 
+import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
-public class HttpJsonMessageWithFaultingPayload extends LinkedHashMap<String, Object> {
+public class HttpJsonMessageWithFaultingPayload extends LinkedHashMap<String, Object> implements IHttpMessage {
     public final static String METHOD = "method";
     public final static String URI = "URI";
     public final static String PROTOCOL = "protocol";
@@ -19,19 +21,22 @@ public HttpJsonMessageWithFaultingPayload(Map<? extends String, ?> m) {
         super(m);
     }
 
+    @Override
     public String method() {
         return (String) this.get(METHOD);
     }
     public void setMethod(String value) {
         this.put(METHOD, value);
     }
-    public String uri() {
+    @Override
+    public String path() {
         return (String) this.get(URI);
     }
-    public void setUri(String value) {
+    public void setPath(String value) {
         this.put(URI, value);
     }
 
+    @Override
     public String protocol() {
         return (String) this.get(PROTOCOL);
     }
@@ -39,6 +44,12 @@ public void setProtocol(String value) {
         this.put(PROTOCOL, value);
     }
 
+
+    @Override
+    public Map<String, Object> headersMap() {
+        return Collections.unmodifiableMap(headers());
+    }
+
     public ListKeyAdaptingCaseInsensitiveHeadersMap headers() {
         return (ListKeyAdaptingCaseInsensitiveHeadersMap) this.get(HEADERS);
     }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java
index eec73ba03..bf4d9e5e1 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java
@@ -2,7 +2,6 @@
 
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.embedded.EmbeddedChannel;
-import io.netty.handler.codec.http.DefaultLastHttpContent;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import lombok.extern.slf4j.Slf4j;
 import org.opensearch.migrations.replay.AggregatedRawResponse;
@@ -11,7 +10,8 @@
 import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer;
 import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
 import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.IAuthTransformerFactory;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -56,14 +56,15 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer<A
     // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed.
     private final List<ByteBuf> chunks;
 
-    public HttpJsonTransformingConsumer(JsonTransformer transformer,
-                                        IPacketFinalizingConsumer transformedPacketReceiver,
+    public HttpJsonTransformingConsumer(IJsonTransformer transformer,
+                                        IAuthTransformerFactory authTransformerFactory, IPacketFinalizingConsumer transformedPacketReceiver,
                                         String diagnosticLabel) {
         chunkSizes = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS);
         chunkSizes.add(new ArrayList<>(EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS));
         chunks = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS + EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS);
         channel = new EmbeddedChannel();
-        pipelineOrchestrator = new RequestPipelineOrchestrator(chunkSizes, transformedPacketReceiver, diagnosticLabel);
+        pipelineOrchestrator = new RequestPipelineOrchestrator(chunkSizes, transformedPacketReceiver,
+                authTransformerFactory, diagnosticLabel);
         pipelineOrchestrator.addInitialHandlers(channel.pipeline(), transformer);
     }
 
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/IHttpMessage.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/IHttpMessage.java
new file mode 100644
index 000000000..d1fadfa77
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/IHttpMessage.java
@@ -0,0 +1,26 @@
+package org.opensearch.migrations.replay.datahandlers.http;
+
+import java.util.List;
+import java.util.Map;
+
+public interface IHttpMessage {
+    String APPLICATION_JSON = "application/json";
+    String CONTENT_TYPE = "content-type";
+
+    String method();
+
+    String path();
+
+    String protocol();
+
+    Map<String,Object> headersMap();
+
+    default String getFirstHeader(String key) {
+        var all = getAllMatchingHeaders(key);
+        return all == null ? null : all.get(0);
+    }
+
+    default List<String> getAllMatchingHeaders(String key) {
+        return ((List<String>) (headersMap().get(key)));
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java
similarity index 61%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestHandler.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java
index 6e7c48453..16c718e9a 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestHandler.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java
@@ -2,41 +2,35 @@
 
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.handler.codec.DecoderException;
 import io.netty.handler.codec.http.HttpContent;
 import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpRequestDecoder;
 import lombok.extern.slf4j.Slf4j;
-import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer;
 import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap;
 import org.opensearch.migrations.replay.datahandlers.PayloadNotLoadedException;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.IAuthTransformer;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 @Slf4j
-public class NettyDecodedHttpRequestHandler extends ChannelInboundHandlerAdapter {
+public class NettyDecodedHttpRequestPreliminaryConvertHandler extends ChannelInboundHandlerAdapter {
     public static final int EXPECTED_PACKET_COUNT_GUESS_FOR_PAYLOAD = 32;
-    /**
-     * This is stored as part of a closure for the pipeline continuation that will be triggered
-     * once it becomes apparent if we need to process the payload stream or if we can pass it
-     * through as is.
-     */
-    final IPacketFinalizingConsumer packetReceiver;
-    final JsonTransformer transformer;
+
+    final RequestPipelineOrchestrator requestPipelineOrchestrator;
+    final IJsonTransformer transformer;
     final List<List<Integer>> chunkSizes;
     final String diagnosticLabel;
 
-    public NettyDecodedHttpRequestHandler(JsonTransformer transformer,
-                                          List<List<Integer>> chunkSizes,
-                                          IPacketFinalizingConsumer packetReceiver,
-                                          String diagnosticLabel) {
-        this.packetReceiver = packetReceiver;
+    public NettyDecodedHttpRequestPreliminaryConvertHandler(IJsonTransformer transformer,
+                                                            List<List<Integer>> chunkSizes,
+                                                            RequestPipelineOrchestrator requestPipelineOrchestrator,
+                                                            String diagnosticLabel) {
         this.transformer = transformer;
         this.chunkSizes = chunkSizes;
+        this.requestPipelineOrchestrator = requestPipelineOrchestrator;
         this.diagnosticLabel = "[" + diagnosticLabel + "] ";
     }
 
@@ -55,17 +49,20 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
 
             // TODO - this is super ugly and sloppy - this has to be improved
             chunkSizes.add(new ArrayList<>(EXPECTED_PACKET_COUNT_GUESS_FOR_PAYLOAD));
-            var pipelineOrchestrator = new RequestPipelineOrchestrator(chunkSizes, packetReceiver, diagnosticLabel);
             var pipeline = ctx.pipeline();
+            var originalHttpJsonMessage = parseHeadersIntoMessage(request);
+            IAuthTransformer authTransformer =
+                    requestPipelineOrchestrator.authTransfomerFactory.getAuthTransformer(originalHttpJsonMessage);
             try {
-                var httpJsonMessage = transform(transformer, parseHeadersIntoMessage(request));
-                handlePayloadNeutralTransformation(ctx, request, httpJsonMessage, pipelineOrchestrator);
+                handlePayloadNeutralTransformationOrThrow(ctx, request, transform(transformer, originalHttpJsonMessage),
+                        authTransformer);
             } catch (PayloadNotLoadedException pnle) {
                 log.debug("The transforms for this message require payload manipulation, " +
                         "all content handlers are being loaded.");
                 // make a fresh message and its headers
-                pipelineOrchestrator.addJsonParsingHandlers(pipeline, transformer);
-                ctx.fireChannelRead(parseHeadersIntoMessage(request));
+                requestPipelineOrchestrator.addJsonParsingHandlers(pipeline, transformer,
+                        getAuthTransformerAsStreamingTransformer(authTransformer));
+                ctx.fireChannelRead(handleAuthHeaders(parseHeadersIntoMessage(request), authTransformer));
             }
         } else if (msg instanceof HttpContent) {
             ctx.fireChannelRead(msg);
@@ -77,7 +74,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         }
     }
 
-    private static HttpJsonMessageWithFaultingPayload transform(JsonTransformer transformer,
+    private static HttpJsonMessageWithFaultingPayload transform(IJsonTransformer transformer,
                                                                 HttpJsonMessageWithFaultingPayload httpJsonMessage) {
         var returnedObject = transformer.transformJson(httpJsonMessage);
         if (returnedObject != httpJsonMessage) {
@@ -87,13 +84,23 @@ private static HttpJsonMessageWithFaultingPayload transform(JsonTransformer tran
         return httpJsonMessage;
     }
 
-    private void handlePayloadNeutralTransformation(ChannelHandlerContext ctx,
-                                                    HttpRequest request,
-                                                    HttpJsonMessageWithFaultingPayload httpJsonMessage,
-                                                    RequestPipelineOrchestrator pipelineOrchestrator)
+    private void handlePayloadNeutralTransformationOrThrow(ChannelHandlerContext ctx,
+                                                           HttpRequest request,
+                                                           HttpJsonMessageWithFaultingPayload httpJsonMessage,
+                                                           IAuthTransformer authTransformer)
     {
+        // if the auth transformer only requires header manipulations, just do it right away, otherwise,
+        // if it's a streaming transformer, require content parsing and send it in there
+        handleAuthHeaders(httpJsonMessage, authTransformer);
+        var streamingAuthTransformer = getAuthTransformerAsStreamingTransformer(authTransformer);
+
         var pipeline = ctx.pipeline();
-        if (headerFieldsAreIdentical(request, httpJsonMessage)) {
+        if (streamingAuthTransformer != null) {
+            log.info(diagnosticLabel + "New headers have been specified that require the payload stream to be " +
+                    "reformatted, adding Content Handlers to this pipeline.");
+            requestPipelineOrchestrator.addContentRepackingHandlers(pipeline, streamingAuthTransformer);
+            ctx.fireChannelRead(httpJsonMessage);
+        } else if (headerFieldsAreIdentical(request, httpJsonMessage)) {
             log.info(diagnosticLabel + "Transformation isn't necessary.  " +
                     "Clearing pipeline to let the parent context redrive directly.");
             while (pipeline.first() != null) {
@@ -104,19 +111,33 @@ private void handlePayloadNeutralTransformation(ChannelHandlerContext ctx,
             log.info(diagnosticLabel + "There were changes to the headers that require the message to be reformatted " +
                     "through this pipeline but the content (payload) doesn't need to be transformed.  " +
                     "Content Handlers are not being added to the pipeline");
-            pipelineOrchestrator.addBaselineHandlers(pipeline);
+            requestPipelineOrchestrator.addBaselineHandlers(pipeline);
             ctx.fireChannelRead(httpJsonMessage);
             RequestPipelineOrchestrator.removeThisAndPreviousHandlers(pipeline, this);
         } else {
             log.info(diagnosticLabel + "New headers have been specified that require the payload stream to be " +
                     "reformatted, adding Content Handlers to this pipeline.");
-            pipelineOrchestrator.addContentRepackingHandlers(pipeline);
+            requestPipelineOrchestrator.addContentRepackingHandlers(pipeline, streamingAuthTransformer);
             ctx.fireChannelRead(httpJsonMessage);
         }
     }
 
+    private static HttpJsonMessageWithFaultingPayload
+    handleAuthHeaders(HttpJsonMessageWithFaultingPayload httpJsonMessage, IAuthTransformer authTransformer) {
+        if (authTransformer != null && authTransformer instanceof IAuthTransformer.HeadersOnlyTransformer) {
+            ((IAuthTransformer.HeadersOnlyTransformer) authTransformer).rewriteHeaders(httpJsonMessage);
+        }
+        return httpJsonMessage;
+    }
+
+    private static IAuthTransformer.StreamingFullMessageTransformer
+    getAuthTransformerAsStreamingTransformer(IAuthTransformer authTransformer) {
+        return (authTransformer instanceof IAuthTransformer.StreamingFullMessageTransformer) ?
+                (IAuthTransformer.StreamingFullMessageTransformer) authTransformer : null;
+    }
+
     private boolean headerFieldsAreIdentical(HttpRequest request, HttpJsonMessageWithFaultingPayload httpJsonMessage) {
-        if (!request.uri().equals(httpJsonMessage.uri()) ||
+        if (!request.uri().equals(httpJsonMessage.path()) ||
                 !request.method().toString().equals(httpJsonMessage.method())) {
             return false;
         }
@@ -130,7 +151,7 @@ private boolean headerFieldsAreIdentical(HttpRequest request, HttpJsonMessageWit
 
     private static HttpJsonMessageWithFaultingPayload parseHeadersIntoMessage(HttpRequest request) {
         var jsonMsg = new HttpJsonMessageWithFaultingPayload();
-        jsonMsg.setUri(request.uri().toString());
+        jsonMsg.setPath(request.uri().toString());
         jsonMsg.setMethod(request.method().toString());
         jsonMsg.setProtocol(request.protocolVersion().text());
         var headers = request.headers().entries().stream()
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java
index 7b7be161f..d022185e8 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java
@@ -2,14 +2,14 @@
 
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.util.Map;
 
 public class NettyJsonBodyConvertHandler extends ChannelInboundHandlerAdapter {
-    private final JsonTransformer transformer;
+    private final IJsonTransformer transformer;
 
-    public NettyJsonBodyConvertHandler(JsonTransformer transformer) {
+    public NettyJsonBodyConvertHandler(IJsonTransformer transformer) {
         this.transformer = transformer;
     }
 
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentAuthSigner.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentAuthSigner.java
new file mode 100644
index 000000000..3772a54eb
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentAuthSigner.java
@@ -0,0 +1,46 @@
+package org.opensearch.migrations.replay.datahandlers.http;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.LastHttpContent;
+import org.opensearch.migrations.transform.IAuthTransformer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class NettyJsonContentAuthSigner extends ChannelInboundHandlerAdapter {
+    IAuthTransformer.StreamingFullMessageTransformer signer;
+    HttpJsonMessageWithFaultingPayload httpMessage;
+    List<HttpContent> receivedHttpContents;
+
+    public NettyJsonContentAuthSigner(IAuthTransformer.StreamingFullMessageTransformer signer) {
+        this.signer = signer;
+        this.receivedHttpContents = new ArrayList<>();
+    }
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+        if (msg instanceof HttpJsonMessageWithFaultingPayload) {
+            httpMessage = (HttpJsonMessageWithFaultingPayload) msg;
+        } else if (msg instanceof HttpContent) {
+            receivedHttpContents.add(((HttpContent) msg).retainedDuplicate());
+            var httpContent = (HttpContent) msg;
+            signer.consumeNextPayloadPart(httpContent.content().nioBuffer());
+            if  (msg instanceof LastHttpContent) {
+                finalizeSignature(ctx);
+            }
+        } else {
+            super.channelRead(ctx, msg);
+        }
+    }
+
+    private void finalizeSignature(ChannelHandlerContext ctx) {
+        signer.finalize(httpMessage);
+        ctx.fireChannelRead(httpMessage);
+        receivedHttpContents.stream().forEach(content->{
+            ctx.fireChannelRead(content);
+            content.content().release();
+        });
+    }
+}
\ No newline at end of file
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentCompressor.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentCompressor.java
index b99b28063..09e17f05d 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentCompressor.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentCompressor.java
@@ -4,7 +4,6 @@
 import io.netty.buffer.ByteBufAllocator;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.handler.codec.DecoderException;
 import io.netty.handler.codec.http.DefaultHttpContent;
 import io.netty.handler.codec.http.DefaultLastHttpContent;
 import io.netty.handler.codec.http.HttpContent;
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java
index 3d3fda398..1b77d07f3 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java
@@ -106,7 +106,7 @@ private void sendEndChunk(ChannelHandlerContext ctx) {
     }
 
     private void finalizeFixedContentStream(ChannelHandlerContext ctx) {
-        bufferedJsonMessage.headers().put(CONTENT_LENGTH_HEADER_NAME, contentBytesReceived).toString();
+        bufferedJsonMessage.headers().put(CONTENT_LENGTH_HEADER_NAME, contentBytesReceived);
         ctx.fireChannelRead(bufferedJsonMessage);
         bufferedJsonMessage = null;
         ctx.fireChannelRead(bufferedContents);
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java
index 67f74743f..8c30afc11 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java
@@ -7,7 +7,6 @@
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.handler.codec.DecoderException;
 import io.netty.handler.codec.http.HttpContent;
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.util.ResourceLeakDetector;
@@ -183,10 +182,10 @@ private static void writeHeadersIntoStream(HttpJsonMessageWithFaultingPayload ht
         try (var osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) {
             osw.append(httpJson.method());
             osw.append(" ");
-            osw.append(httpJson.uri());
+            osw.append(httpJson.path());
             osw.append(" ");
             osw.append(httpJson.protocol());
-            osw.append("\n");
+            osw.append("\r\n");
 
             for (var kvpList : httpJson.headers().asStrictMap().entrySet()) {
                 var key = kvpList.getKey();
@@ -194,10 +193,10 @@ private static void writeHeadersIntoStream(HttpJsonMessageWithFaultingPayload ht
                     osw.append(key);
                     osw.append(": ");
                     osw.append(valueEntry);
-                    osw.append("\n");
+                    osw.append("\r\n");
                 }
             }
-            osw.append("\n");
+            osw.append("\r\n");
             osw.flush();
         }
     }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java
index 0a6cfa3f9..b821e2092 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java
@@ -6,9 +6,12 @@
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.logging.LogLevel;
 import io.netty.handler.logging.LoggingHandler;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.IAuthTransformer;
+import org.opensearch.migrations.transform.IAuthTransformerFactory;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -38,12 +41,17 @@ public class RequestPipelineOrchestrator {
     private final List<List<Integer>> chunkSizes;
     final IPacketFinalizingConsumer packetReceiver;
     final String diagnosticLabel;
+    @Getter
+    final IAuthTransformerFactory authTransfomerFactory;
 
     public RequestPipelineOrchestrator(List<List<Integer>> chunkSizes,
                                        IPacketFinalizingConsumer packetReceiver,
+                                       IAuthTransformerFactory incomingAuthTransformerFactory,
                                        String diagnosticLabel) {
         this.chunkSizes = chunkSizes;
         this.packetReceiver = packetReceiver;
+        this.authTransfomerFactory = incomingAuthTransformerFactory != null ? incomingAuthTransformerFactory :
+                IAuthTransformerFactory.NullAuthTransformerFactory.instance;
         this.diagnosticLabel = diagnosticLabel;
     }
 
@@ -61,15 +69,18 @@ static void removeThisAndPreviousHandlers(ChannelPipeline pipeline, ChannelHandl
         }
     }
 
-    void addContentRepackingHandlers(ChannelPipeline pipeline) {
-        addContentParsingHandlers(pipeline, null);
+    void addContentRepackingHandlers(ChannelPipeline pipeline,
+                                     IAuthTransformer.StreamingFullMessageTransformer authTransfomer) {
+        addContentParsingHandlers(pipeline, null, authTransfomer);
     }
 
-    void addJsonParsingHandlers(ChannelPipeline pipeline, JsonTransformer transformer) {
-        addContentParsingHandlers(pipeline, transformer);
+    void addJsonParsingHandlers(ChannelPipeline pipeline,
+                                IJsonTransformer transformer,
+                                IAuthTransformer.StreamingFullMessageTransformer authTransfomer) {
+        addContentParsingHandlers(pipeline, transformer, authTransfomer);
     }
 
-    void addInitialHandlers(ChannelPipeline pipeline, JsonTransformer transformer) {
+    void addInitialHandlers(ChannelPipeline pipeline, IJsonTransformer transformer) {
         pipeline.addFirst(HTTP_REQUEST_DECODER_NAME, new HttpRequestDecoder());
         addLoggingHandler(pipeline, "A");
         // IN:  Netty HttpRequest(1) + HttpContent(1) blocks (which may be compressed) + EndOfInput + ByteBuf
@@ -83,11 +94,13 @@ void addInitialHandlers(ChannelPipeline pipeline, JsonTransformer transformer) {
         // Note3: ByteBufs will be sent through when there were pending bytes left to be parsed by the
         //        HttpRequestDecoder when the HttpRequestDecoder is removed from the pipeline BEFORE the
         //        NettyDecodedHttpRequestHandler is removed.
-        pipeline.addLast(new NettyDecodedHttpRequestHandler(transformer, chunkSizes, packetReceiver, diagnosticLabel));
+        pipeline.addLast(new NettyDecodedHttpRequestPreliminaryConvertHandler(transformer, chunkSizes, this, diagnosticLabel));
         addLoggingHandler(pipeline, "B");
     }
 
-    void addContentParsingHandlers(ChannelPipeline pipeline, JsonTransformer transformer) {
+    void addContentParsingHandlers(ChannelPipeline pipeline,
+                                   IJsonTransformer transformer,
+                                   IAuthTransformer.StreamingFullMessageTransformer authTransfomer) {
         log.debug("Adding content parsing handlers to pipeline");
         //  IN: Netty HttpRequest(1) + HttpJsonMessage(1) with headers + HttpContent(1) blocks (which may be compressed)
         // OUT: Netty HttpRequest(2) + HttpJsonMessage(1) with headers + HttpContent(2) uncompressed blocks
@@ -106,25 +119,29 @@ void addContentParsingHandlers(ChannelPipeline pipeline, JsonTransformer transfo
             pipeline.addLast(new NettyJsonBodySerializeHandler());
             addLoggingHandler(pipeline, "F");
         }
+        if (authTransfomer != null) {
+            pipeline.addLast(new NettyJsonContentAuthSigner(authTransfomer));
+            addLoggingHandler(pipeline, "G");
+        }
         // IN:  Netty HttpRequest(2) + HttpJsonMessage(3) with headers only + HttpContent(3) blocks
         // OUT: Netty HttpRequest(3) + HttpJsonMessage(4) with headers only + HttpContent(4) blocks
         pipeline.addLast(new NettyJsonContentCompressor());
-        addLoggingHandler(pipeline, "G");
+        addLoggingHandler(pipeline, "H");
         // IN:  Netty HttpRequest(3) + HttpJsonMessage(4) with headers only + HttpContent(4) blocks + EndOfInput
         // OUT: Netty HttpRequest(3) + HttpJsonMessage(4) with headers only + ByteBufs(2)
         pipeline.addLast(new NettyJsonContentStreamToByteBufHandler());
-        addLoggingHandler(pipeline, "H");
+        addLoggingHandler(pipeline, "I");
         addBaselineHandlers(pipeline);
     }
 
     void addBaselineHandlers(ChannelPipeline pipeline) {
-        addLoggingHandler(pipeline, "I");
+        addLoggingHandler(pipeline, "J");
         //  IN: ByteBufs(2) + HttpJsonMessage(4) with headers only + HttpContent(1) (if the repackaging handlers were skipped)
         // OUT: ByteBufs(3) which are sized similarly to how they were received
         pipeline.addLast(new NettyJsonToByteBufHandler(Collections.unmodifiableList(chunkSizes)));
         // IN:  ByteBufs(3)
         // OUT: nothing - terminal!  ByteBufs are routed to the packet handler!
-        addLoggingHandler(pipeline, "J");
+        addLoggingHandler(pipeline, "K");
         pipeline.addLast(OFFLOADING_HANDLER_NAME,
                 new NettySendByteBufsToPacketHandlerHandler(packetReceiver, diagnosticLabel));
     }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformer.java
new file mode 100644
index 000000000..8336e4909
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformer.java
@@ -0,0 +1,31 @@
+package org.opensearch.migrations.transform;
+
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
+
+import java.nio.ByteBuffer;
+
+public interface IAuthTransformer {
+    enum ContextForAuthHeader {
+        HEADERS,
+        HEADERS_AND_CONTENT_PAYLOAD
+    }
+
+    ContextForAuthHeader transformType();
+
+    abstract class HeadersOnlyTransformer implements IAuthTransformer {
+        @Override
+        public ContextForAuthHeader transformType() {
+            return ContextForAuthHeader.HEADERS;
+        }
+        public abstract void rewriteHeaders(HttpJsonMessageWithFaultingPayload msg);
+    }
+
+    abstract class StreamingFullMessageTransformer implements IAuthTransformer {
+        @Override
+        public ContextForAuthHeader transformType() {
+            return ContextForAuthHeader.HEADERS_AND_CONTENT_PAYLOAD; }
+        public abstract void consumeNextPayloadPart(ByteBuffer contentChunk);
+        public abstract void finalize(HttpJsonMessageWithFaultingPayload msg);
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java
new file mode 100644
index 000000000..4587ae9a9
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java
@@ -0,0 +1,19 @@
+package org.opensearch.migrations.transform;
+
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
+
+public interface IAuthTransformerFactory {
+    IAuthTransformer getAuthTransformer(IHttpMessage httpMessage);
+
+    class NullAuthTransformerFactory implements IAuthTransformerFactory {
+        public final static NullAuthTransformerFactory instance = new NullAuthTransformerFactory();
+
+        public NullAuthTransformerFactory() {}
+
+        @Override
+        public IAuthTransformer getAuthTransformer(IHttpMessage httpMessage) {
+            return null;
+        }
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTransformer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java
similarity index 88%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTransformer.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java
index 7d211cab7..fa35a862a 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTransformer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java
@@ -4,6 +4,6 @@
  * This is a simple interface to convert a JSON object (String, Map, or Array) into another
  * JSON object.  Any changes to datastructures, nesting, order, etc should be intentional.
  */
-public interface JsonTransformer {
+public interface IJsonTransformer {
     Object transformJson(Object incomingJson);
 }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/CompositeJsonTransformer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java
similarity index 70%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/CompositeJsonTransformer.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java
index bbdab1d54..1d8ae4fde 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/CompositeJsonTransformer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java
@@ -3,10 +3,10 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 
-public class CompositeJsonTransformer implements JsonTransformer {
-    List<JsonTransformer> jsonTransformerList;
+public class JsonCompositeTransformer implements IJsonTransformer {
+    List<IJsonTransformer> jsonTransformerList;
 
-    public CompositeJsonTransformer(JsonTransformer... jsonTransformers) {
+    public JsonCompositeTransformer(IJsonTransformer... jsonTransformers) {
         this.jsonTransformerList = List.of(jsonTransformers);
     }
 
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformBuilder.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java
similarity index 84%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformBuilder.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java
index 4d408984f..2f69cbf69 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformBuilder.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java
@@ -14,7 +14,7 @@
 import java.util.Map;
 
 @Slf4j
-public class JoltJsonTransformBuilder {
+public class JsonJoltTransformBuilder {
 
     private static String getSubstitutionTemplate(int i) {
         return "%%SUBSTITION_" + (i+1) + "%%";
@@ -40,7 +40,6 @@ private enum OPERATION {
         ADD_GZIP(CANNED_OPERATION.ADD_GZIP.joltOperationTransformName),
         MAKE_CHUNKED(CANNED_OPERATION.MAKE_CHUNKED.joltOperationTransformName),
         PASS_THRU(CANNED_OPERATION.PASS_THRU.joltOperationTransformName),
-        ADD_ADMIN_AUTH("addAdminAuth", 1),
         HOST_SWITCH("hostSwitch", 1);
 
         private final String value;
@@ -73,7 +72,7 @@ public Map<String, Object> loadResourceAsJson(String path) throws IOException {
     }
 
     public static Map<String, Object> loadResourceAsJson(ObjectMapper mapper, String path) throws IOException {
-        try (InputStream inputStream = JoltJsonTransformBuilder.class.getResourceAsStream(path)) {
+        try (InputStream inputStream = JsonJoltTransformBuilder.class.getResourceAsStream(path)) {
             return mapper.readValue(inputStream, TYPE_REFERENCE_FOR_MAP_TYPE);
         }
     }
@@ -87,7 +86,7 @@ private Map<String, Object> parseSpecOperationFromResource(String resource) {
     private Map<String, Object> getOperationWithSubstitutions(OPERATION operation, String...substitutions) {
         var path = "/jolt/operations/" + operation.value + ".jolt.template";
         assert substitutions.length == operation.numberOfTemplateSubstitutions;
-        try (InputStream inputStream = JoltJsonTransformBuilder.class.getResourceAsStream(path)) {
+        try (InputStream inputStream = JsonJoltTransformBuilder.class.getResourceAsStream(path)) {
             var contentBytes = inputStream.readAllBytes();
             var contentsStr = new String(contentBytes, StandardCharsets.UTF_8);
             for (int i=0; i<substitutions.length; ++i) {
@@ -97,27 +96,23 @@ private Map<String, Object> getOperationWithSubstitutions(OPERATION operation, S
         }
     }
 
-    public JoltJsonTransformBuilder addHostSwitchOperation(String hostname) {
+    public JsonJoltTransformBuilder addHostSwitchOperation(String hostname) {
         return addOperationObject(getOperationWithSubstitutions(OPERATION.HOST_SWITCH, hostname));
     }
 
-    public JoltJsonTransformBuilder addAuthorizationOperation(String value) {
-        return addOperationObject(getOperationWithSubstitutions(OPERATION.ADD_ADMIN_AUTH, value));
-    }
-
-    public JoltJsonTransformBuilder addCannedOperation(CANNED_OPERATION operation) {
+    public JsonJoltTransformBuilder addCannedOperation(CANNED_OPERATION operation) {
         return addOperationObject(parseSpecOperationFromResource(operation.joltOperationTransformName));
     }
 
-    public JoltJsonTransformBuilder addOperationObject(Map<String, Object> stringObjectMap) {
+    public JsonJoltTransformBuilder addOperationObject(Map<String, Object> stringObjectMap) {
         chainedSpec.add(stringObjectMap);
         return this;
     }
 
-    public JsonTransformer build() {
+    public IJsonTransformer build() {
         if (chainedSpec.size() == 0) {
             addCannedOperation(CANNED_OPERATION.PASS_THRU);
         }
-        return new JoltJsonTransformer((List) chainedSpec);
+        return new JsonJoltTransformer((List) chainedSpec);
     }
 }
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java
similarity index 59%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformer.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java
index 8498ed54b..a645b4804 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JoltJsonTransformer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java
@@ -4,17 +4,17 @@
 
 import java.util.List;
 
-public class JoltJsonTransformer implements JsonTransformer {
+public class JsonJoltTransformer implements IJsonTransformer {
 
     Chainr spec;
 
 
-    public JoltJsonTransformer(List<Object> joltOperationsSpecList) {
+    public JsonJoltTransformer(List<Object> joltOperationsSpecList) {
         this.spec = Chainr.fromSpec(joltOperationsSpecList);
     }
 
-    public static JoltJsonTransformBuilder newBuilder() {
-        return new JoltJsonTransformBuilder();
+    public static JsonJoltTransformBuilder newBuilder() {
+        return new JsonJoltTransformBuilder();
     }
 
     @Override
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/TypeMappingJsonTransformer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java
similarity index 98%
rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/TypeMappingJsonTransformer.java
rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java
index faff0c2d6..71820716b 100644
--- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/TypeMappingJsonTransformer.java
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java
@@ -10,7 +10,7 @@
  * This is an experimental JsonTransformer that is meant to perform basic URI and payload transformations
  * to excise index type mappings for relevant operations.
  */
-public class TypeMappingJsonTransformer implements JsonTransformer {
+public class JsonTypeMappingTransformer implements IJsonTransformer {
     /**
      * This is used to match a URI of the form /INDEX/TYPE/foo... so that it can be
      * transformed into /INDEX/foo...
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java
new file mode 100644
index 000000000..0ef986147
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java
@@ -0,0 +1,25 @@
+package org.opensearch.migrations.transform;
+
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
+
+public class RemovingAuthTransformerFactory implements IAuthTransformerFactory {
+
+    public static final RemovingAuthTransformerFactory instance = new RemovingAuthTransformerFactory();
+
+    private RemovingAuthTransformerFactory() {}
+
+    @Override
+    public IAuthTransformer getAuthTransformer(IHttpMessage httpMessage) {
+        return RemovingAuthTransformer.instance;
+    }
+
+    private static class RemovingAuthTransformer extends IAuthTransformer.HeadersOnlyTransformer {
+        private static final RemovingAuthTransformer instance = new RemovingAuthTransformer();
+
+        @Override
+        public void rewriteHeaders(HttpJsonMessageWithFaultingPayload msg) {
+            msg.headers().remove("authorization");
+        }
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java
new file mode 100644
index 000000000..8b47a777e
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java
@@ -0,0 +1,23 @@
+package org.opensearch.migrations.transform;
+
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload;
+import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage;
+
+public class StaticAuthTransformerFactory implements IAuthTransformerFactory {
+    private final String authHeaderValue;
+
+    public StaticAuthTransformerFactory(String authHeaderValue) {
+        this.authHeaderValue = authHeaderValue;
+    }
+
+    @Override
+    public IAuthTransformer getAuthTransformer(IHttpMessage httpMessage) {
+        return new IAuthTransformer.HeadersOnlyTransformer() {
+            @Override
+            public void rewriteHeaders(HttpJsonMessageWithFaultingPayload msg) {
+                msg.headers().put("authorization", authHeaderValue);
+                // TODO - wipe out more headers too?
+            }
+        };
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/addAdminAuth.jolt.template b/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/addAdminAuth.jolt.template
deleted file mode 100644
index 88db863e6..000000000
--- a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/addAdminAuth.jolt.template
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-  "operation": "modify-overwrite-beta",
-  "spec": {
-    "headers": {
-      "authorization": "%%SUBSTITION_1%%"
-    }
-  }
-}
\ No newline at end of file
diff --git a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties
index 4b2850560..594ce9d97 100644
--- a/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties
+++ b/TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties
@@ -1,5 +1,7 @@
 status = error
 
+property.tupleDir = ${env:TUPLE_DIR_PATH:-./logs/tuples}
+
 appender.console.type = Console
 appender.console.name = STDERR
 appender.console.target = SYSTEM_ERR
@@ -21,3 +23,25 @@ logger.HttpJsonTransformingConsumer.level = info
 
 #logger.NettySendByteBufsToPacketHandlerHandler.name = org.opensearch.migrations.replay.datahandlers.http.NettySendByteBufsToPacketHandlerHandler
 #logger.NettySendByteBufsToPacketHandlerHandler.level = trace
+
+appender.output_tuples.type = RollingFile
+appender.output_tuples.name = OUTPUT_TUPLES
+# This is the path to the shared volume configured by the deployment tools.
+appender.output_tuples.fileName = ${tupleDir}/tuples.log
+appender.output_tuples.filePattern = ${tupleDir}/tuples-%d{yyyy-MM-dd-HH:mm}.log
+appender.output_tuples.layout.type = JsonLayout
+appender.output_tuples.layout.properties = false
+appender.output_tuples.layout.complete = false
+appender.output_tuples.layout.eventEol = true
+appender.output_tuples.layout.compact = true
+appender.output_tuples.policies.type = Policies
+appender.output_tuples.policies.size.type = SizeBasedTriggeringPolicy
+# The unit for the interval is set based on the most fine-grained unit in the filePattern, so it rolls over each hour.
+appender.output_tuples.policies.size.size = 10 MB
+appender.output_tuples.strategy.type = DefaultRolloverStrategy
+
+logger.OutputTupleJsonLogger.name = OutputTupleJsonLogger
+logger.OutputTupleJsonLogger.level = info
+logger.OutputTupleJsonLogger.additivity = false
+logger.OutputTupleJsonLogger.appenderRefs = output_tuples
+logger.OutputTupleJsonLogger.appenderRef.output_tuples.ref = OUTPUT_TUPLES
\ No newline at end of file
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AWSAuthServiceTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AWSAuthServiceTest.java
new file mode 100644
index 000000000..caf98e77f
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AWSAuthServiceTest.java
@@ -0,0 +1,38 @@
+package org.opensearch.migrations.replay;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
+
+import java.util.function.Consumer;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class AWSAuthServiceTest {
+
+    @Mock
+    private SecretsManagerClient secretsManagerClient;
+
+    @Test
+    public void testBasicAuthHeaderFromSecret() {
+        String testSecretId = "testSecretId";
+        String testUsername = "testAdmin";
+        String expectedResult = "Basic dGVzdEFkbWluOmFkbWluUGFzcw==";
+
+        GetSecretValueResponse response = GetSecretValueResponse.builder().secretString("adminPass").build();
+
+        when(secretsManagerClient.getSecretValue(any(Consumer.class))).thenReturn(response);
+
+        AWSAuthService awsAuthService = new AWSAuthService(secretsManagerClient);
+        String header = awsAuthService.getBasicAuthHeaderFromSecret(testUsername, testSecretId);
+        Assertions.assertEquals(expectedResult, header);
+    }
+
+
+}
\ No newline at end of file
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java
index 8f30e341b..ffac19cb5 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java
@@ -6,9 +6,8 @@
 import org.junit.jupiter.api.Test;
 import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer;
 import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
-import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
-import org.opensearch.migrations.transform.JoltJsonTransformBuilder;
-import org.opensearch.migrations.transform.JoltJsonTransformer;
+import org.opensearch.migrations.transform.JsonJoltTransformBuilder;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -34,9 +33,9 @@ public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionEx
                 null, AggregatedTransformedResponse.HttpRequestTransformationStatus.COMPLETED);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
         var compressingTransformer = new HttpJsonTransformingConsumer(
-                JoltJsonTransformer.newBuilder()
-                        .addCannedOperation(JoltJsonTransformBuilder.CANNED_OPERATION.ADD_GZIP)
-                        .build(), testPacketCapture, "TEST");
+                JsonJoltTransformer.newBuilder()
+                        .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.ADD_GZIP)
+                        .build(), null, testPacketCapture, "TEST");
 
         final var payloadPartSize = 511;
         final var numParts = 1025;
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java
index 9684733a5..3bd283a5f 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java
@@ -5,8 +5,8 @@
 import org.junit.jupiter.api.Test;
 import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer;
 import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
-import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
-import org.opensearch.migrations.transform.JoltJsonTransformer;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
+import org.opensearch.migrations.transform.StaticAuthTransformerFactory;
 
 import java.time.Duration;
 import java.util.AbstractMap;
@@ -30,14 +30,14 @@ public void testTransformer() throws Exception {
         final var dummyAggregatedResponse = new AggregatedTransformedResponse(17, null, null,
                 null, AggregatedTransformedResponse.HttpRequestTransformationStatus.COMPLETED);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
-        var jsonHandler = JoltJsonTransformer.newBuilder()
+        var jsonHandler = JsonJoltTransformer.newBuilder()
                 .addHostSwitchOperation(SILLY_TARGET_CLUSTER_NAME)
                 .build();
-        var transformingHandler = new HttpJsonTransformingConsumer(jsonHandler, testPacketCapture, "TEST");
+        var transformingHandler = new HttpJsonTransformingConsumer(jsonHandler, null, testPacketCapture, "TEST");
         runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture,
-                contentLength -> "GET / HTTP/1.1\n" +
-                        "HoSt: " + SOURCE_CLUSTER_NAME + "\n" +
-                        "content-length: " + contentLength + "\n");
+                contentLength -> "GET / HTTP/1.1\r\n" +
+                        "HoSt: " + SOURCE_CLUSTER_NAME + "\r\n" +
+                        "content-length: " + contentLength + "\r\n");
     }
 
     private void runRandomPayloadWithTransformer(HttpJsonTransformingConsumer transformingHandler,
@@ -82,16 +82,17 @@ public void testMalformedPayloadIsPassedThrough() throws Exception {
         final var dummyAggregatedResponse = new AggregatedTransformedResponse(12, null, null,
                 null, AggregatedTransformedResponse.HttpRequestTransformationStatus.COMPLETED);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
+        var httpBasicAuthTransformer = new StaticAuthTransformerFactory("Basic YWRtaW46YWRtaW4=");
         var transformingHandler = new HttpJsonTransformingConsumer(
-                TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME, "Basic YWRtaW46YWRtaW4="),
-                testPacketCapture, "TEST");
+                TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME),
+                httpBasicAuthTransformer, testPacketCapture, "TEST");
 
         runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture,
-                contentLength -> "GET / HTTP/1.1\n" +
-                        "HoSt: " + SOURCE_CLUSTER_NAME + "\n" +
-                        "content-type: application/json\n" +
-                        "content-length: " + contentLength + "\n" +
-                        "authorization: Basic YWRtaW46YWRtaW4=\n");
+                contentLength -> "GET / HTTP/1.1\r\n" +
+                        "HoSt: " + SOURCE_CLUSTER_NAME + "\r\n" +
+                        "content-type: application/json\r\n" +
+                        "content-length: " + contentLength + "\r\n" +
+                        "authorization: Basic YWRtaW46YWRtaW4=\r\n");
     }
 
     /**
@@ -107,8 +108,8 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce
                 null, AggregatedTransformedResponse.HttpRequestTransformationStatus.COMPLETED);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
         var transformingHandler = new HttpJsonTransformingConsumer(
-                TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME, null),
-                testPacketCapture, "TEST");
+                TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME),
+                null, testPacketCapture, "TEST");
 
         Random r = new Random(2);
         var stringParts = IntStream.range(0, 1).mapToObj(i-> TestUtils.makeRandomString(r, 10)).map(o->(String)o)
@@ -118,10 +119,10 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce
                 TestUtils.chainedDualWriteHeaderAndPayloadParts(transformingHandler,
                         stringParts,
                         referenceStringBuilder,
-                        contentLength -> "PUT /foo HTTP/1.1\n" +
-                                "HoSt: " + SOURCE_CLUSTER_NAME + "\n" +
-                                "content-type: application/json\n" +
-                                "content-length: " + contentLength + "\n"
+                        contentLength -> "PUT /foo HTTP/1.1\r\n" +
+                                "HoSt: " + SOURCE_CLUSTER_NAME + "\r\n" +
+                                "content-type: application/json\r\n" +
+                                "content-length: " + contentLength + "\r\n"
                 );
 
         var finalizationFuture = allConsumesFuture.thenCompose(v->transformingHandler.finalizeRequest(),
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java
index 2c12e12af..319f0934a 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java
@@ -5,25 +5,17 @@
 import io.netty.handler.codec.http.DefaultHttpHeaders;
 import io.netty.util.ResourceLeakDetector;
 import lombok.extern.slf4j.Slf4j;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer;
-import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
-import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
-import org.opensearch.migrations.transform.JoltJsonTransformBuilder;
-import org.opensearch.migrations.transform.JoltJsonTransformer;
-import org.opensearch.migrations.transform.JsonTransformer;
-
-import java.time.Duration;
+import org.opensearch.migrations.transform.JsonJoltTransformBuilder;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
+
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -42,7 +34,7 @@ public static Stream<List<Object>> expandList(Stream<List<Object>> stream, List
     public static Arguments[] makeCombinations() {
         List<Object> allBools = List.of(true, false);
         Stream<List<Object>> seedLists = allBools.stream().map(b->List.of(b));
-        return expandList(expandList(seedLists, allBools), allBools)
+        return expandList(seedLists, allBools)
                .map(list->Arguments.of(list.toArray(Object[]::new)))
                 .toArray(Arguments[]::new);
     }
@@ -52,10 +44,10 @@ public static Arguments[] makeCombinations() {
     public void testSimplePayloadTransform(boolean doGzip, boolean doChunked) throws Exception {
         ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
 
-        var transformerBuilder = JoltJsonTransformer.newBuilder();
+        var transformerBuilder = JsonJoltTransformer.newBuilder();
 
-        if (doGzip) { transformerBuilder.addCannedOperation(JoltJsonTransformBuilder.CANNED_OPERATION.ADD_GZIP); }
-        if (doChunked) { transformerBuilder.addCannedOperation(JoltJsonTransformBuilder.CANNED_OPERATION.MAKE_CHUNKED); }
+        if (doGzip) { transformerBuilder.addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.ADD_GZIP); }
+        if (doChunked) { transformerBuilder.addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.MAKE_CHUNKED); }
 
         Random r = new Random(2);
         var stringParts = IntStream.range(0, 1)
@@ -68,45 +60,11 @@ public void testSimplePayloadTransform(boolean doGzip, boolean doChunked) throws
         expectedRequestHeaders.add("host", "localhost");
         expectedRequestHeaders.add("Content-Length", "46");
 
-        runPipelineAndValidate(transformerBuilder.build(), null, stringParts,
-                expectedRequestHeaders,
+        TestUtils.runPipelineAndValidate(transformerBuilder.build(), null,null,
+                stringParts, expectedRequestHeaders,
                 referenceStringBuilder -> TestUtils.resolveReferenceString(referenceStringBuilder));
     }
 
-    private static void runPipelineAndValidate(JsonTransformer transformer,
-                                               String extraHeaders,
-                                               List<String> stringParts,
-                                               DefaultHttpHeaders expectedRequestHeaders,
-                                               Function<StringBuilder,String> expectedOutputGenerator) throws Exception {
-        var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100),
-                new AggregatedRawResponse(-1, Duration.ZERO, new ArrayList<>(), new ArrayList<>()));
-        var transformingHandler = new HttpJsonTransformingConsumer(transformer, testPacketCapture, "TEST");
-
-        var contentLength = stringParts.stream().mapToInt(s->s.length()).sum();
-        var headerString = "GET / HTTP/1.1\n" +
-                "host: localhost\n" +
-                (extraHeaders == null ? "" : extraHeaders) +
-                "content-length: " + contentLength + "\n\n";
-        var referenceStringBuilder = new StringBuilder();
-        var allConsumesFuture = TestUtils.chainedWriteHeadersAndDualWritePayloadParts(transformingHandler,
-                        stringParts, referenceStringBuilder, headerString);
-
-        var innermostFinalizeCallCount = new AtomicInteger();
-        DiagnosticTrackableCompletableFuture<String,AggregatedTransformedResponse> finalizationFuture =
-                allConsumesFuture.thenCompose(v -> transformingHandler.finalizeRequest(),
-                        ()->"PayloadRepackingTest.runPipelineAndValidate.allConsumeFuture");
-        finalizationFuture.map(f->f.whenComplete((aggregatedRawResponse, t) -> {
-            Assertions.assertNull(t);
-            Assertions.assertNotNull(aggregatedRawResponse);
-            // do nothing but check connectivity between the layers in the bottom most handler
-            innermostFinalizeCallCount.incrementAndGet();
-        }), ()->"PayloadRepackingTest.runPipelineAndValidate.assertCheck");
-        finalizationFuture.get();
-
-        TestUtils.verifyCapturedResponseMatchesExpectedPayload(testPacketCapture.getBytesCaptured(),
-                expectedRequestHeaders, expectedOutputGenerator.apply(referenceStringBuilder));
-    }
-
     String simplePayloadTransform = "" +
             "  {\n" +
             "    \"operation\": \"shift\",\n" +
@@ -130,12 +88,12 @@ private static void runPipelineAndValidate(JsonTransformer transformer,
 
     @Test
     public void testJsonPayloadTransformation() throws Exception {
-        var transformerBuilder = JoltJsonTransformer.newBuilder();
+        var transformerBuilder = JsonJoltTransformer.newBuilder();
 
         ObjectMapper mapper = new ObjectMapper();
         var simpleTransform = mapper.readValue(simplePayloadTransform,
                 new TypeReference<LinkedHashMap<String, Object>>(){});
-        transformerBuilder.addCannedOperation(JoltJsonTransformBuilder.CANNED_OPERATION.PASS_THRU);
+        transformerBuilder.addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.PASS_THRU);
         transformerBuilder.addOperationObject(simpleTransform);
 
         var jsonPayload = "{\"top\": {\"A\": 1,\"B\": 2}}";
@@ -147,8 +105,8 @@ public void testJsonPayloadTransformation() throws Exception {
         expectedRequestHeaders.add("content-type", "application/json; charset=UTF-8");
         expectedRequestHeaders.add("Content-Length", "55");
 
-        runPipelineAndValidate(transformerBuilder.build(), extraHeaders, List.of(jsonPayload),
-                expectedRequestHeaders,
+        TestUtils.runPipelineAndValidate(transformerBuilder.build(), null,
+                extraHeaders, List.of(jsonPayload), expectedRequestHeaders,
                 x -> "{\"top\":[{\"Name\":\"A\",\"Value\":1},{\"Name\":\"B\",\"Value\":2}]}");
     }
 }
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java
new file mode 100644
index 000000000..7309065e9
--- /dev/null
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java
@@ -0,0 +1,64 @@
+package org.opensearch.migrations.replay;
+
+import io.netty.buffer.Unpooled;
+import io.netty.handler.codec.base64.Base64;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.util.ResourceLeakDetector;
+import org.junit.jupiter.api.Test;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Random;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class SigV4SigningTransformationTest {
+    private static String decodeString(String s) {
+        var decodedByteBuf = Base64.decode(Unpooled.wrappedBuffer(s.getBytes(StandardCharsets.UTF_8)));
+        return decodedByteBuf.toString(StandardCharsets.UTF_8);
+    }
+
+    private static class MockCredentialsProvider implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create(decodeString("QUtJQUlPU0ZPRE5ON0VYQU1QTEUK"),
+                    decodeString("d0phbHJYVXRuRkVNSS9LN01ERU5HL2JQeFJmaUNZRVhBTVBMRUtFWQo="));
+        }
+    }
+
+    @Test
+    public void testSignatureProperlyApplied() throws Exception {
+        ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
+
+        Random r = new Random(2);
+        var stringParts = IntStream.range(0, 1)
+                .mapToObj(i -> TestUtils.makeRandomString(r, 64))
+                .map(o -> (String) o)
+                .collect(Collectors.toList());
+
+        var mockCredentialsProvider = new MockCredentialsProvider();
+        DefaultHttpHeaders expectedRequestHeaders = new DefaultHttpHeaders();
+        // netty's decompressor and aggregator remove some header values (& add others)
+        expectedRequestHeaders.add("host", "localhost");
+        expectedRequestHeaders.add("Content-Length".toLowerCase(), "46");
+        expectedRequestHeaders.add("Authorization",
+                "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/19700101/us-east-1/es/aws4_request, " +
+                        "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " +
+                        "Signature=4cb1c423e6fe61216fbaa11398260af7f8daa85e74cd41428711e4df5cd70c97");
+        expectedRequestHeaders.add("x-amz-content-sha256",
+                        "fc0e8e9a1f7697f510bfdd4d55b8612df8a0140b4210967efd87ee9cb7104362");
+        expectedRequestHeaders.add("X-Amz-Date", "19700101T000000Z");
+
+        TestUtils.runPipelineAndValidate(JsonJoltTransformer.newBuilder().build(),
+                msg -> new SigV4Signer(mockCredentialsProvider, "es", "us-east-1", "https",
+                        () ->  Clock.fixed(Instant.EPOCH, ZoneOffset.UTC)),
+                null, stringParts, expectedRequestHeaders,
+                referenceStringBuilder -> TestUtils.resolveReferenceString(referenceStringBuilder));
+    }
+}
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java
index 5616e8d48..6b29859ca 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java
@@ -14,16 +14,21 @@
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Assertions;
 import org.opensearch.migrations.replay.datahandlers.IPacketConsumer;
+import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer;
 import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture;
-import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture;
+import org.opensearch.migrations.transform.IAuthTransformerFactory;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.AbstractMap;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
@@ -73,7 +78,7 @@ static DiagnosticTrackableCompletableFuture<String,Void> chainedWriteHeadersAndD
                                           StringBuilder referenceStringAccumulator,
                                           Function<Integer, String> headersGenerator) {
         var contentLength = stringParts.stream().mapToInt(s->s.length()).sum();
-        String headers = headersGenerator.apply(contentLength) + "\n";
+        String headers = headersGenerator.apply(contentLength) + "\r\n";
         referenceStringAccumulator.append(headers);
         return chainedWriteHeadersAndDualWritePayloadParts(packetConsumer, stringParts, referenceStringAccumulator, headers);
     }
@@ -116,4 +121,40 @@ private static String getStringFromContent(FullHttpRequest fullRequest) throws I
             return new String(baos.toByteArray(), StandardCharsets.UTF_8);
         }
     }
+
+    static void runPipelineAndValidate(IJsonTransformer transformer,
+                                       IAuthTransformerFactory authTransformer,
+                                       String extraHeaders,
+                                       List<String> stringParts,
+                                       DefaultHttpHeaders expectedRequestHeaders,
+                                       Function<StringBuilder, String> expectedOutputGenerator) throws Exception {
+        var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100),
+                new AggregatedRawResponse(-1, Duration.ZERO, new ArrayList<>(), new ArrayList<>()));
+        var transformingHandler = new HttpJsonTransformingConsumer(transformer, authTransformer, testPacketCapture,
+                "TEST");
+
+        var contentLength = stringParts.stream().mapToInt(s->s.length()).sum();
+        var headerString = "GET / HTTP/1.1\r\n" +
+                "host: localhost\r\n" +
+                (extraHeaders == null ? "" : extraHeaders) +
+                "content-length: " + contentLength + "\r\n\r\n";
+        var referenceStringBuilder = new StringBuilder();
+        var allConsumesFuture = chainedWriteHeadersAndDualWritePayloadParts(transformingHandler,
+                        stringParts, referenceStringBuilder, headerString);
+
+        var innermostFinalizeCallCount = new AtomicInteger();
+        DiagnosticTrackableCompletableFuture<String,AggregatedTransformedResponse> finalizationFuture =
+                allConsumesFuture.thenCompose(v -> transformingHandler.finalizeRequest(),
+                        ()->"PayloadRepackingTest.runPipelineAndValidate.allConsumeFuture");
+        finalizationFuture.map(f->f.whenComplete((aggregatedRawResponse, t) -> {
+            Assertions.assertNull(t);
+            Assertions.assertNotNull(aggregatedRawResponse);
+            // do nothing but check connectivity between the layers in the bottom most handler
+            innermostFinalizeCallCount.incrementAndGet();
+        }), ()->"PayloadRepackingTest.runPipelineAndValidate.assertCheck");
+        finalizationFuture.get();
+
+        verifyCapturedResponseMatchesExpectedPayload(testPacketCapture.getBytesCaptured(),
+                expectedRequestHeaders, expectedOutputGenerator.apply(referenceStringBuilder));
+    }
 }
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java
index 672e23a3c..96ae24621 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java
@@ -5,13 +5,12 @@
 import org.opensearch.migrations.replay.AggregatedRawResponse;
 import org.opensearch.migrations.replay.AggregatedTransformedResponse;
 import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler;
-import org.opensearch.migrations.transform.CompositeJsonTransformer;
-import org.opensearch.migrations.transform.JoltJsonTransformer;
-import org.opensearch.migrations.transform.JsonTransformer;
+import org.opensearch.migrations.transform.JsonCompositeTransformer;
+import org.opensearch.migrations.transform.JsonJoltTransformer;
+import org.opensearch.migrations.transform.IJsonTransformer;
 
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 
@@ -20,8 +19,8 @@ class HttpJsonTransformingConsumerTest {
     public void testPassThroughSinglePacketPost() throws Exception {
         final var dummyAggregatedResponse = new AggregatedRawResponse(17, null, null,null);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
-        var transformingHandler = new HttpJsonTransformingConsumer(JoltJsonTransformer.newBuilder().build(),
-                testPacketCapture, "TEST");
+        var transformingHandler = new HttpJsonTransformingConsumer(JsonJoltTransformer.newBuilder().build(),
+                null, testPacketCapture, "TEST");
         byte[] testBytes;
         try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream(
                 "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) {
@@ -41,10 +40,10 @@ public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Ex
         final var dummyAggregatedResponse = new AggregatedRawResponse(17, null, null,null);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
         var transformingHandler = new HttpJsonTransformingConsumer(
-                JoltJsonTransformer.newBuilder()
+                JsonJoltTransformer.newBuilder()
                         .addHostSwitchOperation("test.domain")
                         .build(),
-                testPacketCapture, "TEST");
+                null, testPacketCapture, "TEST");
         byte[] testBytes;
         try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream(
                 "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) {
@@ -66,7 +65,7 @@ public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Ex
     public void testPartialBodyThrowsAndIsRedriven() throws Exception {
         final var dummyAggregatedResponse = new AggregatedRawResponse(17, null, null, null);
         var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse);
-        var complexTransformer = new CompositeJsonTransformer(new JsonTransformer() {
+        var complexTransformer = new JsonCompositeTransformer(new IJsonTransformer() {
             @Override
             public Object transformJson(Object incomingJson) {
                 // just walk everything - that's enough to touch the payload and throw
@@ -81,7 +80,8 @@ private void walkMaps(Object o) {
                 }
             }
         });
-        var transformingHandler = new HttpJsonTransformingConsumer(complexTransformer,  testPacketCapture, "TEST");
+        var transformingHandler =
+                new HttpJsonTransformingConsumer(complexTransformer, null, testPacketCapture, "TEST");
         byte[] testBytes;
         try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream(
                 "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) {
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java
index ef414b76e..a7a960fd7 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java
@@ -21,13 +21,13 @@ public JsonTransformerTest() {
     }
 
     private Map<String, Object> parseStringAsJson(String jsonStr) throws JsonProcessingException {
-        return mapper.readValue(jsonStr, JoltJsonTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE);
+        return mapper.readValue(jsonStr, JsonJoltTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE);
     }
 
     @SneakyThrows
     private Map<String, Object> parseSampleRequestFromResource(String path) {
-        try (InputStream inputStream = JoltJsonTransformBuilder.class.getResourceAsStream("/requests/"+path)) {
-            return mapper.readValue(inputStream, JoltJsonTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE);
+        try (InputStream inputStream = JsonJoltTransformBuilder.class.getResourceAsStream("/requests/"+path)) {
+            return mapper.readValue(inputStream, JsonJoltTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE);
         }
     }
 
@@ -41,8 +41,8 @@ private String emitJson(Object transformedDocument) throws JsonProcessingExcepti
     public void testSimpleTransform() throws JsonProcessingException {
         final String TEST_DOCUMENT = "{\"Hello\":\"world\"}";
         var documentJson = parseStringAsJson(TEST_DOCUMENT);
-        var transformer = JoltJsonTransformer.newBuilder()
-                .addCannedOperation(JoltJsonTransformBuilder.CANNED_OPERATION.PASS_THRU)
+        var transformer = JsonJoltTransformer.newBuilder()
+                .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.PASS_THRU)
                 .build();
         var transformedDocument = transformer.transformJson(documentJson);
         var finalOutputStr = emitJson(transformedDocument);
@@ -54,7 +54,7 @@ public void testSimpleTransform() throws JsonProcessingException {
     public void testHttpTransform() throws IOException {
         var testResourceName = "parsed/post_formUrlEncoded_withFixedLength.json";
         final var documentJson = parseSampleRequestFromResource(testResourceName);
-        var transformer = JoltJsonTransformer.newBuilder()
+        var transformer = JsonJoltTransformer.newBuilder()
                 .addHostSwitchOperation(DUMMY_HOSTNAME_TEST_STRING)
                 .build();
         var transformedDocument = transformer.transformJson(documentJson);
diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java
index 562db19ea..37339309b 100644
--- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java
+++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java
@@ -61,7 +61,7 @@ private static void transformAndVerifyResult(Object json, String expectedValueSo
         Assertions.assertEquals(expectedValue, jsonAsStr);
     }
 
-    static JsonTransformer getJsonTransformer() {
-        return new TypeMappingJsonTransformer();
+    static IJsonTransformer getJsonTransformer() {
+        return new JsonTypeMappingTransformer();
     }
 }
diff --git a/TrafficReplayer/gradle.properties b/TrafficReplayer/gradle.properties
deleted file mode 100644
index 29645a808..000000000
--- a/TrafficReplayer/gradle.properties
+++ /dev/null
@@ -1 +0,0 @@
-version = 0.1.0
\ No newline at end of file
diff --git a/TrafficReplayer/gradle/wrapper/gradle-wrapper.jar b/TrafficReplayer/gradle/wrapper/gradle-wrapper.jar
deleted file mode 100644
index e708b1c02..000000000
Binary files a/TrafficReplayer/gradle/wrapper/gradle-wrapper.jar and /dev/null differ
diff --git a/TrafficReplayer/gradle/wrapper/gradle-wrapper.properties b/TrafficReplayer/gradle/wrapper/gradle-wrapper.properties
deleted file mode 100644
index e1bef7e87..000000000
--- a/TrafficReplayer/gradle/wrapper/gradle-wrapper.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
diff --git a/TrafficReplayer/gradlew b/TrafficReplayer/gradlew
deleted file mode 100755
index 4f906e0c8..000000000
--- a/TrafficReplayer/gradlew
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env sh
-
-#
-# Copyright 2015 the original author or authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=`expr $i + 1`
-    done
-    case $i in
-        0) set -- ;;
-        1) set -- "$args0" ;;
-        2) set -- "$args0" "$args1" ;;
-        3) set -- "$args0" "$args1" "$args2" ;;
-        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=`save "$@"`
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-exec "$JAVACMD" "$@"
diff --git a/TrafficReplayer/gradlew.bat b/TrafficReplayer/gradlew.bat
deleted file mode 100644
index ac1b06f93..000000000
--- a/TrafficReplayer/gradlew.bat
+++ /dev/null
@@ -1,89 +0,0 @@
-@rem
-@rem Copyright 2015 the original author or authors.
-@rem
-@rem Licensed under the Apache License, Version 2.0 (the "License");
-@rem you may not use this file except in compliance with the License.
-@rem You may obtain a copy of the License at
-@rem
-@rem      https://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Resolve any "." and ".." in APP_HOME to make it shorter.
-for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
diff --git a/TrafficReplayer/pull_then_poll_cw_logs.py b/TrafficReplayer/pull_then_poll_cw_logs.py
deleted file mode 100755
index 2bdf53c28..000000000
--- a/TrafficReplayer/pull_then_poll_cw_logs.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-import os
-import time
-import sys
-
-import boto3
-
-TRAFFIC_LOG_GROUP = os.environ["CW_LOG_GROUP_NAME"]
-TRAFFIC_LOG_STREAM = os.environ["CW_LOG_STREAM_NAME"]
-
-
-def main():
-    # This is intended to assume the IAM role of the fargate container.
-    logs_client = boto3.client('logs')
-
-    # Continuously get events and add them to our final list until we see a repeat of the "nextForwardToken"
-    # Then script will start checking periodically
-    print(f"Pulling CW events from {TRAFFIC_LOG_GROUP}:{TRAFFIC_LOG_STREAM}...", file=sys.stderr)
-    current_response = logs_client.get_log_events(logGroupName=TRAFFIC_LOG_GROUP,
-                                                  logStreamName=TRAFFIC_LOG_STREAM, startFromHead=True)
-    current_events_raw = current_response["events"]
-    for event in current_events_raw:
-        print(event["message"])
-    current_token = current_response["nextForwardToken"]
-
-    next_response = logs_client.get_log_events(logGroupName=TRAFFIC_LOG_GROUP,
-                                               logStreamName=TRAFFIC_LOG_STREAM,
-                                               startFromHead=True, nextToken=current_token)
-    next_token = next_response["nextForwardToken"]
-
-    # Now that the first CW log event was logged.
-    # We start checking for new events every now and then, and print new ones, if available.
-
-    while True:
-        next_response = logs_client.get_log_events(logGroupName=TRAFFIC_LOG_GROUP,
-                                                   logStreamName=TRAFFIC_LOG_STREAM,
-                                                   startFromHead=False, nextToken=current_token)
-        next_token = next_response["nextForwardToken"]
-
-        if current_token != next_token:
-
-            print("Found new events", file=sys.stderr)
-            current_response = next_response
-            current_events_raw = current_response["events"]
-            for event in current_events_raw:
-                print(event["message"])
-            current_token = current_response["nextForwardToken"]
-
-            next_response = logs_client.get_log_events(logGroupName=TRAFFIC_LOG_GROUP,
-                                                       logStreamName=TRAFFIC_LOG_STREAM,
-                                                       startFromHead=False, nextToken=current_token)
-            next_token = next_response["nextForwardToken"]
-
-            print("Sleeping for 10 seconds then checking if any additional events are available", file=sys.stderr)
-
-        time.sleep(10)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/cluster_migration_core/requirements.txt b/cluster_migration_core/requirements.txt
deleted file mode 100644
index ecf975e2f..000000000
--- a/cluster_migration_core/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
--e .
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/.gitignore b/deployment/cdk/opensearch-service-migration/.gitignore
index 0b9518478..c681fce7c 100644
--- a/deployment/cdk/opensearch-service-migration/.gitignore
+++ b/deployment/cdk/opensearch-service-migration/.gitignore
@@ -3,7 +3,6 @@
 *.d.ts
 node_modules
 cdk.context.json
-cdkOutput.json
 
 # CDK asset staging directory
 .cdk.staging
diff --git a/deployment/cdk/opensearch-service-migration/README.md b/deployment/cdk/opensearch-service-migration/README.md
index 84ce25dc8..8b593b79e 100644
--- a/deployment/cdk/opensearch-service-migration/README.md
+++ b/deployment/cdk/opensearch-service-migration/README.md
@@ -106,6 +106,7 @@ Additional context on some of these options, can also be found in the Domain con
 | domainRemovalPolicy                             | false    | string       | "RETAIN"                                                                                                                                                                                                                       | Policy to apply when the domain is removed from the CloudFormation stack                                                                                                                                                                                     |
 | mskARN (Not currently available)                | false    | string       | `"arn:aws:kafka:us-east-2:123456789123:cluster/msk-cluster-test/81fbae45-5d25-44bb-aff0-108e71cc079b-7"`                                                                                                                       | Supply an existing MSK cluster ARN to use. **NOTE** As MSK is using an L1 construct this is not currently available for use                                                                                                                                  |
 | mskEnablePublicEndpoints                        | false    | boolean      | true                                                                                                                                                                                                                           | Specify if public endpoints should be enabled on the MSK cluster                                                                                                                                                                                             |
+| mskBrokerNodeCount                              | false    | number       | 2                                                                                                                                                                                                                              | The number of broker nodes to be used by the MSK cluster                                                                                                                                                                                                     |
 
 A template `cdk.context.json` to be used to fill in these values is below:
 ```
@@ -143,7 +144,8 @@ A template `cdk.context.json` to be used to fill in these values is below:
   "openAccessPolicyEnabled": "",
   "domainRemovalPolicy": "",
   "mskARN": "",
-  "mskEnablePublicEndpoints": ""
+  "mskEnablePublicEndpoints": "",
+  "mskBrokerNodeCount": ""
 }
 
 ```
diff --git a/deployment/cdk/opensearch-service-migration/bin/app.ts b/deployment/cdk/opensearch-service-migration/bin/app.ts
index d7199924e..df9f43091 100644
--- a/deployment/cdk/opensearch-service-migration/bin/app.ts
+++ b/deployment/cdk/opensearch-service-migration/bin/app.ts
@@ -7,11 +7,23 @@ const app = new App();
 const account = process.env.CDK_DEFAULT_ACCOUNT
 const region = process.env.CDK_DEFAULT_REGION
 const stage = process.env.CDK_DEPLOYMENT_STAGE
+let copilotAppName = process.env.COPILOT_APP_NAME
+let copilotEnvName = process.env.COPILOT_ENV_NAME
 if (!stage) {
     throw new Error("Required environment variable CDK_DEPLOYMENT_STAGE has not been set (i.e. dev, gamma, PROD)")
 }
+if (!copilotAppName) {
+    console.log("COPILOT_APP_NAME has not been set, defaulting to 'migration-copilot' for stack export identifier")
+    copilotAppName = "migration-copilot"
+}
+if (!copilotEnvName) {
+    console.log(`COPILOT_ENV_NAME has not been set, defaulting to CDK stage: ${stage} for stack export identifier`)
+    copilotEnvName = stage
+}
 
 new StackComposer(app, {
     env: { account: account, region: region },
-    stage: stage
+    stage: stage,
+    copilotAppName: copilotAppName,
+    copilotEnvName: copilotEnvName
 });
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/default-values.json b/deployment/cdk/opensearch-service-migration/default-values.json
index 15790ac2e..d968406aa 100644
--- a/deployment/cdk/opensearch-service-migration/default-values.json
+++ b/deployment/cdk/opensearch-service-migration/default-values.json
@@ -1,6 +1,7 @@
 {
-  "engineVersion": "OS_2.5",
+  "engineVersion": "OS_2.7",
   "domainName": "os-service-domain",
+  "tlsSecurityPolicy": "TLS_1_2",
   "enforceHTTPS": true,
   "nodeToNodeEncryptionEnabled": true,
   "encryptionAtRestEnabled": true
diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts
index 42bebed74..90a1cbdf1 100644
--- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts
@@ -22,6 +22,7 @@ export interface migrationStackProps extends StackPropsExt {
     // Future support needed to allow importing an existing MSK cluster
     readonly mskARN?: string,
     readonly mskEnablePublicEndpoints?: boolean
+    readonly mskBrokerNodeCount?: number
 }
 
 
@@ -57,7 +58,7 @@ export class MigrationAssistanceStack extends Stack {
         const mskCluster = new CfnCluster(this, 'migrationMSKCluster', {
             clusterName: 'migration-msk-cluster',
             kafkaVersion: '2.8.1',
-            numberOfBrokerNodes: 2,
+            numberOfBrokerNodes: props.mskBrokerNodeCount ? props.mskBrokerNodeCount : 2,
             brokerNodeGroupInfo: {
                 instanceType: 'kafka.m5.large',
                 clientSubnets: props.vpc.selectSubnets({subnetType: SubnetType.PUBLIC}).subnetIds,
@@ -104,7 +105,7 @@ export class MigrationAssistanceStack extends Stack {
 
         const comparatorSQLiteSG = new SecurityGroup(this, 'comparatorSQLiteSG', {
             vpc: props.vpc,
-            allowAllOutbound: true,
+            allowAllOutbound: false,
         });
         comparatorSQLiteSG.addIngressRule(comparatorSQLiteSG, Port.allTraffic());
 
@@ -114,22 +115,16 @@ export class MigrationAssistanceStack extends Stack {
             securityGroup: comparatorSQLiteSG
         });
 
-        // Creates a security group with open access via ssh
-        const oinoSecurityGroup = new SecurityGroup(this, 'orchestratorSecurityGroup', {
+        const replayerOutputSG = new SecurityGroup(this, 'replayerOutputSG', {
             vpc: props.vpc,
-            allowAllOutbound: true,
+            allowAllOutbound: false,
         });
-        oinoSecurityGroup.addIngressRule(Peer.anyIpv4(), Port.tcp(22));
+        replayerOutputSG.addIngressRule(replayerOutputSG, Port.allTraffic());
 
-        // Create EC2 instance for analysis of cluster in VPC
-        const oino = new Instance(this, "orchestratorEC2Instance", {
+        // Create an EFS file system for Traffic Replayer output
+        const replayerOutputEFS = new FileSystem(this, 'replayerOutputEFS', {
             vpc: props.vpc,
-            vpcSubnets: { subnetType: SubnetType.PUBLIC },
-            instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.MICRO),
-            machineImage: MachineImage.latestAmazonLinux2(),
-            securityGroup: oinoSecurityGroup,
-            // Manually created for now, to be automated in future
-            //keyName: "es-node-key"
+            securityGroup: replayerOutputSG
         });
 
         let publicSubnetString = props.vpc.publicSubnets.map(_ => _.subnetId).join(",")
@@ -138,7 +133,9 @@ export class MigrationAssistanceStack extends Stack {
             `export MIGRATION_VPC_ID=${props.vpc.vpcId}`,
             `export MIGRATION_CAPTURE_MSK_SG_ID=${mskSecurityGroup.securityGroupId}`,
             `export MIGRATION_COMPARATOR_EFS_ID=${comparatorSQLiteEFS.fileSystemId}`,
-            `export MIGRATION_COMPARATOR_EFS_SG_ID=${comparatorSQLiteSG.securityGroupId}`]
+            `export MIGRATION_COMPARATOR_EFS_SG_ID=${comparatorSQLiteSG.securityGroupId}`,
+            `export MIGRATION_REPLAYER_OUTPUT_EFS_ID=${replayerOutputEFS.fileSystemId}`,
+            `export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=${replayerOutputSG.securityGroupId}`]
         if (publicSubnetString) exports.push(`export MIGRATION_PUBLIC_SUBNETS=${publicSubnetString}`)
         if (privateSubnetString) exports.push(`export MIGRATION_PRIVATE_SUBNETS=${privateSubnetString}`)
 
@@ -146,6 +143,12 @@ export class MigrationAssistanceStack extends Stack {
             value: exports.join(";"),
             description: 'Exported migration resource values created by CDK that are needed by Copilot container deployments',
         });
+        // Create export of MSK cluster ARN for Copilot stacks to use
+        new CfnOutput(this, 'migrationMSKClusterARN', {
+            value: mskCluster.attrArn,
+            exportName: `${props.copilotAppName}-${props.copilotEnvName}-msk-cluster-arn`,
+            description: 'Migration MSK Cluster ARN'
+        });
 
     }
 }
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts b/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts
index b343c94ca..ed83d3e4d 100644
--- a/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts
@@ -27,22 +27,23 @@ export class MSKUtilityStack extends Stack {
                 actions: ["lambda:InvokeFunction"],
                 resources: ["*"]
             })
-            const mskUpdateConnectivityStatement = new PolicyStatement({
+            // Updating connectivity for an MSK cluster requires some VPC permissions
+            // (https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonmanagedstreamingforapachekafka.html#amazonmanagedstreamingforapachekafka-cluster)
+            const describeVPCStatement = new PolicyStatement( {
                 effect: Effect.ALLOW,
                 actions: ["ec2:DescribeSubnets",
-                    "ec2:DescribeVpcs",
-                    "ec2:DescribeSecurityGroups",
-                    "ec2:DescribeRouteTables",
-                    "ec2:DescribeVpcEndpoints",
-                    "ec2:DescribeVpcAttribute",
-                    "ec2:DescribeNetworkAcls",
-                    "kafka:UpdateConnectivity",
+                    "ec2:DescribeRouteTables"],
+                resources: ["*"]
+            })
+            const mskUpdateConnectivityStatement = new PolicyStatement({
+                effect: Effect.ALLOW,
+                actions: ["kafka:UpdateConnectivity",
                     "kafka:DescribeClusterV2",
                     "kafka:GetBootstrapBrokers"],
-                resources: ["*"]
+                resources: [props.mskARN]
             })
             const lambdaExecDocument = new PolicyDocument({
-                statements: [lambdaInvokeStatement, mskUpdateConnectivityStatement]
+                statements: [lambdaInvokeStatement, describeVPCStatement, mskUpdateConnectivityStatement]
             })
 
             const lambdaExecRole = new Role(this, 'mskAccessLambda', {
@@ -95,7 +96,7 @@ export class MSKUtilityStack extends Stack {
         }
         else {
             const mskGetBrokersCustomResource = getBrokersCustomResource(this, props.vpc, props.mskARN)
-            brokerEndpointsOutput = mskGetBrokersCustomResource.getResponseField("BootstrapBrokerStringSaslIam")
+            brokerEndpointsOutput = `export MIGRATION_KAFKA_BROKER_ENDPOINTS=${mskGetBrokersCustomResource.getResponseField("BootstrapBrokerStringSaslIam")}`
             //brokerEndpointsOutput = mskGetBrokersCustomResource.getResponseField("BootstrapBrokerStringPublicSaslIam")
         }
 
diff --git a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts
index 1ca7f7a65..3542bdd0a 100644
--- a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts
@@ -1,4 +1,4 @@
-import {Stack, StackProps} from "aws-cdk-lib";
+import {CfnOutput, Stack, StackProps} from "aws-cdk-lib";
 import {
     IpAddresses,
     ISecurityGroup,
@@ -71,24 +71,25 @@ export class NetworkStack extends Stack {
         }
 
         // Retrieve existing SGs to apply to VPC Domain endpoints
+        const securityGroups: ISecurityGroup[] = []
         if (props.vpcSecurityGroupIds) {
-            const securityGroups: ISecurityGroup[] = []
             for (let i = 0; i < props.vpcSecurityGroupIds.length; i++) {
                 securityGroups.push(SecurityGroup.fromLookupById(this, "domainSecurityGroup-" + i, props.vpcSecurityGroupIds[i]))
             }
-            this.domainSecurityGroups = securityGroups
-        }
-        // Create a default SG to allow open access to Domain within VPC. This should be further restricted for users
-        // who want limited access to their domain within the VPC
-        else {
-            const defaultSecurityGroup = new SecurityGroup(this, 'domainSecurityGroup', {
-                vpc: this.vpc,
-                allowAllOutbound: true,
-            });
-            defaultSecurityGroup.addIngressRule(Peer.ipv4('0.0.0.0/0'), Port.allTcp());
-            defaultSecurityGroup.addIngressRule(defaultSecurityGroup, Port.allTraffic());
-            this.domainSecurityGroups = [defaultSecurityGroup]
         }
+        // Create a default SG which only allows members of this SG to access the Domain endpoints
+        const defaultSecurityGroup = new SecurityGroup(this, 'domainMigrationAccessSG', {
+            vpc: this.vpc,
+            allowAllOutbound: false,
+        });
+        defaultSecurityGroup.addIngressRule(defaultSecurityGroup, Port.allTraffic());
+        securityGroups.push(defaultSecurityGroup)
+        this.domainSecurityGroups = securityGroups
+
+        new CfnOutput(this, 'CopilotDomainSGExports', {
+            value: `export MIGRATION_DOMAIN_SG_ID=${defaultSecurityGroup.securityGroupId}`,
+            description: 'Domain Security Group created by CDK that is needed for Copilot container deployments',
+        });
 
     }
 }
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/lib/opensearch-service-domain-cdk-stack.ts b/deployment/cdk/opensearch-service-migration/lib/opensearch-service-domain-cdk-stack.ts
index 4d30b073c..00ce37cdc 100644
--- a/deployment/cdk/opensearch-service-migration/lib/opensearch-service-domain-cdk-stack.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/opensearch-service-domain-cdk-stack.ts
@@ -5,7 +5,7 @@ import {CfnOutput, RemovalPolicy, SecretValue, Stack} from "aws-cdk-lib";
 import {IKey, Key} from "aws-cdk-lib/aws-kms";
 import {PolicyStatement} from "aws-cdk-lib/aws-iam";
 import {ILogGroup, LogGroup} from "aws-cdk-lib/aws-logs";
-import {Secret} from "aws-cdk-lib/aws-secretsmanager";
+import {ISecret, Secret} from "aws-cdk-lib/aws-secretsmanager";
 import {StackPropsExt} from "./stack-composer";
 
 
@@ -55,8 +55,9 @@ export class OpensearchServiceDomainCdkStack extends Stack {
     const earKmsKey: IKey|undefined = props.encryptionAtRestKmsKeyARN && props.encryptionAtRestEnabled ?
         Key.fromKeyArn(this, "earKey", props.encryptionAtRestKmsKeyARN) : undefined
 
-    let adminUserSecret: SecretValue|undefined = props.fineGrainedManagerUserSecretManagerKeyARN ?
-        Secret.fromSecretCompleteArn(this, "managerSecret", props.fineGrainedManagerUserSecretManagerKeyARN).secretValue : undefined
+    let adminUserSecret: ISecret|undefined = props.fineGrainedManagerUserSecretManagerKeyARN ?
+        Secret.fromSecretCompleteArn(this, "managerSecret", props.fineGrainedManagerUserSecretManagerKeyARN) : undefined
+
 
     const appLG: ILogGroup|undefined = props.appLogGroup && props.appLogEnabled ?
         LogGroup.fromLogGroupArn(this, "appLogGroup", props.appLogGroup) : undefined
@@ -67,7 +68,11 @@ export class OpensearchServiceDomainCdkStack extends Stack {
     // Enable demo mode setting
     if (props.enableDemoAdmin) {
       adminUserName = "admin"
-      adminUserSecret = SecretValue.unsafePlainText("Admin123!")
+      adminUserSecret = new Secret(this, "demoUserSecret", {
+        secretName: `demo-user-secret-${props.stage}-${props.env?.region}`,
+        // This is unsafe and strictly for ease of use in a demo mode setup
+        secretStringValue: SecretValue.unsafePlainText("Admin123!")
+      })
     }
     const zoneAwarenessConfig: ZoneAwarenessConfig|undefined = props.availabilityZoneCount ?
         {enabled: true, availabilityZoneCount: props.availabilityZoneCount} : undefined
@@ -88,7 +93,7 @@ export class OpensearchServiceDomainCdkStack extends Stack {
       fineGrainedAccessControl: {
         masterUserArn: props.fineGrainedManagerUserARN,
         masterUserName: adminUserName,
-        masterUserPassword: adminUserSecret
+        masterUserPassword: adminUserSecret ? adminUserSecret.secretValue : undefined
       },
       nodeToNodeEncryption: props.nodeToNodeEncryptionEnabled,
       encryptionAtRest: {
@@ -116,8 +121,17 @@ export class OpensearchServiceDomainCdkStack extends Stack {
 
     this.domainEndpoint = domain.domainEndpoint
 
+    const exports = [
+      `export MIGRATION_DOMAIN_ENDPOINT=${this.domainEndpoint}`
+    ]
+    if (domain.masterUserPassword && !adminUserSecret) {
+      console.log("A master user was configured without an existing Secrets Manager secret, will not export MIGRATION_DOMAIN_USER_AND_SECRET_ARN for Copilot")
+    }
+    else if (domain.masterUserPassword && adminUserSecret) {
+      exports.push(`export MIGRATION_DOMAIN_USER_AND_SECRET_ARN=${adminUserName} ${adminUserSecret.secretArn}`)
+    }
     new CfnOutput(this, 'CopilotDomainExports', {
-      value: `export MIGRATION_DOMAIN_ENDPOINT=${this.domainEndpoint}`,
+      value: exports.join(";"),
       description: 'Exported Domain resource values created by CDK that are needed by Copilot container deployments',
     });
   }
diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
index 426735680..e80ca52d9 100644
--- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
@@ -11,7 +11,9 @@ import {HistoricalCaptureStack} from "./historical-capture-stack";
 import {MSKUtilityStack} from "./msk-utility-stack";
 
 export interface StackPropsExt extends StackProps {
-    readonly stage: string
+    readonly stage: string,
+    readonly copilotAppName: string,
+    readonly copilotEnvName: string
 }
 
 export class StackComposer {
@@ -57,6 +59,7 @@ export class StackComposer {
         const migrationAssistanceEnabled = getContextForType('migrationAssistanceEnabled', 'boolean')
         const mskARN = getContextForType('mskARN', 'string')
         const mskEnablePublicEndpoints = getContextForType('mskEnablePublicEndpoints', 'boolean')
+        const mskBrokerNodeCount = getContextForType('mskBrokerNodeCount', 'number')
         const sourceClusterEndpoint = getContextForType('sourceClusterEndpoint', 'string')
         const historicalCaptureEnabled = getContextForType('historicalCaptureEnabled', 'boolean')
         const logstashConfigFilePath = getContextForType('logstashConfigFilePath', 'string')
@@ -167,6 +170,7 @@ export class StackComposer {
                 vpc: networkStack.vpc,
                 mskARN: mskARN,
                 mskEnablePublicEndpoints: mskEnablePublicEndpoints,
+                mskBrokerNodeCount: mskBrokerNodeCount,
                 stackName: `OSServiceMigrationCDKStack-${stage}-${region}`,
                 description: "This stack contains resources to assist migrating an OpenSearch Service domain",
                 ...props,
diff --git a/deployment/cdk/opensearch-service-migration/test/domain-cdk-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/domain-cdk-stack.test.ts
index 8156c137b..37142ad08 100644
--- a/deployment/cdk/opensearch-service-migration/test/domain-cdk-stack.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/domain-cdk-stack.test.ts
@@ -1,9 +1,9 @@
 import {App} from 'aws-cdk-lib';
 import {Template} from 'aws-cdk-lib/assertions';
-import {StackComposer} from "../lib/stack-composer";
 import * as testDefaultValues from "./default-values-test.json";
 import {OpensearchServiceDomainCdkStack} from "../lib/opensearch-service-domain-cdk-stack";
 import {NetworkStack} from "../lib/network-stack";
+import {createStackComposer} from "./test-utils";
 
 test('Test primary context options are mapped with standard data type', () => {
     // The cdk.context.json and default-values.json files allow multiple data types
@@ -47,9 +47,7 @@ test('Test primary context options are mapped with standard data type', () => {
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -95,9 +93,7 @@ test('Test primary context options are mapped with only string data type', () =>
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -123,9 +119,7 @@ test('Test alternate context options are mapped with standard data type', () =>
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -146,9 +140,7 @@ test('Test alternate context options are mapped with only string data type', ()
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -162,9 +154,7 @@ test('Test openAccessPolicy setting creates access policy when enabled', () => {
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -180,9 +170,7 @@ test('Test openAccessPolicy setting does not create access policy when disabled'
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -198,9 +186,7 @@ test('Test openAccessPolicy setting is mapped with string data type', () => {
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -214,9 +200,7 @@ test( 'Test default stack is created with default values when no context options
         context: {}
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const defaultValues: { [x: string]: (string); } = testDefaultValues
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
@@ -262,9 +246,7 @@ test( 'Test default stack is created when empty context options are provided for
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
diff --git a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
index 6fed47234..81ebebe30 100644
--- a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
@@ -2,6 +2,7 @@ import {App} from "aws-cdk-lib";
 import {StackComposer} from "../lib/stack-composer";
 import {NetworkStack} from "../lib/network-stack";
 import {Template} from "aws-cdk-lib/assertions";
+import {createStackComposer} from "./test-utils";
 
 test('Test vpcEnabled setting that is disabled does not create stack', () => {
     const app = new App({
@@ -10,9 +11,7 @@ test('Test vpcEnabled setting that is disabled does not create stack', () => {
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     openSearchStacks.stacks.forEach(function(stack) {
         expect(!(stack instanceof NetworkStack))
@@ -29,9 +28,7 @@ test('Test vpcEnabled setting that is enabled without existing resources creates
         }
     })
 
-    const openSearchStacks = new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const networkStack: NetworkStack = (openSearchStacks.stacks.filter((s) => s instanceof NetworkStack)[0]) as NetworkStack
     const networkTemplate = Template.fromStack(networkStack)
diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
index 553ff17eb..363cddee3 100644
--- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
@@ -2,8 +2,9 @@ import {App} from "aws-cdk-lib";
 import {StackComposer} from "../lib/stack-composer";
 import {Template} from "aws-cdk-lib/assertions";
 import {OpensearchServiceDomainCdkStack} from "../lib/opensearch-service-domain-cdk-stack";
+import {createStackComposer} from "./test-utils";
 
-test('Test missing domain name throws error', () => {
+test('Test empty string provided for a parameter which has a default value, uses the default value', () => {
 
     const app = new App({
         context: {
@@ -11,27 +12,13 @@ test('Test missing domain name throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-       env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks =  createStackComposer(app)
 
-    expect(createStackFunc).toThrowError()
+    const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
+    const domainTemplate = Template.fromStack(domainStack)
+    domainTemplate.resourceCountIs("AWS::OpenSearchService::Domain", 1)
 })
 
-test('Test missing engine version throws error', () => {
-
-    const app = new App({
-        context: {
-            engineVersion: ""
-        }
-    })
-
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
-
-    expect(createStackFunc).toThrowError()
-})
 
 test('Test invalid engine version format throws error', () => {
 
@@ -42,9 +29,7 @@ test('Test invalid engine version format throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -57,9 +42,7 @@ test('Test ES 7.10 engine version format is parsed', () => {
         }
     })
 
-    const openSearchStacks =  new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -74,9 +57,7 @@ test('Test OS 1.3 engine version format is parsed', () => {
         }
     })
 
-    const openSearchStacks =  new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -107,9 +88,7 @@ test('Test access policy is parsed for proper array format', () => {
         }
     })
 
-    const openSearchStacks =  new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -134,9 +113,7 @@ test('Test access policy is parsed for proper block format', () => {
         }
     })
 
-    const openSearchStacks =  new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const openSearchStacks = createStackComposer(app)
 
     const domainStack = openSearchStacks.stacks.filter((s) => s instanceof OpensearchServiceDomainCdkStack)[0]
     const domainTemplate = Template.fromStack(domainStack)
@@ -152,9 +129,7 @@ test('Test access policy missing Statement throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -167,9 +142,7 @@ test('Test access policy with empty Statement array throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -182,9 +155,7 @@ test('Test access policy with empty Statement block throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -198,9 +169,7 @@ test('Test access policy with improper Statement throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -213,9 +182,7 @@ test('Test invalid TLS security policy throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -228,9 +195,7 @@ test('Test invalid EBS volume type throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
@@ -243,9 +208,7 @@ test('Test invalid domain removal policy type throws error', () => {
         }
     })
 
-    const createStackFunc = () => new StackComposer(app, {
-        env: {account: "test-account", region: "us-east-1"}, stage: "unittest"
-    })
+    const createStackFunc = () => createStackComposer(app)
 
     expect(createStackFunc).toThrowError()
 })
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/test/test-utils.ts b/deployment/cdk/opensearch-service-migration/test/test-utils.ts
new file mode 100644
index 000000000..0f67bfd5d
--- /dev/null
+++ b/deployment/cdk/opensearch-service-migration/test/test-utils.ts
@@ -0,0 +1,8 @@
+import {Construct} from "constructs";
+import {StackComposer} from "../lib/stack-composer";
+
+export function createStackComposer(app: Construct) {
+    return new StackComposer(app, {
+        env: {account: "test-account", region: "us-east-1"}, stage: "unittest", copilotAppName: "test-app", copilotEnvName: "unittest"
+    })
+}
\ No newline at end of file
diff --git a/deployment/copilot/README.md b/deployment/copilot/README.md
index 302b173fd..e387dfcf7 100644
--- a/deployment/copilot/README.md
+++ b/deployment/copilot/README.md
@@ -29,20 +29,37 @@ Otherwise, please follow the manual instructions [here](https://aws.github.io/co
 ### Deploy with an automated script
 
 The following script command can be executed to deploy both the CDK infrastructure and Copilot services for a development environment
-```
+```shell
 ./devDeploy.sh
 ```
-Options:
-```
---skip-bootstrap                Skips installing packages with npm for CDK, bootstrapping CDK, and building the docker images used by Copilot
---skip-copilot-init             Skips Copilot initialization of app, environments, and services
---destroy                       Cleans up all deployed resources from this script (CDK and Copilot)
+Options can be found with:
+```shell
+./devDeploy.sh --help
 ```
 
 Requirements:
 * AWS credentials have been configured
 * CDK and Copilot CLIs have been installed
 
+#### How is an Authorization header set for requests from the Replayer to the target cluster?
+
+See Replayer explanation [here](../../TrafficCapture/trafficReplayer/README.md#authorization-header-for-replayed-requests)
+
+### How to run multiple Replayer scenarios
+
+The migration solution has support for running multiple Replayer services simultaneously, such that captured traffic from the Capture Proxy (which has been stored on Kafka) can be replayed on multiple different cluster configurations at the same time. These additional independent and distinct Replayer services can either be spun up together initially to replay traffic as it comes in, or added later, in which case they will begin processing captured traffic from the beginning of what is stored in Kafka.
+
+A **prerequisite** to use this functionality is that the migration solution has been deployed with the `devDeploy.sh` script, so that necessary environment values from CDK resources like the VPC, MSK, and target Domain can be retrieved for additional Replayer services
+
+To spin up another Replayer service, a command similar to below can be ran where `id` is a unique label to apply to this Replayer service and `target-uri` is the accessible endpoint of the target cluster to replay traffic to.
+```shell
+./createReplayer.sh --id new-domain-test --target-uri https://vpc-aos-domain-123.us-east-1.es.amazonaws.com:443
+```
+More options can be found with:
+```shell
+./createReplayer.sh --help
+```
+
 ### Deploy commands one at a time
 
 The following sections list out commands line-by-line for deploying this solution
@@ -50,17 +67,25 @@ The following sections list out commands line-by-line for deploying this solutio
 #### Importing values from CDK
 The typical use case for this Copilot app is to initially use the `opensearch-service-migration` CDK to deploy the surrounding infrastructure (VPC, OpenSearch Domain, Managed Kafka (MSK)) that Copilot requires, and then deploy the desired Copilot services. Documentation for setting up and deploying these resources can be found in the CDK [README](../cdk/opensearch-service-migration/README.md).
 
-The provided CDK will output export commands once deployed that can be ran on a given deployment machine to meet the required environment variables this Copilot app uses:
+The provided CDK will output export commands once deployed that can be ran on a given deployment machine to meet the required environment variables this Copilot app uses i.e.:
 ```
-export MIGRATION_VPC_ID=vpc-123;
+export MIGRATION_DOMAIN_SG_ID=sg-123;
 export MIGRATION_DOMAIN_ENDPOINT=vpc-aos-domain-123.us-east-1.es.amazonaws.com;
+export MIGRATION_DOMAIN_USER_AND_SECRET_ARN=admin arn:aws:secretsmanager:us-east-1:123456789123:secret:demo-user-secret-123abc
+export MIGRATION_VPC_ID=vpc-123;
 export MIGRATION_CAPTURE_MSK_SG_ID=sg-123;
 export MIGRATION_COMPARATOR_EFS_ID=fs-123;
 export MIGRATION_COMPARATOR_EFS_SG_ID=sg-123;
+export MIGRATION_REPLAYER_OUTPUT_EFS_ID=fs-124
+export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=sg-124
 export MIGRATION_PUBLIC_SUBNETS=subnet-123,subnet-124;
 export MIGRATION_PRIVATE_SUBNETS=subnet-125,subnet-126;
 export MIGRATION_KAFKA_BROKER_ENDPOINTS=b-1-public.loggingmskcluster.123.45.kafka.us-east-1.amazonaws.com:9198,b-2-public.loggingmskcluster.123.46.kafka.us-east-1.amazonaws.com:9198
 ```
+Additionally, if not using the deploy script, the following export is needed for the Replayer service:
+```
+export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --auth-header-user-and-secret $MIGRATION_DOMAIN_USER_AND_SECRET_ARN | nc traffic-comparator 9220"
+```
 
 #### Setting up existing Copilot infrastructure
 
@@ -73,6 +98,8 @@ If using temporary environment credentials when initializing an environment:
 * When prompted ` Would you like to use the default configuration for a new environment?` select `Yes, use default.` as this will ultimately get ignored for what has been configured in the existing `manifest.yml`
 * The last prompt will ask for the desired deployment region and should be filled out as Copilot will store this internally.
 
+This Copilot app supports deploying the Capture Proxy and Elasticsearch as a single service `capture-proxy-es` (as shown below) or as separate services `capture-proxy` and `elasticsearch`
+
 **Note**: This app also contains `kafka-broker` and `kafka-zookeeper` services which are currently experimental and usage of MSK is preferred. These services do not need to be deployed, and as so are not listed below.
 ```
 // Initialize app
@@ -86,10 +113,8 @@ copilot env init --name dev
 copilot svc init --name traffic-replayer
 copilot svc init --name traffic-comparator
 copilot svc init --name traffic-comparator-jupyter
-
-copilot svc init --name elasticsearch
-copilot svc init --name capture-proxy
-copilot svc init --name opensearch-benchmark
+copilot svc init --name capture-proxy-es
+copilot svc init --name migration-console
 
 ```
 
@@ -106,28 +131,35 @@ copilot env deploy --name dev
 copilot svc deploy --name traffic-comparator-jupyter --env dev
 copilot svc deploy --name traffic-comparator --env dev
 copilot svc deploy --name traffic-replayer --env dev
-
-copilot svc deploy --name elasticsearch --env dev
-copilot svc deploy --name capture-proxy --env dev
-copilot svc deploy --name opensearch-benchmark --env dev
+copilot svc deploy --name capture-proxy-es --env dev
+copilot svc deploy --name migration-console --env dev
 ```
 
 ### Running Benchmarks on the Deployed Solution
 
-Once the solution is deployed, the easiest way to test the solution is to exec into the benchmark container and run a benchmark test through, as the following steps illustrate
+Once the solution is deployed, the easiest way to test the solution is to exec into the migration-console container and run a benchmark test through, as the following steps illustrate
 
 ```
 // Exec into container
-copilot svc exec -a migration-copilot -e dev -n opensearch-benchmark -c "bash"
+copilot svc exec -a migration-copilot -e dev -n migration-console -c "bash"
+
+// Run opensearch-benchmark workload (i.e. geonames, nyc_taxis, http_logs)
 
-// Run benchmark workload (i.e. geonames, nyc_taxis, http_logs)
-opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture-proxy:443 --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options "use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin"
+// Option 1: Automated script
+./runTestBenchmarks.sh
+
+// Option 2: Manually execute command
+opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture-proxy-es:9200 --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1"  --client-options "use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin"
 ```
 
-After the benchmark has been run, the indices and documents of the source and target clusters can be checked from the same benchmark container to confirm
+After the benchmark has been run, the indices and documents of the source and target clusters can be checked from the same migration-console container to confirm
 ```
+// Option 1: Automated script
+./catIndices.sh
+
+// Option 2: Manually execute cluster requests
 // Check source cluster
-curl https://capture-proxy:443/_cat/indices?v --insecure -u admin:admin
+curl https://capture-proxy-es:9200/_cat/indices?v --insecure -u admin:admin
 
 // Check target cluster
 curl https://$MIGRATION_DOMAIN_ENDPOINT:443/_cat/indices?v --insecure -u admin:Admin123!
@@ -142,7 +174,8 @@ copilot svc exec -a migration-copilot -e dev -n traffic-comparator -c "bash"
 copilot svc exec -a migration-copilot -e dev -n traffic-replayer -c "bash"
 copilot svc exec -a migration-copilot -e dev -n elasticsearch -c "bash"
 copilot svc exec -a migration-copilot -e dev -n capture-proxy -c "bash"
-copilot svc exec -a migration-copilot -e dev -n opensearch-benchmark -c "bash"
+copilot svc exec -a migration-copilot -e dev -n capture-proxy-es -c "bash"
+copilot svc exec -a migration-copilot -e dev -n migration-console -c "bash"
 ```
 
 ### Addons
diff --git a/deployment/copilot/traffic-replayer/addons/taskRole.yml b/deployment/copilot/capture-proxy-es/addons/taskRole.yml
similarity index 57%
rename from deployment/copilot/traffic-replayer/addons/taskRole.yml
rename to deployment/copilot/capture-proxy-es/addons/taskRole.yml
index c853eb7bf..3320b44a1 100644
--- a/deployment/copilot/traffic-replayer/addons/taskRole.yml
+++ b/deployment/copilot/capture-proxy-es/addons/taskRole.yml
@@ -1,3 +1,5 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
 # You can use any of these parameters to create conditions or mappings in your template.
 Parameters:
   App:
@@ -11,32 +13,33 @@ Parameters:
     Description: Your workload's name.
     
 Resources:
-  MSKConsumerAccessPolicy:
+  MSKPublisherAccessPolicy:
     Type: AWS::IAM::ManagedPolicy
     Properties:
       Description: Allow compute host to consume from MSK
       PolicyDocument:
         Version: '2012-10-17'
-        # We should enhance IAM policy here to further restrict the Resource
         Statement:
           - Action:
               - kafka-cluster:Connect
-              - kafka-cluster:DescribeCluster
-            Effect: Allow
-            Resource: "*"
-          - Action:
-              - kafka-cluster:*Topic*
-              - kafka-cluster:ReadData
             Effect: Allow
-            Resource: "*"
+            Resource:
+              - Fn::ImportValue: !Sub "${App}-${Env}-msk-cluster-arn"
           - Action:
-              - kafka-cluster:AlterGroup
-              - kafka-cluster:DescribeGroup
+              - kafka-cluster:CreateTopic
+              - kafka-cluster:DescribeTopic
+              - kafka-cluster:WriteData
             Effect: Allow
-            Resource: "*"
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join" : [ ":topic", { "Fn::Split": [":cluster", {"Fn::ImportValue" : !Sub "${App}-${Env}-msk-cluster-arn"}]}] }
+                - "/*"
 
 Outputs:
   # 1. You need to output the IAM ManagedPolicy so that Copilot can add it as a managed policy to your ECS task role.
-  MSKConsumerAccessPolicyArn:
+  MSKPublisherAccessPolicyArn:
     Description: "The ARN of the ManagedPolicy to attach to the task role."
-    Value: !Ref MSKConsumerAccessPolicy
\ No newline at end of file
+    Value: !Ref MSKPublisherAccessPolicy
\ No newline at end of file
diff --git a/deployment/copilot/capture-proxy-es/manifest.yml b/deployment/copilot/capture-proxy-es/manifest.yml
new file mode 100644
index 000000000..ee1f5af57
--- /dev/null
+++ b/deployment/copilot/capture-proxy-es/manifest.yml
@@ -0,0 +1,47 @@
+# The manifest for the "capture-proxy-es" service. This service will spin up a Capture Proxy instance
+# and an Elasticsearch with Search Guard instance on a single container. You will also find in this repo these two
+# items split into their own services to give more flexibility in setup. A current limitation of this joined approach
+# is that we are unable to expose two ports (one for the capture proxy and one for elasticsearch) to other services
+# as Copilot seems to have a limitation to only allow exposing the default path "/" to a single port, in our case
+# we will expose the Capture Proxy port. If elasticsearch needs to be accessed directly it would need to be done
+# from this container.
+#
+#
+# Read the full specification for the "Backend Service" type at:
+#  https://aws.github.io/copilot-cli/docs/manifest/backend-service/
+
+# Your service name will be used in naming your resources like log groups, ECS services, etc.
+name: capture-proxy-es
+type: Backend Service
+
+# Allow service-to-service communication with ECS Service Connect
+network:
+  connect: true
+  vpc:
+    security_groups: [ "${MIGRATION_CAPTURE_MSK_SG_ID}" ]
+
+
+# Configuration for your containers and service.
+image:
+  # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#image-build
+  build: ../TrafficCapture/dockerSolution/build/docker/trafficCaptureProxyServer/Dockerfile
+  # Port exposed through your container to route traffic to it.
+  port: 9200
+
+command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --enableMSKAuth --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml & wait -n 1'
+
+cpu: 1024       # Number of CPU units for the task.
+memory: 4096    # Amount of memory in MiB used by the task.
+count: 1       # Number of tasks that should be running in your service.
+exec: true     # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html).
+
+# Pass environment variables as key value pairs.
+variables:
+  # Set Elasticsearch port to 19200 to allow capture proxy at port 9200
+  http.port: 19200
+
+environments:
+  dev:
+    count: 1               # Number of tasks to run for the "dev" environment.
+    deployment:            # The deployment strategy for the "dev" environment.
+      rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments.
\ No newline at end of file
diff --git a/deployment/copilot/capture-proxy/addons/taskRole.yml b/deployment/copilot/capture-proxy/addons/taskRole.yml
index fed0c52f1..3320b44a1 100644
--- a/deployment/copilot/capture-proxy/addons/taskRole.yml
+++ b/deployment/copilot/capture-proxy/addons/taskRole.yml
@@ -1,3 +1,5 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
 # You can use any of these parameters to create conditions or mappings in your template.
 Parameters:
   App:
@@ -17,24 +19,24 @@ Resources:
       Description: Allow compute host to consume from MSK
       PolicyDocument:
         Version: '2012-10-17'
-        # We should enhance IAM policy here to further restrict the Resource
         Statement:
           - Action:
               - kafka-cluster:Connect
-              - kafka-cluster:DescribeCluster
             Effect: Allow
-            Resource: "*"
+            Resource:
+              - Fn::ImportValue: !Sub "${App}-${Env}-msk-cluster-arn"
           - Action:
-              - kafka-cluster:*Topic*
-              - kafka-cluster:ReadData
+              - kafka-cluster:CreateTopic
+              - kafka-cluster:DescribeTopic
               - kafka-cluster:WriteData
             Effect: Allow
-            Resource: "*"
-          - Action:
-              - kafka-cluster:AlterGroup
-              - kafka-cluster:DescribeGroup
-            Effect: Allow
-            Resource: "*"
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join" : [ ":topic", { "Fn::Split": [":cluster", {"Fn::ImportValue" : !Sub "${App}-${Env}-msk-cluster-arn"}]}] }
+                - "/*"
 
 Outputs:
   # 1. You need to output the IAM ManagedPolicy so that Copilot can add it as a managed policy to your ECS task role.
diff --git a/deployment/copilot/capture-proxy/manifest.yml b/deployment/copilot/capture-proxy/manifest.yml
index 3aeb76f0a..2ac80b5ad 100644
--- a/deployment/copilot/capture-proxy/manifest.yml
+++ b/deployment/copilot/capture-proxy/manifest.yml
@@ -18,9 +18,9 @@ image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#image-build
   build: ../TrafficCapture/dockerSolution/build/docker/trafficCaptureProxyServer/Dockerfile
   # Port exposed through your container to route traffic to it.
-  port: 443
+  port: 9200
 
-command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --enableMSKAuth --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 443 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
+command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.Main  --kafkaConnection ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --enableMSKAuth --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml
 
 cpu: 512       # Number of CPU units for the task.
 memory: 2048    # Amount of memory in MiB used by the task.
diff --git a/deployment/copilot/createReplayer.sh b/deployment/copilot/createReplayer.sh
new file mode 100755
index 000000000..8c9bd0d58
--- /dev/null
+++ b/deployment/copilot/createReplayer.sh
@@ -0,0 +1,144 @@
+#!/bin/bash
+
+# Wish list
+# * Allow adding additional security group ids, https://opensearch.atlassian.net/browse/MIGRATIONS-1305
+
+# Allow executing this script from any dir
+script_abs_path=$(readlink -f "$0")
+script_dir_abs_path=$(dirname "$script_abs_path")
+cd $script_dir_abs_path
+
+usage() {
+  echo ""
+  echo "Create and Deploy Copilot Replayer service"
+  echo ""
+  echo "Usage: "
+  echo "  ./createReplayer.sh <--id STRING> <--target-uri URI> [--kafka-group-id STRING, --extra-args STRING, --delete-id STRING, --copilot-app-name STRING, --skip-copilot-init, --region STRING, --stage STRING]"
+  echo ""
+  echo "Options:"
+  echo "  --id                                  [string] The unique ID to give this particular Replayer service, will be used in service naming (e.g. traffic-replayer-ID)"
+  echo "  --target-uri                          [string] The URI of the target cluster that captured requests will be replayed to (e.g. https://my-target-cluster.com:443)"
+  echo "  --kafka-group-id                      [string, default: logging-group-<ID>] The Kafka consumer group ID the Replayer will use, if not specified an ID will be generated"
+  echo "  --extra-args                          [string, default: null] Extra arguments to provide to the Replayer command (e.g. --extra-args '--sigv4-auth-header-service-region es,us-east-1')"
+  echo "  --delete-id                           [string, default: null] Delete the Replayer directory with the given ID (e.g. traffic-replayer-ID) and remove the Copilot service"
+  echo "  --copilot-app-name                    [string, default: migration-copilot] Specify the Copilot application name to use for deployment"
+  echo "  --skip-copilot-init                   Skip one-time Copilot initialization of Replayer service"
+  echo "  -r, --region                          [string, default: us-east-1] Specify the AWS region to deploy the CloudFormation stack and resources."
+  echo "  -s, --stage                           [string, default: dev] Specify the stage name to associate with the deployed resources"
+  echo ""
+  exit 1
+}
+
+ID=""
+TARGET_URI=""
+KAFKA_GROUP_ID=""
+EXTRA_ARGS=""
+ID_TO_DELETE=""
+COPILOT_APP_NAME=migration-copilot
+SKIP_COPILOT_INIT=false
+REGION=us-east-1
+STAGE=dev
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --id)
+      ID="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --target-uri)
+      TARGET_URI="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --kafka-group-id)
+      KAFKA_GROUP_ID="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --extra-args)
+      EXTRA_ARGS="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --delete-id)
+      ID_TO_DELETE="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --copilot-app-name)
+      COPILOT_APP_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --skip-copilot-init)
+      SKIP_COPILOT_INIT=true
+      shift # past argument
+      ;;
+    -r|--region)
+      REGION="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -s|--stage)
+      STAGE="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -h|--help)
+      usage
+      ;;
+    -*)
+      echo "Unknown option $1"
+      usage
+      ;;
+    *)
+      shift # past argument
+      ;;
+  esac
+done
+
+
+# Remove service from Copilot and delete created service directory
+if [[ -n "${ID_TO_DELETE}" ]]; then
+  copilot svc delete -a $COPILOT_APP_NAME --name traffic-replayer-$ID_TO_DELETE --yes
+  rm -r traffic-replayer-$ID_TO_DELETE
+  exit 1
+fi
+
+# Check required parameters
+if [[ -z "${ID}" || -z "${TARGET_URI}" ]]; then
+  echo "Missing at least one required parameter: [--id, --target-uri]"
+  exit 1
+fi
+
+# Load environment variables generated from devDeploy.sh
+ENV_EXPORTS_FILE="./environments/${STAGE}/envExports.sh"
+if [ ! -f "${ENV_EXPORTS_FILE}" ]; then
+  echo "Required exports file ${ENV_EXPORTS_FILE} does not exist. This file will get generated when deploying the ./devDeploy.sh script"
+  exit 1
+else
+  echo "Loading environment from ${ENV_EXPORTS_FILE}"
+  source "${ENV_EXPORTS_FILE}"
+fi
+
+SERVICE_NAME="traffic-replayer-${ID}"
+if [[ -z "${KAFKA_GROUP_ID}" ]]; then
+  KAFKA_GROUP_ID="logging-group-${ID}"
+fi
+
+SERVICE_DIR="./${SERVICE_NAME}"
+if [ ! -d "${SERVICE_DIR}" ]; then
+  echo "Service directory: ${SERVICE_DIR} does not exist. Creating from ./templates/traffic-replayer"
+  cp -r ./templates/traffic-replayer/. "${SERVICE_DIR}"
+  sed -i.bak "s/SERVICE_NAME_PLACEHOLDER/${SERVICE_NAME}/g" "./${SERVICE_NAME}/manifest.yml" && rm "./${SERVICE_NAME}/manifest.yml.bak"
+fi
+
+if [ "${SKIP_COPILOT_INIT}" = false ] ; then
+  copilot svc init -a "${COPILOT_APP_NAME}" --name "${SERVICE_NAME}"
+fi
+
+REPLAY_COMMAND="/bin/sh -c \"/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer ${TARGET_URI} --insecure --kafka-traffic-brokers ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id ${KAFKA_GROUP_ID} --kafka-traffic-enable-msk-auth ${EXTRA_ARGS}\""
+echo "Constructed replay command: ${REPLAY_COMMAND}"
+export MIGRATION_REPLAYER_COMMAND="${REPLAY_COMMAND}"
+
+copilot svc deploy -a "${COPILOT_APP_NAME}" --name "${SERVICE_NAME}" --env "${STAGE}"
\ No newline at end of file
diff --git a/deployment/copilot/devDeploy.sh b/deployment/copilot/devDeploy.sh
index 474b9e8e3..ee255c3b5 100755
--- a/deployment/copilot/devDeploy.sh
+++ b/deployment/copilot/devDeploy.sh
@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/bash
 
-# Automation script to deploy the migration solution pipline to AWS for development use case. The current requirement
+# Automation script to deploy the migration solution pipeline to AWS for development use case. The current requirement
 # for use is having valid AWS credentials available to the environment
 
 # Stop script on command failures
@@ -13,47 +13,113 @@ cd $script_dir_abs_path
 
 SECONDS=0
 
-# Allow --skip-bootstrap flag to avoid one-time setups
-skip_bootstrap=false
-if [[ $* == *--skip-bootstrap* ]]
-then
-  skip_bootstrap=true
-fi
-# Allow --skip-copilot-init flag to avoid initializing Copilot components
-skip_copilot_init=false
-if [[ $* == *--skip-copilot-init* ]]
-then
-  skip_copilot_init=true
-fi
-# Allow --destroy flag to clean up existing resources
-destroy=false
-if [[ $* == *--destroy* ]]
-then
-  destroy=true
-fi
-
-export CDK_DEPLOYMENT_STAGE=dev
-export COPILOT_DEPLOYMENT_STAGE=dev
-# Will be used for CDK and Copilot
-export AWS_DEFAULT_REGION=us-east-1
-export COPILOT_DEPLOYMENT_NAME=migration-copilot
+usage() {
+  echo ""
+  echo "Deploy migration solution infrastructure composed of resources deployed by CDK and Copilot"
+  echo ""
+  echo "Options:"
+  echo "  --skip-bootstrap                      Skip one-time setup of installing npm package, bootstrapping CDK, and building Docker images."
+  echo "  --skip-copilot-init                   Skip one-time Copilot initialization of app, environments, and services"
+  echo "  --copilot-app-name                    [string, default: migration-copilot] Specify the Copilot application name to use for deployment"
+  echo "  --destroy-env                         Destroy all CDK and Copilot CloudFormation stacks deployed, excluding the Copilot app level stack, for the given env/stage and return to a clean state."
+  echo "  --destroy-all-copilot                 Destroy Copilot app and all Copilot CloudFormation stacks deployed for the given app across all regions."
+  echo "  -r, --region                          [string, default: us-east-1] Specify the AWS region to deploy the CloudFormation stacks and resources."
+  echo "  -s, --stage                           [string, default: dev] Specify the stage name to associate with the deployed resources"
+  exit 1
+}
+
+SKIP_BOOTSTRAP=false
+SKIP_COPILOT_INIT=false
+COPILOT_APP_NAME=migration-copilot
+DESTROY_ENV=false
+DESTROY_ALL_COPILOT=false
+REGION=us-east-1
+STAGE=dev
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-bootstrap)
+      SKIP_BOOTSTRAP=true
+      shift # past argument
+      ;;
+    --skip-copilot-init)
+      SKIP_COPILOT_INIT=true
+      shift # past argument
+      ;;
+    --copilot-app-name)
+      COPILOT_APP_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --destroy-env)
+      DESTROY_ENV=true
+      shift # past argument
+      ;;
+    --destroy-all-copilot)
+      DESTROY_ALL_COPILOT=true
+      shift # past argument
+      ;;
+    -r|--region)
+      REGION="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -s|--stage)
+      STAGE="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -h|--help)
+      usage
+      ;;
+    -*)
+      echo "Unknown option $1"
+      usage
+      ;;
+    *)
+      shift # past argument
+      ;;
+  esac
+done
+
+COPILOT_DEPLOYMENT_STAGE=$STAGE
+export AWS_DEFAULT_REGION=$REGION
+export CDK_DEPLOYMENT_STAGE=$STAGE
 # Used to overcome error: "failed to solve with frontend dockerfile.v0: failed to create LLB definition: unexpected
 # status code [manifests latest]: 400 Bad Request" but may not be practical
 export DOCKER_BUILDKIT=0
 export COMPOSE_DOCKER_CLI_BUILD=0
 
-if [ "$destroy" = true ] ; then
+if [ "$DESTROY_ENV" = true ] ; then
   set +e
-  copilot app delete
+  # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138)
+  export AWS_DEFAULT_REGION=""
+  copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --yes
+  copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --yes
+  copilot svc delete -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE --yes
+  copilot svc delete -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE --yes
+  copilot svc delete -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE --yes
+  copilot env delete -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --yes
+  rm ./environments/$COPILOT_DEPLOYMENT_STAGE/manifest.yml
+  echo "Destroying a region will not remove the Copilot app level stack that gets created in each region. This must be manually deleted from CloudFormation or automatically removed by deleting the Copilot app"
+
+  export AWS_DEFAULT_REGION=$REGION
   cd ../cdk/opensearch-service-migration
-  cdk destroy "*" --c domainName="aos-domain" --c engineVersion="OS_1.3" --c  dataNodeCount=2 --c vpcEnabled=true --c availabilityZoneCount=2 --c openAccessPolicyEnabled=true --c domainRemovalPolicy="DESTROY" --c migrationAssistanceEnabled=true --c mskEnablePublicEndpoints=true --c enableDemoAdmin=true
+  cdk destroy "*" --c domainName="aos-domain" --c engineVersion="OS_2.7" --c  dataNodeCount=2 --c vpcEnabled=true --c availabilityZoneCount=2 --c openAccessPolicyEnabled=true --c domainRemovalPolicy="DESTROY" --c migrationAssistanceEnabled=true --c enableDemoAdmin=true
   exit 1
 fi
 
+if [ "$DESTROY_ALL_COPILOT" = true ] ; then
+    # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138)
+    export AWS_DEFAULT_REGION=""
+    copilot app delete
+    echo "Destroying a Copilot app will not remove generated manifest.yml files in the copilot/environments directory. These should be manually deleted before deploying again. "
+    exit 1
+fi
+
 # === CDK Deployment ===
 
 cd ../cdk/opensearch-service-migration
-if [ "$skip_bootstrap" = false ] ; then
+if [ "$SKIP_BOOTSTRAP" = false ] ; then
   cd ../../../TrafficCapture
   ./gradlew :dockerSolution:buildDockerImages
   cd ../deployment/cdk/opensearch-service-migration
@@ -64,51 +130,55 @@ fi
 # This command deploys the required infrastructure for the migration solution with CDK that Copilot requires.
 # The options provided to `cdk deploy` here will cause a VPC, Opensearch Domain, and MSK(Kafka) resources to get created in AWS (among other resources)
 # More details on the CDK used here can be found at opensearch-migrations/deployment/cdk/opensearch-service-migration/README.md
-cdk deploy "*" --c domainName="aos-domain" --c engineVersion="OS_1.3" --c  dataNodeCount=2 --c vpcEnabled=true --c availabilityZoneCount=2 --c openAccessPolicyEnabled=true --c domainRemovalPolicy="DESTROY" --c migrationAssistanceEnabled=true --c mskEnablePublicEndpoints=true --c enableDemoAdmin=true -O cdkOutput.json --require-approval never
+cdk deploy "*" --c domainName="aos-domain" --c engineVersion="OS_2.7" --c  dataNodeCount=2 --c vpcEnabled=true --c availabilityZoneCount=2 --c openAccessPolicyEnabled=true --c domainRemovalPolicy="DESTROY" --c migrationAssistanceEnabled=true --c enableDemoAdmin=true -O cdk.out/cdkOutput.json --require-approval never --concurrency 3
 
-# Gather CDK output which includes export commands needed by Copilot, and make them available to the environment
-found_exports=$(grep -o "export [a-zA-Z0-9_]*=[^\\;\"]*" cdkOutput.json)
-eval "$(grep -o "export [a-zA-Z0-9_]*=[^\\;\"]*" cdkOutput.json)"
-printf "The following exports were added from CDK:\n%s\n" "$found_exports"
+# Collect export commands from CDK output, which are needed by Copilot, wrap the commands in double quotes and store them within the "environment" dir
+export_file_path=../../copilot/environments/$COPILOT_DEPLOYMENT_STAGE/envExports.sh
+grep -o "export [a-zA-Z0-9_]*=[^\\;\"]*" cdk.out/cdkOutput.json | sed 's/=/="/' | sed 's/.*/&"/' > "${export_file_path}"
+source "${export_file_path}"
+chmod +x "${export_file_path}"
+echo "The following exports were stored from CDK in ${export_file_path}"
+cat $export_file_path
 
 # Future enhancement needed here to make our Copilot deployment able to be reran without error even if no changes are deployed
 # === Copilot Deployment ===
 
 cd ../../copilot
 
+# Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138)
+export AWS_DEFAULT_REGION=""
+
 # Allow script to continue on error for copilot services, as copilot will error when no changes are needed
 set +e
 
-if [ "$skip_copilot_init" = false ] ; then
+if [ "$SKIP_COPILOT_INIT" = false ] ; then
   # Init app
-  copilot app init $COPILOT_DEPLOYMENT_NAME
+  copilot app init $COPILOT_APP_NAME
 
   # Init env, start state does not contain existing manifest but is created on the fly here to accommodate varying numbers of public and private subnets
-  copilot env init -a $COPILOT_DEPLOYMENT_NAME --name $COPILOT_DEPLOYMENT_STAGE --import-vpc-id $MIGRATION_VPC_ID --import-public-subnets $MIGRATION_PUBLIC_SUBNETS --import-private-subnets $MIGRATION_PRIVATE_SUBNETS --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $AWS_DEFAULT_REGION
-  #copilot env init -a $COPILOT_DEPLOYMENT_NAME --name $COPILOT_DEPLOYMENT_STAGE --default-config --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $AWS_DEFAULT_REGION
+  copilot env init -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --import-vpc-id $MIGRATION_VPC_ID --import-public-subnets $MIGRATION_PUBLIC_SUBNETS --import-private-subnets $MIGRATION_PRIVATE_SUBNETS --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $REGION
+  #copilot env init -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --default-config --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $REGION
 
   # Init services
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name traffic-comparator-jupyter
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name traffic-comparator
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name traffic-replayer
-
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name elasticsearch
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name capture-proxy
-  copilot svc init -a $COPILOT_DEPLOYMENT_NAME --name opensearch-benchmark
+  copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator-jupyter
+  copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator
+  copilot svc init -a $COPILOT_APP_NAME --name capture-proxy-es
+  copilot svc init -a $COPILOT_APP_NAME --name migration-console
+else
+  REPLAYER_SKIP_INIT_ARG="--skip-copilot-init"
 fi
 
 
 # Deploy env
-copilot env deploy -a $COPILOT_DEPLOYMENT_NAME --name $COPILOT_DEPLOYMENT_STAGE
+copilot env deploy -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE
 
 # Deploy services
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE
+copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE
+copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE
+copilot svc deploy -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE
+copilot svc deploy -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE
 
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name elasticsearch --env $COPILOT_DEPLOYMENT_STAGE
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name capture-proxy --env $COPILOT_DEPLOYMENT_STAGE
-copilot svc deploy -a $COPILOT_DEPLOYMENT_NAME --name opensearch-benchmark --env $COPILOT_DEPLOYMENT_STAGE
+./createReplayer.sh --id default --target-uri "https://${MIGRATION_DOMAIN_ENDPOINT}:443" --extra-args "--auth-header-user-and-secret ${MIGRATION_DOMAIN_USER_AND_SECRET_ARN} | nc traffic-comparator 9220" "${REPLAYER_SKIP_INIT_ARG}"
 
 
 # Output deployment time
diff --git a/deployment/copilot/opensearch-benchmark/manifest.yml b/deployment/copilot/migration-console/manifest.yml
similarity index 64%
rename from deployment/copilot/opensearch-benchmark/manifest.yml
rename to deployment/copilot/migration-console/manifest.yml
index 5def59ac8..ecdfda9b0 100644
--- a/deployment/copilot/opensearch-benchmark/manifest.yml
+++ b/deployment/copilot/migration-console/manifest.yml
@@ -1,25 +1,39 @@
-# The manifest for the "opensearch-benchmark" service.
+# The manifest for the "migration-console" service.
 # Read the full specification for the "Backend Service" type at:
 #  https://aws.github.io/copilot-cli/docs/manifest/backend-service/
 
 # Your service name will be used in naming your resources like log groups, ECS services, etc.
-name: opensearch-benchmark
+name: migration-console
 type: Backend Service
 
 # Allow service-to-service communication with ECS Service Connect
 network:
   connect: true
+  vpc:
+    security_groups: [ "${MIGRATION_DOMAIN_SG_ID}", "${MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID}" ]
 
 # Configuration for your containers and service.
 image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build
-  build: ../TrafficCapture/dockerSolution/src/main/docker/openSearchBenchmark/Dockerfile
+  build: ../TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile
+
+storage:
+  volumes:
+    sharedReplayerOutputVolume: # This is a variable key and can be set to an arbitrary string.
+      path: '/shared-replayer-output'
+      read_only: false
+      efs:
+        id: ${MIGRATION_REPLAYER_OUTPUT_EFS_ID}
 
 cpu: 512       # Number of CPU units for the task.
 memory: 1024    # Amount of memory in MiB used by the task.
 count: 1       # Number of tasks that should be running in your service.
 exec: true     # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html).
 
+# Pass environment variables as key value pairs.
+variables:
+  MIGRATION_DOMAIN_ENDPOINT: ${MIGRATION_DOMAIN_ENDPOINT}
+
 environments:
   dev:
     count: 1               # Number of tasks to run for the "dev" environment.
diff --git a/deployment/copilot/templates/traffic-replayer/addons/taskRole.yml b/deployment/copilot/templates/traffic-replayer/addons/taskRole.yml
new file mode 100644
index 000000000..d3f48ec8b
--- /dev/null
+++ b/deployment/copilot/templates/traffic-replayer/addons/taskRole.yml
@@ -0,0 +1,60 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
+# You can use any of these parameters to create conditions or mappings in your template.
+Parameters:
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: Your workload's name.
+    
+Resources:
+  MSKConsumerAccessPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: Allow compute host to consume from MSK
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Action:
+              - kafka-cluster:Connect
+            Effect: Allow
+            Resource:
+              - Fn::ImportValue: !Sub "${App}-${Env}-msk-cluster-arn"
+          - Action:
+              - kafka-cluster:DescribeTopic
+              - kafka-cluster:ReadData
+            Effect: Allow
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join" : [ ":topic", { "Fn::Split": [":cluster", {"Fn::ImportValue" : !Sub "${App}-${Env}-msk-cluster-arn"}]}] }
+                - "/*"
+          - Action:
+              - kafka-cluster:AlterGroup
+              - kafka-cluster:DescribeGroup
+            Effect: Allow
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join" : [ ":group", { "Fn::Split": [":cluster", {"Fn::ImportValue" : !Sub "${App}-${Env}-msk-cluster-arn"}]}] }
+                - "/*"
+          - Action:
+              - secretsmanager:GetSecretValue
+              - secretsmanager:DescribeSecret
+            Effect: Allow
+            Resource: "*"
+
+Outputs:
+  # 1. You need to output the IAM ManagedPolicy so that Copilot can add it as a managed policy to your ECS task role.
+  MSKConsumerAccessPolicyArn:
+    Description: "The ARN of the ManagedPolicy to attach to the task role."
+    Value: !Ref MSKConsumerAccessPolicy
\ No newline at end of file
diff --git a/deployment/copilot/traffic-replayer/manifest.yml b/deployment/copilot/templates/traffic-replayer/manifest.yml
similarity index 66%
rename from deployment/copilot/traffic-replayer/manifest.yml
rename to deployment/copilot/templates/traffic-replayer/manifest.yml
index 9eb28954b..11eef3c44 100644
--- a/deployment/copilot/traffic-replayer/manifest.yml
+++ b/deployment/copilot/templates/traffic-replayer/manifest.yml
@@ -1,16 +1,16 @@
-# The manifest for the "traffic-replayer" service.
+# The manifest for the SERVICE_NAME_PLACEHOLDER service.
 # Read the full specification for the "Backend Service" type at:
 #  https://aws.github.io/copilot-cli/docs/manifest/backend-service/
 
 # Your service name will be used in naming your resources like log groups, ECS services, etc.
-name: traffic-replayer
+name: SERVICE_NAME_PLACEHOLDER
 type: Backend Service
 
 # Allow service-to-service communication with ECS Service Connect
 network:
   connect: true
   vpc:
-    security_groups: [ "${MIGRATION_CAPTURE_MSK_SG_ID}" ]
+    security_groups: [ "${MIGRATION_CAPTURE_MSK_SG_ID}", "${MIGRATION_DOMAIN_SG_ID}", "${MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID}" ]
 
 # Configuration for your containers and service.
 image:
@@ -18,13 +18,25 @@ image:
   build:
     dockerfile: ../TrafficCapture/dockerSolution/build/docker/trafficReplayer/Dockerfile
 
-command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://${MIGRATION_DOMAIN_ENDPOINT}:443 --insecure --auth-header-value Basic\\ YWRtaW46QWRtaW4xMjMh --kafka-traffic-brokers ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth | nc traffic-comparator 9220"
+command: "${MIGRATION_REPLAYER_COMMAND}"
+
+storage:
+  volumes:
+    sharedReplayerOutputVolume: # This is a variable key and can be set to an arbitrary string.
+      path: '/shared-replayer-output'
+      read_only: false
+      efs:
+        id: ${MIGRATION_REPLAYER_OUTPUT_EFS_ID}
 
 cpu: 1024       # Number of CPU units for the task.
 memory: 4096    # Amount of memory in MiB used by the task.
 count: 1       # Number of tasks that should be running in your service.
 exec: true     # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html).
 
+# Pass environment variables as key value pairs.
+variables:
+  TUPLE_DIR_PATH: /shared-replayer-output/SERVICE_NAME_PLACEHOLDER
+
 environments:
   dev:
     count: 1               # Number of tasks to run for the "dev" environment.
diff --git a/deployment/docker/cw-puller/Dockerfile b/deployment/docker/cw-puller/Dockerfile
deleted file mode 100644
index baa992097..000000000
--- a/deployment/docker/cw-puller/Dockerfile
+++ /dev/null
@@ -1,24 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM python:3.8-slim-buster AS setup
-RUN apt-get update && apt-get install -y git
-
-WORKDIR /app
-
-RUN git clone https://github.com/opensearch-project/opensearch-migrations.git
-
-FROM python:3.8-slim-buster
-RUN apt-get update && apt-get install -y netcat
-
-WORKDIR /app
-
-#ENV CW_LOG_GROUP_NAME cw-feeder-lg
-#ENV CW_LOG_STREAM_NAME cw-feeder-stream
-
-COPY --from=setup /app/opensearch-migrations/TrafficReplayer/pull_then_poll_cw_logs.py .
-COPY ./run_app.sh .
-# Ideally this should be in opensearch-migrations repo
-COPY requirements.txt requirements.txt
-
-RUN pip3 install -r requirements.txt
-
-CMD ./run_app.sh
\ No newline at end of file
diff --git a/deployment/docker/cw-puller/requirements.txt b/deployment/docker/cw-puller/requirements.txt
deleted file mode 100644
index 1db657b6b..000000000
--- a/deployment/docker/cw-puller/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-boto3
\ No newline at end of file
diff --git a/deployment/docker/cw-puller/run_app.sh b/deployment/docker/cw-puller/run_app.sh
deleted file mode 100755
index 0782fab3a..000000000
--- a/deployment/docker/cw-puller/run_app.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# Pipe output to STDERR as well as to port with netcat
-while true
-do
-  python3 -u pull_then_poll_cw_logs.py | tee /dev/stderr | nc localhost 9210
-  >&2 echo "Command has encountered error. Restarting now ..."
-  sleep 5
-done
\ No newline at end of file
diff --git a/deployment/docker/logstash-setup/Dockerfile b/deployment/docker/logstash-setup/Dockerfile
deleted file mode 100644
index 7330639fe..000000000
--- a/deployment/docker/logstash-setup/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:7.13.4
-
-COPY ./run_app.sh .
-
-#ARG LOGSTASH_CONFIG
-#ENV LOGSTASH_CONFIG ${LOGSTASH_CONFIG}
-
-CMD ./run_app.sh
\ No newline at end of file
diff --git a/deployment/docker/logstash-setup/run_app.sh b/deployment/docker/logstash-setup/run_app.sh
deleted file mode 100755
index c77969a27..000000000
--- a/deployment/docker/logstash-setup/run_app.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# Temporary measure, if we pursue this longer term should see about properly passing file
-echo $LOGSTASH_CONFIG > logstash.conf
-sed -i 's/PUT_LINE/\n/g' logstash.conf
-
-# ECS does not have clear convention for running a task only once, again if pursued longer term
-# we should do something other than stall on success and retry on failure
-if /usr/share/logstash/bin/logstash -f logstash.conf ; then
-  while true
-  do
-    echo "Logstash migration finished successfully"
-    sleep 600
-  done
-else
-  echo "Logstash migration has failed. Will relaunch shortly..."
-  sleep 60
-fi
\ No newline at end of file
diff --git a/deployment/docker/traffic-comparator/Dockerfile b/deployment/docker/traffic-comparator/Dockerfile
deleted file mode 100644
index 2bb495666..000000000
--- a/deployment/docker/traffic-comparator/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM python:3.10.10
-
-RUN apt-get update && apt-get install -y netcat && apt-get install -y git
-
-# Be aware that this layer can be cached and not capture recent commits. Run docker build --no-cache to avoid this
-RUN git clone https://github.com/opensearch-project/traffic-comparator.git
-
-WORKDIR /traffic-comparator
-
-COPY run_app.sh .
-
-RUN pip3 install --editable .
-
-CMD ./run_app.sh
-
diff --git a/deployment/docker/traffic-comparator/README.md b/deployment/docker/traffic-comparator/README.md
deleted file mode 100644
index cfde143ad..000000000
--- a/deployment/docker/traffic-comparator/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-The dockerfile in this directory will build an image that will run the traffic comparator on the json-formatted triples
-that it receives and prints performance statistic.
-
-It will run the necessary steps to setup the traffic comparator "run_app.sh" script which will run the traffic 
-comparator on the triples it receives via port 9220.OPTIONALLY give a logfile that contains the diffs between the 
-responses which can be saved (S3 bucket, shared volume, etc) by inserting a command that does that within the 
-"run_app.sh", after the command that runs the traffic-comparator.
-
-This part (--export-reports DiffReport diffs.log) of the command in "run_app.sh" should only be included if the user
-wants the diffs. This part will give a logfile that contains the diffs between the responses which can be 
-saved (S3 bucket, shared volume, etc) by inserting a command that does that within the "run_app.sh", after the command
-that runs the traffic-comparator
-'nc -l -p 9220 | trafficcomparator -v stream | trafficcomparator stream-report --export-reports DiffReport diffs.log'
\ No newline at end of file
diff --git a/deployment/docker/traffic-comparator/run_app.sh b/deployment/docker/traffic-comparator/run_app.sh
deleted file mode 100755
index add451090..000000000
--- a/deployment/docker/traffic-comparator/run_app.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# IF the user cares about the diffs.log file, then another command to save it somewhere can be inserted here.
-# AND, the user can use the "--export-reports DiffReport" option of the trafficcomparator
-
-while true
-do
-  nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -v stream | trafficcomparator stream-report
-  >&2 echo "Command has encountered error. Restarting now ..."
-  sleep 1
-done
-
-
-# Next steps: Save the diffs.log somewhere. (Again, if the user needs them)
-# In this case, we will need to add the part of the command mentioned above to save the diff logs e.g
-# "--export-reports DiffReport diffs.log".
-# So it's either you include the full command AND insert another command here to save the file, or neither of them.
-# The command to save the diffs.log can to save it to either a shared volume (which is docker and fargate compatible),
-# S3 bucket, or whichever works and is a more convenient solution for the user.
-# A task was created to add this command (https://opensearch.atlassian.net/browse/MIGRATIONS-1043)
diff --git a/cluster_migration_core/README.md b/experimental/cluster_migration_core/README.md
similarity index 100%
rename from cluster_migration_core/README.md
rename to experimental/cluster_migration_core/README.md
diff --git a/cluster_migration_core/cluster_migration_core/clients/__init__.py b/experimental/cluster_migration_core/cluster_migration_core/clients/__init__.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/clients/__init__.py
rename to experimental/cluster_migration_core/cluster_migration_core/clients/__init__.py
diff --git a/cluster_migration_core/cluster_migration_core/clients/rest_client_base.py b/experimental/cluster_migration_core/cluster_migration_core/clients/rest_client_base.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/clients/rest_client_base.py
rename to experimental/cluster_migration_core/cluster_migration_core/clients/rest_client_base.py
diff --git a/cluster_migration_core/cluster_migration_core/clients/rest_client_default.py b/experimental/cluster_migration_core/cluster_migration_core/clients/rest_client_default.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/clients/rest_client_default.py
rename to experimental/cluster_migration_core/cluster_migration_core/clients/rest_client_default.py
diff --git a/cluster_migration_core/cluster_migration_core/clients/rest_ops.py b/experimental/cluster_migration_core/cluster_migration_core/clients/rest_ops.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/clients/rest_ops.py
rename to experimental/cluster_migration_core/cluster_migration_core/clients/rest_ops.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/__init__.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/__init__.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/__init__.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/__init__.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/cluster.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/cluster.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/cluster.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/cluster.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/cluster_objects.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/cluster_objects.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/cluster_objects.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/cluster_objects.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/container_configuration.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/container_configuration.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/container_configuration.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/container_configuration.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/docker_command_gen.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/docker_command_gen.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/docker_command_gen.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/docker_command_gen.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/docker_framework_client.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/docker_framework_client.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/docker_framework_client.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/docker_framework_client.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/node.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/node.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/node.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/node.py
diff --git a/cluster_migration_core/cluster_migration_core/cluster_management/node_configuration.py b/experimental/cluster_migration_core/cluster_migration_core/cluster_management/node_configuration.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/cluster_management/node_configuration.py
rename to experimental/cluster_migration_core/cluster_migration_core/cluster_management/node_configuration.py
diff --git a/cluster_migration_core/cluster_migration_core/core/__init__.py b/experimental/cluster_migration_core/cluster_migration_core/core/__init__.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/__init__.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/__init__.py
diff --git a/cluster_migration_core/cluster_migration_core/core/constants.py b/experimental/cluster_migration_core/cluster_migration_core/core/constants.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/constants.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/constants.py
diff --git a/cluster_migration_core/cluster_migration_core/core/exception_base.py b/experimental/cluster_migration_core/cluster_migration_core/core/exception_base.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/exception_base.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/exception_base.py
diff --git a/cluster_migration_core/cluster_migration_core/core/expectation.py b/experimental/cluster_migration_core/cluster_migration_core/core/expectation.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/expectation.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/expectation.py
diff --git a/cluster_migration_core/cluster_migration_core/core/framework_runner.py b/experimental/cluster_migration_core/cluster_migration_core/core/framework_runner.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/framework_runner.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/framework_runner.py
diff --git a/cluster_migration_core/cluster_migration_core/core/framework_state.py b/experimental/cluster_migration_core/cluster_migration_core/core/framework_state.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/framework_state.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/framework_state.py
diff --git a/cluster_migration_core/cluster_migration_core/core/framework_step.py b/experimental/cluster_migration_core/cluster_migration_core/core/framework_step.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/framework_step.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/framework_step.py
diff --git a/cluster_migration_core/cluster_migration_core/core/logging_wrangler.py b/experimental/cluster_migration_core/cluster_migration_core/core/logging_wrangler.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/logging_wrangler.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/logging_wrangler.py
diff --git a/cluster_migration_core/cluster_migration_core/core/shell_interactions.py b/experimental/cluster_migration_core/cluster_migration_core/core/shell_interactions.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/shell_interactions.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/shell_interactions.py
diff --git a/cluster_migration_core/cluster_migration_core/core/test_config_wrangling.py b/experimental/cluster_migration_core/cluster_migration_core/core/test_config_wrangling.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/test_config_wrangling.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/test_config_wrangling.py
diff --git a/cluster_migration_core/cluster_migration_core/core/versions_engine.py b/experimental/cluster_migration_core/cluster_migration_core/core/versions_engine.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/versions_engine.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/versions_engine.py
diff --git a/cluster_migration_core/cluster_migration_core/core/workspace_wrangler.py b/experimental/cluster_migration_core/cluster_migration_core/core/workspace_wrangler.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/core/workspace_wrangler.py
rename to experimental/cluster_migration_core/cluster_migration_core/core/workspace_wrangler.py
diff --git a/cluster_migration_core/cluster_migration_core/robot_actions/action_executor.py b/experimental/cluster_migration_core/cluster_migration_core/robot_actions/action_executor.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/robot_actions/action_executor.py
rename to experimental/cluster_migration_core/cluster_migration_core/robot_actions/action_executor.py
diff --git a/cluster_migration_core/cluster_migration_core/robot_actions/cluster_action_executor.py b/experimental/cluster_migration_core/cluster_migration_core/robot_actions/cluster_action_executor.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/robot_actions/cluster_action_executor.py
rename to experimental/cluster_migration_core/cluster_migration_core/robot_actions/cluster_action_executor.py
diff --git a/cluster_migration_core/cluster_migration_core/robot_actions/result_reporter.py b/experimental/cluster_migration_core/cluster_migration_core/robot_actions/result_reporter.py
similarity index 100%
rename from cluster_migration_core/cluster_migration_core/robot_actions/result_reporter.py
rename to experimental/cluster_migration_core/cluster_migration_core/robot_actions/result_reporter.py
diff --git a/experimental/cluster_migration_core/requirements.txt b/experimental/cluster_migration_core/requirements.txt
new file mode 100644
index 000000000..d6e1198b1
--- /dev/null
+++ b/experimental/cluster_migration_core/requirements.txt
@@ -0,0 +1 @@
+-e .
diff --git a/cluster_migration_core/setup.py b/experimental/cluster_migration_core/setup.py
similarity index 100%
rename from cluster_migration_core/setup.py
rename to experimental/cluster_migration_core/setup.py
diff --git a/cluster_migration_core/unit_tests/clients/test_rest_client_default.py b/experimental/cluster_migration_core/unit_tests/clients/test_rest_client_default.py
similarity index 100%
rename from cluster_migration_core/unit_tests/clients/test_rest_client_default.py
rename to experimental/cluster_migration_core/unit_tests/clients/test_rest_client_default.py
diff --git a/cluster_migration_core/unit_tests/clients/test_rest_ops.py b/experimental/cluster_migration_core/unit_tests/clients/test_rest_ops.py
similarity index 100%
rename from cluster_migration_core/unit_tests/clients/test_rest_ops.py
rename to experimental/cluster_migration_core/unit_tests/clients/test_rest_ops.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_cluster.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_cluster.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_cluster.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_cluster.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_container_configuration.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_container_configuration.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_container_configuration.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_container_configuration.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_docker_command_gen.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_docker_command_gen.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_docker_command_gen.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_docker_command_gen.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_docker_framework_client.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_docker_framework_client.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_docker_framework_client.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_docker_framework_client.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_node.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_node.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_node.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_node.py
diff --git a/cluster_migration_core/unit_tests/cluster_management/test_node_configuration.py b/experimental/cluster_migration_core/unit_tests/cluster_management/test_node_configuration.py
similarity index 100%
rename from cluster_migration_core/unit_tests/cluster_management/test_node_configuration.py
rename to experimental/cluster_migration_core/unit_tests/cluster_management/test_node_configuration.py
diff --git a/cluster_migration_core/unit_tests/core/test_exception_base.py b/experimental/cluster_migration_core/unit_tests/core/test_exception_base.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_exception_base.py
rename to experimental/cluster_migration_core/unit_tests/core/test_exception_base.py
diff --git a/cluster_migration_core/unit_tests/core/test_expectation.py b/experimental/cluster_migration_core/unit_tests/core/test_expectation.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_expectation.py
rename to experimental/cluster_migration_core/unit_tests/core/test_expectation.py
diff --git a/cluster_migration_core/unit_tests/core/test_framework_runner.py b/experimental/cluster_migration_core/unit_tests/core/test_framework_runner.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_framework_runner.py
rename to experimental/cluster_migration_core/unit_tests/core/test_framework_runner.py
diff --git a/cluster_migration_core/unit_tests/core/test_framework_state.py b/experimental/cluster_migration_core/unit_tests/core/test_framework_state.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_framework_state.py
rename to experimental/cluster_migration_core/unit_tests/core/test_framework_state.py
diff --git a/cluster_migration_core/unit_tests/core/test_framework_step.py b/experimental/cluster_migration_core/unit_tests/core/test_framework_step.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_framework_step.py
rename to experimental/cluster_migration_core/unit_tests/core/test_framework_step.py
diff --git a/cluster_migration_core/unit_tests/core/test_logging_wrangler.py b/experimental/cluster_migration_core/unit_tests/core/test_logging_wrangler.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_logging_wrangler.py
rename to experimental/cluster_migration_core/unit_tests/core/test_logging_wrangler.py
diff --git a/cluster_migration_core/unit_tests/core/test_shell_interactions.py b/experimental/cluster_migration_core/unit_tests/core/test_shell_interactions.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_shell_interactions.py
rename to experimental/cluster_migration_core/unit_tests/core/test_shell_interactions.py
diff --git a/cluster_migration_core/unit_tests/core/test_test_config_wrangling.py b/experimental/cluster_migration_core/unit_tests/core/test_test_config_wrangling.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_test_config_wrangling.py
rename to experimental/cluster_migration_core/unit_tests/core/test_test_config_wrangling.py
diff --git a/cluster_migration_core/unit_tests/core/test_versions_engine.py b/experimental/cluster_migration_core/unit_tests/core/test_versions_engine.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_versions_engine.py
rename to experimental/cluster_migration_core/unit_tests/core/test_versions_engine.py
diff --git a/cluster_migration_core/unit_tests/core/test_workspace_wrangler.py b/experimental/cluster_migration_core/unit_tests/core/test_workspace_wrangler.py
similarity index 100%
rename from cluster_migration_core/unit_tests/core/test_workspace_wrangler.py
rename to experimental/cluster_migration_core/unit_tests/core/test_workspace_wrangler.py
diff --git a/cluster_migration_core/unit_tests/robot_actions/test_action_executor.py b/experimental/cluster_migration_core/unit_tests/robot_actions/test_action_executor.py
similarity index 100%
rename from cluster_migration_core/unit_tests/robot_actions/test_action_executor.py
rename to experimental/cluster_migration_core/unit_tests/robot_actions/test_action_executor.py
diff --git a/cluster_migration_core/unit_tests/robot_actions/test_cluster_action_executor.py b/experimental/cluster_migration_core/unit_tests/robot_actions/test_cluster_action_executor.py
similarity index 100%
rename from cluster_migration_core/unit_tests/robot_actions/test_cluster_action_executor.py
rename to experimental/cluster_migration_core/unit_tests/robot_actions/test_cluster_action_executor.py
diff --git a/cluster_migration_core/unit_tests/robot_actions/test_result_reporter.py b/experimental/cluster_migration_core/unit_tests/robot_actions/test_result_reporter.py
similarity index 100%
rename from cluster_migration_core/unit_tests/robot_actions/test_result_reporter.py
rename to experimental/cluster_migration_core/unit_tests/robot_actions/test_result_reporter.py
diff --git a/cluster_traffic_capture/README.md b/experimental/cluster_traffic_capture/README.md
similarity index 100%
rename from cluster_traffic_capture/README.md
rename to experimental/cluster_traffic_capture/README.md
diff --git a/cluster_traffic_capture/build_docker_images.py b/experimental/cluster_traffic_capture/build_docker_images.py
similarity index 100%
rename from cluster_traffic_capture/build_docker_images.py
rename to experimental/cluster_traffic_capture/build_docker_images.py
diff --git a/cluster_traffic_capture/cluster_traffic_capture/__init__.py b/experimental/cluster_traffic_capture/cluster_traffic_capture/__init__.py
similarity index 100%
rename from cluster_traffic_capture/cluster_traffic_capture/__init__.py
rename to experimental/cluster_traffic_capture/cluster_traffic_capture/__init__.py
diff --git a/cluster_traffic_capture/cluster_traffic_capture/gen_haproxy_cfg.py b/experimental/cluster_traffic_capture/cluster_traffic_capture/gen_haproxy_cfg.py
similarity index 100%
rename from cluster_traffic_capture/cluster_traffic_capture/gen_haproxy_cfg.py
rename to experimental/cluster_traffic_capture/cluster_traffic_capture/gen_haproxy_cfg.py
diff --git a/cluster_traffic_capture/demo_haproxy.py b/experimental/cluster_traffic_capture/demo_haproxy.py
similarity index 100%
rename from cluster_traffic_capture/demo_haproxy.py
rename to experimental/cluster_traffic_capture/demo_haproxy.py
diff --git a/cluster_traffic_capture/docker_config_capture/Dockerfile b/experimental/cluster_traffic_capture/docker_config_capture/Dockerfile
similarity index 100%
rename from cluster_traffic_capture/docker_config_capture/Dockerfile
rename to experimental/cluster_traffic_capture/docker_config_capture/Dockerfile
diff --git a/cluster_traffic_capture/docker_config_capture/amazon-cloudwatch-agent.json b/experimental/cluster_traffic_capture/docker_config_capture/amazon-cloudwatch-agent.json
similarity index 100%
rename from cluster_traffic_capture/docker_config_capture/amazon-cloudwatch-agent.json
rename to experimental/cluster_traffic_capture/docker_config_capture/amazon-cloudwatch-agent.json
diff --git a/cluster_traffic_capture/docker_config_capture/mirror.conf b/experimental/cluster_traffic_capture/docker_config_capture/mirror.conf
similarity index 100%
rename from cluster_traffic_capture/docker_config_capture/mirror.conf
rename to experimental/cluster_traffic_capture/docker_config_capture/mirror.conf
diff --git a/cluster_traffic_capture/docker_config_capture/rsyslog.conf b/experimental/cluster_traffic_capture/docker_config_capture/rsyslog.conf
similarity index 100%
rename from cluster_traffic_capture/docker_config_capture/rsyslog.conf
rename to experimental/cluster_traffic_capture/docker_config_capture/rsyslog.conf
diff --git a/cluster_traffic_capture/docker_config_demo/demo_entrypoint.sh b/experimental/cluster_traffic_capture/docker_config_demo/demo_entrypoint.sh
similarity index 100%
rename from cluster_traffic_capture/docker_config_demo/demo_entrypoint.sh
rename to experimental/cluster_traffic_capture/docker_config_demo/demo_entrypoint.sh
diff --git a/cluster_traffic_capture/docker_config_traffic_gen/Dockerfile b/experimental/cluster_traffic_capture/docker_config_traffic_gen/Dockerfile
similarity index 100%
rename from cluster_traffic_capture/docker_config_traffic_gen/Dockerfile
rename to experimental/cluster_traffic_capture/docker_config_traffic_gen/Dockerfile
diff --git a/cluster_traffic_capture/elasticsearch_with_loggging/Dockerfile b/experimental/cluster_traffic_capture/elasticsearch_with_loggging/Dockerfile
similarity index 100%
rename from cluster_traffic_capture/elasticsearch_with_loggging/Dockerfile
rename to experimental/cluster_traffic_capture/elasticsearch_with_loggging/Dockerfile
diff --git a/cluster_traffic_capture/elasticsearch_with_loggging/elasticsearch.yml b/experimental/cluster_traffic_capture/elasticsearch_with_loggging/elasticsearch.yml
similarity index 100%
rename from cluster_traffic_capture/elasticsearch_with_loggging/elasticsearch.yml
rename to experimental/cluster_traffic_capture/elasticsearch_with_loggging/elasticsearch.yml
diff --git a/cluster_traffic_capture/elasticsearch_with_loggging/log4j2.properties b/experimental/cluster_traffic_capture/elasticsearch_with_loggging/log4j2.properties
similarity index 100%
rename from cluster_traffic_capture/elasticsearch_with_loggging/log4j2.properties
rename to experimental/cluster_traffic_capture/elasticsearch_with_loggging/log4j2.properties
diff --git a/upgrades/requirements.txt b/experimental/cluster_traffic_capture/requirements.txt
similarity index 85%
rename from upgrades/requirements.txt
rename to experimental/cluster_traffic_capture/requirements.txt
index 8219a264e..1b584249b 100644
--- a/upgrades/requirements.txt
+++ b/experimental/cluster_traffic_capture/requirements.txt
@@ -1,2 +1,2 @@
 -e ../cluster_migration_core
--e .
\ No newline at end of file
+-e .
diff --git a/cluster_traffic_capture/setup.py b/experimental/cluster_traffic_capture/setup.py
similarity index 100%
rename from cluster_traffic_capture/setup.py
rename to experimental/cluster_traffic_capture/setup.py
diff --git a/knowledge_base/README.md b/experimental/knowledge_base/README.md
similarity index 81%
rename from knowledge_base/README.md
rename to experimental/knowledge_base/README.md
index 5af75e6d4..db84c4341 100644
--- a/knowledge_base/README.md
+++ b/experimental/knowledge_base/README.md
@@ -2,6 +2,6 @@
 
 The Knowledge Base is the collection of expectations we have about the behavior of clusters of various versions and across upgrades.
 
-It can be used in various ways, but was designed to be a component of the Upgrade Testing Framework. There is more extensive documentation of how it works in [upgrades/README.md](../upgrades/README.md).
+It can be used in various ways, but was designed to be a component of the Upgrade Testing Framework. There is more extensive documentation of how it works in [experimental/upgrades/README.md](../experimental/upgrades/README.md).
 
 Further discussion on adding additional expectations to this knowledge base, can be found in the PR [here](https://github.com/opensearch-project/opensearch-migrations/pull/68)
diff --git a/knowledge_base/consistent-document-count.json b/experimental/knowledge_base/consistent-document-count.json
similarity index 100%
rename from knowledge_base/consistent-document-count.json
rename to experimental/knowledge_base/consistent-document-count.json
diff --git a/knowledge_base/doy-format-date-range-query.json b/experimental/knowledge_base/doy-format-date-range-query.json
similarity index 100%
rename from knowledge_base/doy-format-date-range-query.json
rename to experimental/knowledge_base/doy-format-date-range-query.json
diff --git a/upgrades/README.md b/experimental/upgrades/README.md
similarity index 97%
rename from upgrades/README.md
rename to experimental/upgrades/README.md
index 92eced701..118e1dcd1 100644
--- a/upgrades/README.md
+++ b/experimental/upgrades/README.md
@@ -10,10 +10,10 @@ The UTF is based around the following concepts:
 * A sequence of "Steps" that are executed in a set order as part of a "Workflow"
 * A "Runner" that takes a Workflow and executes the steps in it sequentially, while managing things like error handling, logging, and state
 * A shared "State" that exists in-memory of the process and enables the results of one step to be re-used by a later step
-* A "Test Config" JSON file that encapsulates the specific upgrade to be tested (e.g. snapshot/restore from ES 7.10.2 to OS 1.3.6).  This package will contain a collection of pre-canned Test Configs (`./test_configs/`) that should represent the limits of the UTF's knowledge about what is true about how Upgrades work.  In other words - if there isn't a Test Config file in the included collection that covers the specific cluster setup and upgrade type you're interested in, the UTF is not testing that setup/upgrade type.
+* A "Test Config" JSON file that encapsulates the specific upgrade to be tested (e.g. snapshot/restore from ES 7.10.2 to OS 1.3.6).  This package will contain a collection of pre-canned Test Configs (`./test_configs/`) that should represent the limits of the UTF's knowledge about what is true about how upgrades work.  In other words - if there isn't a Test Config file in the included collection that covers the specific cluster setup and upgrade type you're interested in, the UTF is not testing that setup/upgrade type.
 * A set of "Test Actions" that are performed at various points in a Workflow via an existing library called the [Robot Framework](https://robotframework.org/?tab=2#getting-started).  These are currently at `./upgrade_testing_framework/robot_test_defs/`
 * A set of "Expectations" that represent things we expect to be true about the upgrade specified in the Test Config file (number of documents should remain the same before/after).  Each Expectation has an Expectation ID that is used to track which Test Actions are associated with a given Expectation, and determine if the Expectation was true or not.  These Expectation IDs are associated with each Test Action as a tag that the UTF search for and enable selective invocation.
-* A "Knowledge Base" that represents all the Expectations we're currently tracking and (hopefully) testing, currently located at `../knowledge_base/`.
+* A "Knowledge Base" that represents all the Expectations we're currently tracking and (hopefully) testing, currently located at `../experimental/knowledge_base/`.
 
 ## Running the Upgrade Testing Framework
 
@@ -64,11 +64,11 @@ At the end of a UTF run, the final results of the test should be printed to STDO
         "doy-format-date-range-query-bug"
     ]
 }
-[ReportResults] For more information about how to interpret these results, please consult the Upgrade Testing Framework's README file: https://github.com/opensearch-project/opensearch-migrations/blob/main/upgrades/README.md
+[ReportResults] For more information about how to interpret these results, please consult the Upgrade Testing Framework's README file: https://github.com/opensearch-project/opensearch-migrations/blob/main/experimental/upgrades/README.md
 [FrameworkRunner] Step Succeeded: ReportResults
 ```
 
-As explained above, the UTF is focused on checking whether our Expectations about the upgrade-under-test are true or not.  In this example above, we had two Expectations in our Knowledge Base that were relevant to this upgrade - `consistent-document-count` and `doy-format-date-range-query-bug`.  These strings are both Expectation IDs, and can be used to find more details about the specific expectation is/what versions it applies to (check `../knowledge_base/`) and how they are being tested (`./upgrade_testing_framework/robot_test_defs/`).  The ID can be searched for in both place to find the relevant bits.
+As explained above, the UTF is focused on checking whether our Expectations about the upgrade-under-test are true or not.  In this example above, we had two Expectations in our Knowledge Base that were relevant to this upgrade - `consistent-document-count` and `doy-format-date-range-query-bug`.  These strings are both Expectation IDs, and can be used to find more details about the specific expectation is/what versions it applies to (check `../experimental/knowledge_base/`) and how they are being tested (`./upgrade_testing_framework/robot_test_defs/`).  The ID can be searched for in both place to find the relevant bits.
 
 We split our results into three categories:
 * Passing Expectations: These Expectations had Test Actions associated with them, and the Test Actions' results matched what we thought they should.  In the example above, we checked that the document count was the same before after an upgrade (a happy-path test).  However, if we were aware of a bug that should present itself as part of the upgrade, and that bug did present itself in the way we expected, then that would be considered a "passing" Expectation as well.  In other words - passing Expectations indicate that the upgrade went how we thought it should, for good or ill.
diff --git a/cluster_traffic_capture/requirements.txt b/experimental/upgrades/requirements.txt
similarity index 85%
rename from cluster_traffic_capture/requirements.txt
rename to experimental/upgrades/requirements.txt
index 8219a264e..1b584249b 100644
--- a/cluster_traffic_capture/requirements.txt
+++ b/experimental/upgrades/requirements.txt
@@ -1,2 +1,2 @@
 -e ../cluster_migration_core
--e .
\ No newline at end of file
+-e .
diff --git a/upgrades/run_utf.py b/experimental/upgrades/run_utf.py
similarity index 100%
rename from upgrades/run_utf.py
rename to experimental/upgrades/run_utf.py
diff --git a/upgrades/setup.py b/experimental/upgrades/setup.py
similarity index 100%
rename from upgrades/setup.py
rename to experimental/upgrades/setup.py
diff --git a/upgrades/test_configs/snapshot_restore_es_7_10_2_to_os_1_3_6.json b/experimental/upgrades/test_configs/snapshot_restore_es_7_10_2_to_os_1_3_6.json
similarity index 100%
rename from upgrades/test_configs/snapshot_restore_es_7_10_2_to_os_1_3_6.json
rename to experimental/upgrades/test_configs/snapshot_restore_es_7_10_2_to_os_1_3_6.json
diff --git a/upgrades/upgrade_testing_framework/__init__.py b/experimental/upgrades/upgrade_testing_framework/__init__.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/__init__.py
rename to experimental/upgrades/upgrade_testing_framework/__init__.py
diff --git a/upgrades/upgrade_testing_framework/robot_lib/OpenSearchRESTActions.py b/experimental/upgrades/upgrade_testing_framework/robot_lib/OpenSearchRESTActions.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/robot_lib/OpenSearchRESTActions.py
rename to experimental/upgrades/upgrade_testing_framework/robot_lib/OpenSearchRESTActions.py
diff --git a/upgrades/upgrade_testing_framework/robot_lib/__init__.py b/experimental/upgrades/upgrade_testing_framework/robot_lib/__init__.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/robot_lib/__init__.py
rename to experimental/upgrades/upgrade_testing_framework/robot_lib/__init__.py
diff --git a/upgrades/upgrade_testing_framework/robot_test_defs/__init__.py b/experimental/upgrades/upgrade_testing_framework/robot_test_defs/__init__.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/robot_test_defs/__init__.py
rename to experimental/upgrades/upgrade_testing_framework/robot_test_defs/__init__.py
diff --git a/upgrades/upgrade_testing_framework/robot_test_defs/common_upgrade_test.robot b/experimental/upgrades/upgrade_testing_framework/robot_test_defs/common_upgrade_test.robot
similarity index 100%
rename from upgrades/upgrade_testing_framework/robot_test_defs/common_upgrade_test.robot
rename to experimental/upgrades/upgrade_testing_framework/robot_test_defs/common_upgrade_test.robot
diff --git a/upgrades/upgrade_testing_framework/steps/__init__.py b/experimental/upgrades/upgrade_testing_framework/steps/__init__.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/__init__.py
rename to experimental/upgrades/upgrade_testing_framework/steps/__init__.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_bootstrap_docker.py b/experimental/upgrades/upgrade_testing_framework/steps/step_bootstrap_docker.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_bootstrap_docker.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_bootstrap_docker.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_create_source_snapshot.py b/experimental/upgrades/upgrade_testing_framework/steps/step_create_source_snapshot.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_create_source_snapshot.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_create_source_snapshot.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_load_test_config.py b/experimental/upgrades/upgrade_testing_framework/steps/step_load_test_config.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_load_test_config.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_load_test_config.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_perform_post_upgrade_test.py b/experimental/upgrades/upgrade_testing_framework/steps/step_perform_post_upgrade_test.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_perform_post_upgrade_test.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_perform_post_upgrade_test.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_perform_pre_upgrade_test.py b/experimental/upgrades/upgrade_testing_framework/steps/step_perform_pre_upgrade_test.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_perform_pre_upgrade_test.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_perform_pre_upgrade_test.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_report_results.py b/experimental/upgrades/upgrade_testing_framework/steps/step_report_results.py
similarity index 91%
rename from upgrades/upgrade_testing_framework/steps/step_report_results.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_report_results.py
index 309edecfc..5dfe177d1 100644
--- a/upgrades/upgrade_testing_framework/steps/step_report_results.py
+++ b/experimental/upgrades/upgrade_testing_framework/steps/step_report_results.py
@@ -37,12 +37,14 @@ def _run(self):
 
         self._log_results(passing_expectations, failing_expectations, untested_expectations)
 
-        readme_url = "https://github.com/opensearch-project/opensearch-migrations/blob/main/upgrades/README.md"
+        readme_url = \
+            "https://github.com/opensearch-project/opensearch-migrations/blob/main/experimental/upgrades/README.md"
         help_blurb = ("For more information about how to interpret these results, please consult the Upgrade Testing"
                       f" Framework's README file: {readme_url}")
         self.logger.info(help_blurb)
 
-        kb_url = "https://github.com/opensearch-project/opensearch-migrations/tree/main/knowledge_base"
+        kb_url = \
+            "https://github.com/opensearch-project/opensearch-migrations/tree/main/experimental/knowledge_base"
         kb_blurb = (f"You can find the expectation definitions here: {kb_url}")
         self.logger.info(kb_blurb)
 
diff --git a/upgrades/upgrade_testing_framework/steps/step_restore_source_snapshot.py b/experimental/upgrades/upgrade_testing_framework/steps/step_restore_source_snapshot.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_restore_source_snapshot.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_restore_source_snapshot.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_select_expectations.py b/experimental/upgrades/upgrade_testing_framework/steps/step_select_expectations.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_select_expectations.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_select_expectations.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_setup.py b/experimental/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_setup.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_snapshot_restore_setup.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_setup.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_teardown.py b/experimental/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_teardown.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_snapshot_restore_teardown.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_snapshot_restore_teardown.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_start_source_cluster.py b/experimental/upgrades/upgrade_testing_framework/steps/step_start_source_cluster.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_start_source_cluster.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_start_source_cluster.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_start_target_cluster.py b/experimental/upgrades/upgrade_testing_framework/steps/step_start_target_cluster.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_start_target_cluster.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_start_target_cluster.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_stop_source_cluster.py b/experimental/upgrades/upgrade_testing_framework/steps/step_stop_source_cluster.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_stop_source_cluster.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_stop_source_cluster.py
diff --git a/upgrades/upgrade_testing_framework/steps/step_stop_target_cluster.py b/experimental/upgrades/upgrade_testing_framework/steps/step_stop_target_cluster.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/steps/step_stop_target_cluster.py
rename to experimental/upgrades/upgrade_testing_framework/steps/step_stop_target_cluster.py
diff --git a/upgrades/upgrade_testing_framework/workflows/__init__.py b/experimental/upgrades/upgrade_testing_framework/workflows/__init__.py
similarity index 100%
rename from upgrades/upgrade_testing_framework/workflows/__init__.py
rename to experimental/upgrades/upgrade_testing_framework/workflows/__init__.py
diff --git a/index_configuration_tool/index_operations.py b/index_configuration_tool/index_operations.py
deleted file mode 100644
index 22385c5ad..000000000
--- a/index_configuration_tool/index_operations.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import sys
-from typing import Optional
-
-import requests
-
-# Constants
-SETTINGS_KEY = "settings"
-MAPPINGS_KEY = "mappings"
-__INDEX_KEY = "index"
-__ALL_INDICES_ENDPOINT = "*"
-__INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"]
-
-
-def fetch_all_indices(endpoint: str, optional_auth: Optional[tuple] = None, verify: bool = True) -> dict:
-    actual_endpoint = endpoint + __ALL_INDICES_ENDPOINT
-    resp = requests.get(actual_endpoint, auth=optional_auth, verify=verify)
-    # Remove internal settings
-    result = dict(resp.json())
-    for index in result:
-        for setting in __INTERNAL_SETTINGS_KEYS:
-            index_settings = result[index][SETTINGS_KEY]
-            if __INDEX_KEY in index_settings:
-                index_settings[__INDEX_KEY].pop(setting, None)
-    return result
-
-
-def create_indices(indices_data: dict, endpoint: str, auth_tuple: Optional[tuple]):
-    for index in indices_data:
-        actual_endpoint = endpoint + index
-        data_dict = dict()
-        data_dict[SETTINGS_KEY] = indices_data[index][SETTINGS_KEY]
-        data_dict[MAPPINGS_KEY] = indices_data[index][MAPPINGS_KEY]
-        try:
-            resp = requests.put(actual_endpoint, auth=auth_tuple, json=data_dict)
-            resp.raise_for_status()
-        except requests.exceptions.RequestException as e:
-            print(f"Failed to create index [{index}] - {e!s}", file=sys.stderr)
diff --git a/test/requirements.txt b/test/requirements.txt
index 3a293bea2..e23befe0b 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,4 +1,4 @@
-certifi==2023.5.7
+certifi==2023.7.22
 charset-normalizer==3.1.0
 idna==3.4
 iniconfig==2.0.0
diff --git a/test/tests.py b/test/tests.py
index ea2f7ae4a..ce27a9604 100644
--- a/test/tests.py
+++ b/test/tests.py
@@ -1,4 +1,4 @@
-from operations import create_index, check_index, create_document,\
+from operations import create_index, check_index, create_document, \
     delete_document, delete_index, get_document
 from http import HTTPStatus
 from typing import Tuple, Callable