diff --git a/FetchMigration/Dockerfile b/FetchMigration/Dockerfile index ac889edd5..db3bd0a47 100644 --- a/FetchMigration/Dockerfile +++ b/FetchMigration/Dockerfile @@ -1,5 +1,5 @@ FROM opensearch-data-prepper:2.4.0-SNAPSHOT -COPY index_configuration_tool/requirements.txt . +COPY python/requirements.txt . # Install dependencies to local user directory RUN apk update @@ -9,7 +9,7 @@ RUN pip install --user -r requirements.txt ENV ICT_CODE_PATH /code WORKDIR $ICT_CODE_PATH # Copy only source code -COPY ./index_configuration_tool/*.py . +COPY python/*.py . # update PATH ENV PATH=/root/.local:$PATH diff --git a/FetchMigration/README.md b/FetchMigration/README.md index 85772ca09..79d5ba0ab 100644 --- a/FetchMigration/README.md +++ b/FetchMigration/README.md @@ -55,7 +55,7 @@ python -m pip install -r index_configuration_tool/requirements.txt After [setup](#setup), the tool can be executed using: ```shell -python index_configuration_tool/main.py +python index_configuration_tool/pre_migration.py ``` ### Docker diff --git a/FetchMigration/index_configuration_tool/__init__.py b/FetchMigration/python/__init__.py similarity index 100% rename from FetchMigration/index_configuration_tool/__init__.py rename to FetchMigration/python/__init__.py diff --git a/FetchMigration/index_configuration_tool/dev-requirements.txt b/FetchMigration/python/dev-requirements.txt similarity index 100% rename from FetchMigration/index_configuration_tool/dev-requirements.txt rename to FetchMigration/python/dev-requirements.txt diff --git a/FetchMigration/index_configuration_tool/endpoint_info.py b/FetchMigration/python/endpoint_info.py similarity index 100% rename from FetchMigration/index_configuration_tool/endpoint_info.py rename to FetchMigration/python/endpoint_info.py diff --git a/FetchMigration/index_configuration_tool/index_operations.py b/FetchMigration/python/index_operations.py similarity index 100% rename from FetchMigration/index_configuration_tool/index_operations.py rename to FetchMigration/python/index_operations.py diff --git a/FetchMigration/index_configuration_tool/monitor.py b/FetchMigration/python/migration_monitor.py similarity index 94% rename from FetchMigration/index_configuration_tool/monitor.py rename to FetchMigration/python/migration_monitor.py index 7f9f3ba61..3b5be7781 100644 --- a/FetchMigration/index_configuration_tool/monitor.py +++ b/FetchMigration/python/migration_monitor.py @@ -7,6 +7,7 @@ from prometheus_client.parser import text_string_to_metric_families from endpoint_info import EndpointInfo +from migration_monitor_params import MigrationMonitorParams __PROMETHEUS_METRICS_ENDPOINT = "/metrics/prometheus" __SHUTDOWN_ENDPOINT = "/shutdown" @@ -53,7 +54,7 @@ def check_if_complete(doc_count: Optional[int], in_flight: Optional[int], no_par return False -def run(args: argparse.Namespace, wait_seconds: int) -> None: +def run(args: MigrationMonitorParams, wait_seconds: int = 30) -> None: # TODO Remove hardcoded EndpointInfo default_auth = ('admin', 'admin') endpoint = EndpointInfo(args.dp_endpoint, default_auth, False) @@ -97,7 +98,7 @@ def run(args: argparse.Namespace, wait_seconds: int) -> None: type=int, help="Target doc_count to reach, after which the Data Prepper pipeline will be terminated" ) - cli_args = arg_parser.parse_args() + namespace = arg_parser.parse_args() print("\n##### Starting monitor tool... #####\n") - run(cli_args, 30) + run(MigrationMonitorParams(namespace.target_count, namespace.dp_endpoint)) print("\n##### Ending monitor tool... #####\n") diff --git a/FetchMigration/python/migration_monitor_params.py b/FetchMigration/python/migration_monitor_params.py new file mode 100644 index 000000000..b147b978d --- /dev/null +++ b/FetchMigration/python/migration_monitor_params.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class MigrationMonitorParams: + target_count: int + dp_endpoint: str = "https://localhost:4900" diff --git a/FetchMigration/index_configuration_tool/main.py b/FetchMigration/python/pre_migration.py similarity index 94% rename from FetchMigration/index_configuration_tool/main.py rename to FetchMigration/python/pre_migration.py index 79e6ddd6e..eda146f06 100644 --- a/FetchMigration/index_configuration_tool/main.py +++ b/FetchMigration/python/pre_migration.py @@ -7,6 +7,8 @@ # Constants from endpoint_info import EndpointInfo +from pre_migration_params import PreMigrationParams +from pre_migration_result import PreMigrationResult SUPPORTED_ENDPOINTS = ["opensearch", "elasticsearch"] SOURCE_KEY = "source" @@ -143,12 +145,6 @@ def print_report(index_differences: tuple[set, set, set], count: int): # pragma print("Total documents to be moved: " + str(count)) -def dump_count_and_indices(count: int, indices: set): # pragma no cover - print(count) - for index_name in indices: - print(index_name) - - def compute_endpoint_and_fetch_indices(config: dict, key: str) -> tuple[EndpointInfo, dict]: endpoint = get_supported_endpoint(config, key) # Endpoint is a tuple of (type, config) @@ -157,7 +153,7 @@ def compute_endpoint_and_fetch_indices(config: dict, key: str) -> tuple[Endpoint return endpoint_info, indices -def run(args: argparse.Namespace) -> None: +def run(args: PreMigrationParams) -> PreMigrationResult: # Sanity check if not args.report and len(args.output_file) == 0: raise ValueError("No output file specified") @@ -174,14 +170,13 @@ def run(args: argparse.Namespace) -> None: diff = get_index_differences(source_indices, target_indices) # The first element in the tuple is the set of indices to create indices_to_create = diff[0] - doc_count = 0 + result = PreMigrationResult() if indices_to_create: - doc_count = index_operations.doc_count(indices_to_create, source_endpoint_info) + result.created_indices = indices_to_create + result.target_doc_count = index_operations.doc_count(indices_to_create, source_endpoint_info) if args.report: - print_report(diff, doc_count) + print_report(diff, result.target_doc_count) if indices_to_create: - if not args.report: - dump_count_and_indices(doc_count, indices_to_create) # Write output YAML if len(args.output_file) > 0: write_output(dp_config, indices_to_create, args.output_file) @@ -192,12 +187,13 @@ def run(args: argparse.Namespace) -> None: for index_name in indices_to_create: index_data[index_name] = source_indices[index_name] index_operations.create_indices(index_data, target_endpoint_info) + return result if __name__ == '__main__': # pragma no cover # Set up parsing for command line arguments arg_parser = argparse.ArgumentParser( - prog="python main.py", + prog="python pre_migration.py", description="This tool creates indices on a target cluster based on the contents of a source cluster.\n" + "The first input to the tool is a path to a Data Prepper pipeline YAML file, which is parsed to obtain " + "the source and target cluster endpoints.\nThe second input is an output path to which a modified version " + @@ -223,4 +219,5 @@ def run(args: argparse.Namespace) -> None: help="Print a report of the index differences") arg_parser.add_argument("--dryrun", action="store_true", help="Skips the actual creation of indices on the target cluster") - run(arg_parser.parse_args()) + namespace = arg_parser.parse_args() + run(PreMigrationParams(namespace.config_file_path, namespace.output_file, namespace.report, namespace.dryrun)) diff --git a/FetchMigration/python/pre_migration_params.py b/FetchMigration/python/pre_migration_params.py new file mode 100644 index 000000000..f6b01b492 --- /dev/null +++ b/FetchMigration/python/pre_migration_params.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass +class PreMigrationParams: + config_file_path: str + output_file: str = "" + report: bool = False + dryrun: bool = False diff --git a/FetchMigration/python/pre_migration_result.py b/FetchMigration/python/pre_migration_result.py new file mode 100644 index 000000000..65cac54fd --- /dev/null +++ b/FetchMigration/python/pre_migration_result.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass, field + + +@dataclass +class PreMigrationResult: + target_doc_count: int = 0 + created_indices: set = field(default_factory=set) diff --git a/FetchMigration/index_configuration_tool/requirements.txt b/FetchMigration/python/requirements.txt similarity index 100% rename from FetchMigration/index_configuration_tool/requirements.txt rename to FetchMigration/python/requirements.txt diff --git a/FetchMigration/index_configuration_tool/tests/__init__.py b/FetchMigration/python/tests/__init__.py similarity index 100% rename from FetchMigration/index_configuration_tool/tests/__init__.py rename to FetchMigration/python/tests/__init__.py diff --git a/FetchMigration/index_configuration_tool/tests/resources/expected_parse_output.pickle b/FetchMigration/python/tests/resources/expected_parse_output.pickle similarity index 100% rename from FetchMigration/index_configuration_tool/tests/resources/expected_parse_output.pickle rename to FetchMigration/python/tests/resources/expected_parse_output.pickle diff --git a/FetchMigration/index_configuration_tool/tests/resources/test_pipeline_input.yaml b/FetchMigration/python/tests/resources/test_pipeline_input.yaml similarity index 100% rename from FetchMigration/index_configuration_tool/tests/resources/test_pipeline_input.yaml rename to FetchMigration/python/tests/resources/test_pipeline_input.yaml diff --git a/FetchMigration/index_configuration_tool/tests/test_constants.py b/FetchMigration/python/tests/test_constants.py similarity index 100% rename from FetchMigration/index_configuration_tool/tests/test_constants.py rename to FetchMigration/python/tests/test_constants.py diff --git a/FetchMigration/index_configuration_tool/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py similarity index 100% rename from FetchMigration/index_configuration_tool/tests/test_index_operations.py rename to FetchMigration/python/tests/test_index_operations.py diff --git a/FetchMigration/index_configuration_tool/tests/test_monitor.py b/FetchMigration/python/tests/test_migration_monitor.py similarity index 72% rename from FetchMigration/index_configuration_tool/tests/test_monitor.py rename to FetchMigration/python/tests/test_migration_monitor.py index e52d41c62..8a7907bf1 100644 --- a/FetchMigration/index_configuration_tool/tests/test_monitor.py +++ b/FetchMigration/python/tests/test_migration_monitor.py @@ -1,4 +1,3 @@ -import argparse import unittest from unittest.mock import patch, MagicMock, PropertyMock @@ -6,10 +5,12 @@ import responses from prometheus_client.parser import text_string_to_metric_families -import monitor +import migration_monitor from endpoint_info import EndpointInfo # Constants +from migration_monitor_params import MigrationMonitorParams + TEST_ENDPOINT = "test" TEST_AUTH = ("user", "pass") TEST_FLAG = False @@ -20,12 +21,12 @@ + TEST_METRIC_NAME + "{serviceName=\"unittest\",} " + str(TEST_METRIC_VALUE) -class TestMonitor(unittest.TestCase): +class TestMigrationMonitor(unittest.TestCase): @patch('requests.post') def test_shutdown(self, mock_post: MagicMock): expected_shutdown_url = TEST_ENDPOINT + "/shutdown" test_endpoint = EndpointInfo(TEST_ENDPOINT, TEST_AUTH, TEST_FLAG) - monitor.shutdown_pipeline(test_endpoint) + migration_monitor.shutdown_pipeline(test_endpoint) mock_post.assert_called_once_with(expected_shutdown_url, auth=TEST_AUTH, verify=TEST_FLAG) @patch('requests.get') @@ -38,7 +39,7 @@ def test_fetch_prometheus_metrics(self, mock_get: MagicMock): type(mock_response).content = mock_content mock_get.return_value = mock_response # Test fetch - raw_metrics_list = monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT)) + raw_metrics_list = migration_monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT)) mock_get.assert_called_once_with(expected_url, auth=None, verify=True) self.assertEqual(1, len(raw_metrics_list)) test_metric = raw_metrics_list[0] @@ -57,7 +58,7 @@ def test_fetch_prometheus_metrics_failure(self): expected_url = TEST_ENDPOINT + "/metrics/prometheus" responses.get(expected_url, body=requests.Timeout()) # Test fetch - result = monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT)) + result = migration_monitor.fetch_prometheus_metrics(EndpointInfo(TEST_ENDPOINT)) self.assertIsNone(result) def test_get_metric_value(self): @@ -65,30 +66,28 @@ def test_get_metric_value(self): expected_val = int(TEST_METRIC_VALUE) test_input = list(text_string_to_metric_families(TEST_PROMETHEUS_METRIC_STRING)) # Should fetch by suffix - val = monitor.get_metric_value(test_input, "metric") + val = migration_monitor.get_metric_value(test_input, "metric") self.assertEqual(expected_val, val) # No matching metric returns None - val = monitor.get_metric_value(test_input, "invalid") + val = migration_monitor.get_metric_value(test_input, "invalid") self.assertEqual(None, val) - @patch('monitor.shutdown_pipeline') + @patch('migration_monitor.shutdown_pipeline') @patch('time.sleep') - @patch('monitor.check_if_complete') - @patch('monitor.get_metric_value') - @patch('monitor.fetch_prometheus_metrics') + @patch('migration_monitor.check_if_complete') + @patch('migration_monitor.get_metric_value') + @patch('migration_monitor.fetch_prometheus_metrics') # Note that mock objects are passed bottom-up from the patch order above def test_run(self, mock_fetch: MagicMock, mock_get: MagicMock, mock_check: MagicMock, mock_sleep: MagicMock, mock_shut: MagicMock): - test_input = argparse.Namespace() - # The values here don't matter since we've mocked the check method - test_input.dp_endpoint = "test" - test_input.target_count = 1 + # The param values don't matter since we've mocked the check method + test_input = MigrationMonitorParams(1, "test") mock_get.return_value = None # Check will first fail, then pass mock_check.side_effect = [False, True] # Run test method wait_time = 3 - monitor.run(test_input, wait_time) + migration_monitor.run(test_input, wait_time) # Test that fetch was called with the expected EndpointInfo expected_endpoint_info = EndpointInfo(test_input.dp_endpoint, ('admin', 'admin'), False) self.assertEqual(2, mock_fetch.call_count) @@ -97,25 +96,23 @@ def test_run(self, mock_fetch: MagicMock, mock_get: MagicMock, mock_check: Magic mock_sleep.assert_called_once_with(wait_time) mock_shut.assert_called_once_with(expected_endpoint_info) - @patch('monitor.shutdown_pipeline') + @patch('migration_monitor.shutdown_pipeline') @patch('time.sleep') - @patch('monitor.check_if_complete') - @patch('monitor.get_metric_value') - @patch('monitor.fetch_prometheus_metrics') + @patch('migration_monitor.check_if_complete') + @patch('migration_monitor.get_metric_value') + @patch('migration_monitor.fetch_prometheus_metrics') # Note that mock objects are passed bottom-up from the patch order above def test_run_with_fetch_failure(self, mock_fetch: MagicMock, mock_get: MagicMock, mock_check: MagicMock, mock_sleep: MagicMock, mock_shut: MagicMock): - test_input = argparse.Namespace() - # The values here don't matter since we've mocked the check method - test_input.dp_endpoint = "test" - test_input.target_count = 1 + # The param values don't matter since we've mocked the check method + test_input = MigrationMonitorParams(1, "test") mock_get.return_value = None mock_check.return_value = True # Fetch call will first fail, then succeed mock_fetch.side_effect = [None, MagicMock()] # Run test method wait_time = 3 - monitor.run(test_input, wait_time) + migration_monitor.run(test_input, wait_time) # Test that fetch was called with the expected EndpointInfo expected_endpoint_info = EndpointInfo(test_input.dp_endpoint, ('admin', 'admin'), False) self.assertEqual(2, mock_fetch.call_count) @@ -126,17 +123,17 @@ def test_run_with_fetch_failure(self, mock_fetch: MagicMock, mock_get: MagicMock def test_check_if_complete(self): # If any of the optional values are missing, we are not complete - self.assertFalse(monitor.check_if_complete(None, 0, 1, 0, 2)) - self.assertFalse(monitor.check_if_complete(2, None, 1, 0, 2)) - self.assertFalse(monitor.check_if_complete(2, 0, None, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(None, 0, 1, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(2, None, 1, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(2, 0, None, 0, 2)) # Target count not reached - self.assertFalse(monitor.check_if_complete(1, None, None, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(1, None, None, 0, 2)) # Target count reached, but has records in flight - self.assertFalse(monitor.check_if_complete(2, 1, None, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(2, 1, None, 0, 2)) # Target count reached, no records in flight, but no prev no_part_count - self.assertFalse(monitor.check_if_complete(2, 0, 1, 0, 2)) + self.assertFalse(migration_monitor.check_if_complete(2, 0, 1, 0, 2)) # Terminal state - self.assertTrue(monitor.check_if_complete(2, 0, 2, 1, 2)) + self.assertTrue(migration_monitor.check_if_complete(2, 0, 2, 1, 2)) if __name__ == '__main__': diff --git a/FetchMigration/index_configuration_tool/tests/test_main.py b/FetchMigration/python/tests/test_pre_migration.py similarity index 78% rename from FetchMigration/index_configuration_tool/tests/test_main.py rename to FetchMigration/python/tests/test_pre_migration.py index 7f9f1cf80..9cfe76311 100644 --- a/FetchMigration/index_configuration_tool/tests/test_main.py +++ b/FetchMigration/python/tests/test_pre_migration.py @@ -1,4 +1,3 @@ -import argparse import copy import pickle import random @@ -6,7 +5,8 @@ from typing import Optional from unittest.mock import patch, MagicMock, ANY -import main +import pre_migration +from pre_migration_params import PreMigrationParams from tests import test_constants # Constants @@ -37,32 +37,32 @@ def create_plugin_config(host_list: list[str], # Utility method to creat a test config section def create_config_section(plugin_config: dict) -> dict: valid_plugin = dict() - valid_plugin[random.choice(main.SUPPORTED_ENDPOINTS)] = plugin_config + valid_plugin[random.choice(pre_migration.SUPPORTED_ENDPOINTS)] = plugin_config config_section = copy.deepcopy(BASE_CONFIG_SECTION) config_section[TEST_KEY].append(valid_plugin) return config_section -class TestMain(unittest.TestCase): +class TestPreMigration(unittest.TestCase): # Run before each test def setUp(self) -> None: with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: self.loaded_pipeline_config = pickle.load(f) def test_is_insecure_default_value(self): - self.assertFalse(main.is_insecure({})) + self.assertFalse(pre_migration.is_insecure({})) def test_is_insecure_top_level_key(self): test_input = {"key": 123, INSECURE_KEY: True} - self.assertTrue(main.is_insecure(test_input)) + self.assertTrue(pre_migration.is_insecure(test_input)) def test_is_insecure_nested_key(self): test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}} - self.assertTrue(main.is_insecure(test_input)) + self.assertTrue(pre_migration.is_insecure(test_input)) def test_is_insecure_missing_nested(self): test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} - self.assertFalse(main.is_insecure(test_input)) + self.assertFalse(pre_migration.is_insecure(test_input)) def test_get_auth_returns_none(self): # The following inputs should not return an auth tuple: @@ -71,11 +71,11 @@ def test_get_auth_returns_none(self): # - password without user input_list = [{}, {"username": "test"}, {"password": "test"}] for test_input in input_list: - self.assertIsNone(main.get_auth(test_input)) + self.assertIsNone(pre_migration.get_auth(test_input)) def test_get_auth_for_valid_input(self): # Test valid input - result = main.get_auth({"username": "user", "password": "pass"}) + result = pre_migration.get_auth({"username": "user", "password": "pass"}) self.assertEqual(tuple, type(result)) self.assertEqual("user", result[0]) self.assertEqual("pass", result[1]) @@ -87,31 +87,31 @@ def test_get_endpoint_info(self): test_password = "password" # Simple base case test_config = create_plugin_config([host_input]) - result = main.get_endpoint_info(test_config) + result = pre_migration.get_endpoint_info(test_config) self.assertEqual(expected_endpoint, result.url) self.assertIsNone(result.auth) self.assertTrue(result.verify_ssl) # Invalid auth config test_config = create_plugin_config([host_input], test_user) - result = main.get_endpoint_info(test_config) + result = pre_migration.get_endpoint_info(test_config) self.assertEqual(expected_endpoint, result.url) self.assertIsNone(result.auth) # Valid auth config test_config = create_plugin_config([host_input], user=test_user, password=test_password) - result = main.get_endpoint_info(test_config) + result = pre_migration.get_endpoint_info(test_config) self.assertEqual(expected_endpoint, result.url) self.assertEqual(test_user, result.auth[0]) self.assertEqual(test_password, result.auth[1]) # Array of hosts uses the first entry test_config = create_plugin_config([host_input, "other_host"], test_user, test_password) - result = main.get_endpoint_info(test_config) + result = pre_migration.get_endpoint_info(test_config) self.assertEqual(expected_endpoint, result.url) self.assertEqual(test_user, result.auth[0]) self.assertEqual(test_password, result.auth[1]) def test_get_index_differences_empty(self): # Base case should return an empty list - result_tuple = main.get_index_differences(dict(), dict()) + result_tuple = pre_migration.get_index_differences(dict(), dict()) # Invariant self.assertEqual(3, len(result_tuple)) # All diffs should be empty @@ -120,7 +120,7 @@ def test_get_index_differences_empty(self): self.assertEqual(set(), result_tuple[2]) def test_get_index_differences_empty_target(self): - result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, dict()) + result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, dict()) # Invariant self.assertEqual(3, len(result_tuple)) # No conflicts or identical indices @@ -136,7 +136,7 @@ def test_get_index_differences_identical_index(self): test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) del test_data[test_constants.INDEX2_NAME] del test_data[test_constants.INDEX3_NAME] - result_tuple = main.get_index_differences(test_data, test_data) + result_tuple = pre_migration.get_index_differences(test_data, test_data) # Invariant self.assertEqual(3, len(result_tuple)) # No indices to move, or conflicts @@ -151,7 +151,7 @@ def test_get_index_differences_settings_conflict(self): # Set up conflict in settings index_settings = test_data[test_constants.INDEX2_NAME][test_constants.SETTINGS_KEY] index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 - result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) + result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) # Invariant self.assertEqual(3, len(result_tuple)) # No indices to move @@ -168,7 +168,7 @@ def test_get_index_differences_mappings_conflict(self): test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) # Set up conflict in mappings test_data[test_constants.INDEX3_NAME][test_constants.MAPPINGS_KEY] = {} - result_tuple = main.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) + result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) # Invariant self.assertEqual(3, len(result_tuple)) # No indices to move @@ -183,34 +183,34 @@ def test_get_index_differences_mappings_conflict(self): def test_validate_plugin_config_unsupported_endpoints(self): # No supported endpoints - self.assertRaises(ValueError, main.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY) + self.assertRaises(ValueError, pre_migration.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY) def test_validate_plugin_config_missing_host(self): test_data = create_config_section({}) - self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY) + self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) def test_validate_plugin_config_missing_auth(self): test_data = create_config_section(create_plugin_config(["host"])) - self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY) + self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) def test_validate_plugin_config_missing_password(self): test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=False)) - self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY) + self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) def test_validate_plugin_config_missing_user(self): test_data = create_config_section(create_plugin_config(["host"], password="test")) - self.assertRaises(ValueError, main.validate_plugin_config, test_data, TEST_KEY) + self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) def test_validate_plugin_config_auth_disabled(self): test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=True)) # Should complete without errors - main.validate_plugin_config(test_data, TEST_KEY) + pre_migration.validate_plugin_config(test_data, TEST_KEY) def test_validate_plugin_config_happy_case(self): plugin_config = create_plugin_config(["host"], "user", "password") test_data = create_config_section(plugin_config) # Should complete without errors - main.validate_plugin_config(test_data, TEST_KEY) + pre_migration.validate_plugin_config(test_data, TEST_KEY) def test_validate_pipeline_config_missing_required_keys(self): # Test cases: @@ -219,16 +219,16 @@ def test_validate_pipeline_config_missing_required_keys(self): # - missing input bad_configs = [{}, {"source": {}}, {"sink": {}}] for config in bad_configs: - self.assertRaises(ValueError, main.validate_pipeline_config, config) + self.assertRaises(ValueError, pre_migration.validate_pipeline_config, config) def test_validate_pipeline_config_happy_case(self): # Get top level value test_config = next(iter(self.loaded_pipeline_config.values())) - main.validate_pipeline_config(test_config) + pre_migration.validate_pipeline_config(test_config) @patch('index_operations.doc_count') - @patch('main.write_output') - @patch('main.print_report') + @patch('pre_migration.write_output') + @patch('pre_migration.print_report') @patch('index_operations.create_indices') @patch('index_operations.fetch_all_indices') # Note that mock objects are passed bottom-up from the patch order above @@ -251,27 +251,20 @@ def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: Ma index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 # Fetch indices is called first for source, then for target mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data] - # Set up test input - test_input = argparse.Namespace() - test_input.config_file_path = test_constants.PIPELINE_CONFIG_RAW_FILE_PATH - # Default value for missing output file - test_input.output_file = "" - test_input.report = True - test_input.dryrun = False - main.run(test_input) + test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, report=True) + pre_migration.run(test_input) mock_create_indices.assert_called_once_with(expected_create_payload, ANY) mock_doc_count.assert_called() mock_print_report.assert_called_once_with(expected_diff, 1) mock_write_output.assert_not_called() @patch('index_operations.doc_count') - @patch('main.dump_count_and_indices') - @patch('main.print_report') - @patch('main.write_output') + @patch('pre_migration.print_report') + @patch('pre_migration.write_output') @patch('index_operations.fetch_all_indices') # Note that mock objects are passed bottom-up from the patch order above def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: MagicMock, - mock_print_report: MagicMock, mock_dump: MagicMock, mock_doc_count: MagicMock): + mock_print_report: MagicMock, mock_doc_count: MagicMock): index_to_create = test_constants.INDEX1_NAME mock_doc_count.return_value = 1 expected_output_path = "dummy" @@ -280,18 +273,14 @@ def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: Magi del target_indices_data[index_to_create] # Fetch indices is called first for source, then for target mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data] - # Set up test input - test_input = argparse.Namespace() - test_input.config_file_path = test_constants.PIPELINE_CONFIG_RAW_FILE_PATH - test_input.output_file = expected_output_path - test_input.dryrun = True - test_input.report = False - main.run(test_input) + test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, expected_output_path, dryrun=True) + test_result = pre_migration.run(test_input) + self.assertEqual(mock_doc_count.return_value, test_result.target_doc_count) + self.assertEqual({index_to_create}, test_result.created_indices) mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_create}, expected_output_path) mock_doc_count.assert_called() - # Report should not be printed, but dump should be invoked + # Report should not be printed mock_print_report.assert_not_called() - mock_dump.assert_called_once_with(mock_doc_count.return_value, {index_to_create}) @patch('yaml.dump') def test_write_output(self, mock_dump: MagicMock): @@ -316,18 +305,13 @@ def test_write_output(self, mock_dump: MagicMock): del test_input['test-pipeline-input']['source']['opensearch']['indices']['include'] # Call method under test with patch('builtins.open') as mock_open: - main.write_output(test_input, {index_to_create}, expected_output_path) + pre_migration.write_output(test_input, {index_to_create}, expected_output_path) mock_open.assert_called_once_with(expected_output_path, 'w') mock_dump.assert_called_once_with(expected_output_data, ANY) def test_missing_output_file_non_report(self): - # Set up test input - test_input = argparse.Namespace() - test_input.config_file_path = test_constants.PIPELINE_CONFIG_RAW_FILE_PATH - # Default value for missing output file - test_input.output_file = "" - test_input.report = False - self.assertRaises(ValueError, main.run, test_input) + test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH) + self.assertRaises(ValueError, pre_migration.run, test_input) if __name__ == '__main__': diff --git a/FetchMigration/index_configuration_tool/tests/test_utils.py b/FetchMigration/python/tests/test_utils.py similarity index 100% rename from FetchMigration/index_configuration_tool/tests/test_utils.py rename to FetchMigration/python/tests/test_utils.py diff --git a/FetchMigration/index_configuration_tool/utils.py b/FetchMigration/python/utils.py similarity index 100% rename from FetchMigration/index_configuration_tool/utils.py rename to FetchMigration/python/utils.py