[Console API] Add basic get status API (#903)

This change introduces a simple get status API to our Console API for retrieving the status of an OSI migration. This logic currently only returns the status and statusMessage from the OSI getPipeline API with the anticipation to add more details like progress in future iterations.

Also included is some reformatting of the response given by each of the API endpoints to keep a consistent response structure on error.

---------

Signed-off-by: Tanner Lewis <[email protected]>

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/README.md

@@ -0,0 +1,23 @@
+# Console API
+
+### Running Tests
+
+If pipenv is not installed, install with below
+```shell
+python3 -m pip install --upgrade pipenv
+```
+
+Install dependencies
+```shell
+pipenv install --deploy --dev
+```
+
+Run test cases
+```shell
+pipenv run coverage run --source='.' manage.py test console_api
+```
+
+Generate coverage report
+```shell
+pipenv run coverage report
+```

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/console_api/apps/orchestrator/serializers.py

@@ -32,6 +32,6 @@ class OpenSearchIngestionCreateRequestSerializer(serializers.Serializer):
     )
 
 
-class OpenSearchIngestionUpdateRequestSerializer(serializers.Serializer):
+class OpenSearchIngestionDefaultRequestSerializer(serializers.Serializer):
     PipelineName = serializers.CharField(max_length=28)
     PipelineManagerAssumeRoleArn = serializers.CharField(max_length=2048, required=False)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/console_api/apps/orchestrator/tests/test_views.py

@@ -2,10 +2,13 @@
 import botocore
 from django.test import Client, SimpleTestCase
 from moto import mock_aws
+import json
 from rest_framework import status
 from unittest.mock import patch
 
 PIPELINE_NAME = 'test-pipeline'
+RESPONSE_STATUS = 'STARTING'
+RESPONSE_STATUS_REASON = {'Description': 'Pipeline is starting'}
 VALID_CREATE_PAYLOAD = {
     'PipelineRoleArn': 'arn:aws:iam::123456789012:role/pipeline-access-role',
     'PipelineName': PIPELINE_NAME,
@@ -46,6 +49,10 @@ def mock_make_api_call(self, operation_name, kwarg):
     if (operation_name == 'CreatePipeline' or operation_name == 'DeletePipeline' or
             operation_name == 'StartPipeline' or operation_name == 'StopPipeline'):
         return {'Pipeline': {'PipelineName': PIPELINE_NAME}}
+    elif operation_name == 'GetPipeline':
+        return {'Pipeline': {'PipelineName': PIPELINE_NAME,
+                             'Status': RESPONSE_STATUS,
+                             'StatusReason': RESPONSE_STATUS_REASON}}
     # If we don't want to patch the API call
     return orig(self, operation_name, kwarg)
 
@@ -61,7 +68,7 @@ def test_osi_create_migration(self):
         response = self.client.post('/orchestrator/osi-create-migration', data=VALID_CREATE_PAYLOAD,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @mock_aws
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
@@ -71,7 +78,7 @@ def test_osi_create_migration_assume_role(self):
         response = self.client.post('/orchestrator/osi-create-migration', data=payload,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
     def test_osi_create_migration_fails_for_missing_field(self):
@@ -95,7 +102,7 @@ def test_osi_start_migration(self):
                                     content_type='application/json')
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @mock_aws
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
@@ -104,34 +111,59 @@ def test_osi_start_migration_assume_role(self):
                                     content_type='application/json')
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
     def test_osi_stop_migration(self):
         response = self.client.post('/orchestrator/osi-stop-migration', data=VALID_UPDATE_PAYLOAD,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @mock_aws
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
     def test_osi_stop_migration_assume_role(self):
         response = self.client.post('/orchestrator/osi-stop-migration', data=VALID_ASSUME_ROLE_UPDATE_PAYLOAD,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
     def test_osi_delete_migration(self):
         response = self.client.post('/orchestrator/osi-delete-migration', data=VALID_UPDATE_PAYLOAD,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
 
     @mock_aws
     @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
     def test_osi_delete_migration_assume_role(self):
         response = self.client.post('/orchestrator/osi-delete-migration', data=VALID_ASSUME_ROLE_UPDATE_PAYLOAD,
                                     content_type='application/json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn('Timestamp', response.data)
+        self.assertIn('timestamp', response.json())
+
+    @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
+    def test_osi_get_status_migration(self):
+        response = self.client.generic(method='GET',
+                                       path='/orchestrator/osi-get-status-migration',
+                                       data=json.dumps(VALID_UPDATE_PAYLOAD),
+                                       content_type='application/json')
+        data = response.json()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertIn('timestamp', data)
+        self.assertEqual(data['status'], RESPONSE_STATUS)
+        self.assertEqual(data['statusMessage'], RESPONSE_STATUS_REASON['Description'])
+
+    @mock_aws
+    @patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call)
+    def test_osi_get_status_assume_role(self):
+        response = self.client.generic(method='GET',
+                                       path='/orchestrator/osi-get-status-migration',
+                                       data=json.dumps(VALID_ASSUME_ROLE_UPDATE_PAYLOAD),
+                                       content_type='application/json')
+        data = response.json()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertIn('timestamp', data)
+        self.assertEqual(data['status'], RESPONSE_STATUS)
+        self.assertEqual(data['statusMessage'], RESPONSE_STATUS_REASON['Description'])

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/console_api/apps/orchestrator/urls.py

@@ -6,4 +6,5 @@
     path('osi-start-migration', views.osi_start_migration, name='osi-start-migration'),
     path('osi-stop-migration', views.osi_stop_migration, name='osi-stop-migration'),
     path('osi-delete-migration', views.osi_delete_migration, name='osi-delete-migration'),
+    path('osi-get-status-migration', views.osi_get_status_migration, name='osi-get-status-migration'),
 ]

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/console_api/apps/orchestrator/views.py

@@ -1,15 +1,16 @@
 from console_api.apps.orchestrator.serializers import (OpenSearchIngestionCreateRequestSerializer,
-                                                       OpenSearchIngestionUpdateRequestSerializer)
+                                                       OpenSearchIngestionDefaultRequestSerializer)
 from console_link.models.osi_utils import (InvalidAuthParameters, create_pipeline_from_json, start_pipeline,
-                                           stop_pipeline, delete_pipeline, get_assume_role_session)
+                                           stop_pipeline, delete_pipeline, get_status, get_assume_role_session)
+from django.http import JsonResponse
 from rest_framework.decorators import api_view, parser_classes
 from rest_framework.parsers import JSONParser
-from rest_framework.response import Response
 from rest_framework import status
 from pathlib import Path
 from typing import Callable
 import boto3
 import datetime
+import json
 import logging
 
 logger = logging.getLogger(__name__)
@@ -52,20 +53,21 @@ def osi_update_workflow(request, osi_action_func: Callable, action_name: str):
     request_data = request.data
     logger.info(pretty_request(request, request_data))
 
-    osi_serializer = OpenSearchIngestionUpdateRequestSerializer(data=request_data)
+    osi_serializer = OpenSearchIngestionDefaultRequestSerializer(data=request_data)
     osi_serializer.is_valid(raise_exception=True)
     pipeline_name = request_data.get('PipelineName')
+    response_data = {
+        'timestamp': datetime.datetime.now(datetime.timezone.utc)
+    }
     try:
         osi_action_func(osi_client=determine_osi_client(request_data=request_data, action_name=action_name),
                         pipeline_name=pipeline_name)
     except Exception as e:
         logger.error(f'Error performing OSI {action_name} Pipeline API: {e}')
-        return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        response_data['error'] = str(e)
+        return JsonResponse(response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-    response_data = {
-        'Timestamp': datetime.datetime.now(datetime.timezone.utc)
-    }
-    return Response(response_data, status=status.HTTP_200_OK)
+    return JsonResponse(response_data, status=status.HTTP_200_OK)
 
 
 @api_view(['POST'])
@@ -77,21 +79,23 @@ def osi_create_migration(request):
 
     osi_serializer = OpenSearchIngestionCreateRequestSerializer(data=request_data)
     osi_serializer.is_valid(raise_exception=True)
+    response_data = {
+        'timestamp': datetime.datetime.now(datetime.timezone.utc)
+    }
     try:
         create_pipeline_from_json(osi_client=determine_osi_client(request_data=request_data, action_name=action_name),
                                   input_json=request_data,
                                   pipeline_template_path=PIPELINE_TEMPLATE_PATH)
     except InvalidAuthParameters as i:
         logger.error(f'Error performing OSI {action_name} Pipeline API: {i}')
-        return Response(str(i), status=status.HTTP_400_BAD_REQUEST)
+        response_data['error'] = str(i)
+        return JsonResponse(response_data, status=status.HTTP_400_BAD_REQUEST)
     except Exception as e:
         logger.error(f'Error performing OSI {action_name} Pipeline API: {e}')
-        return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        response_data['error'] = str(e)
+        return JsonResponse(response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-    response_data = {
-        'Timestamp': datetime.datetime.now(datetime.timezone.utc)
-    }
-    return Response(response_data, status=status.HTTP_200_OK)
+    return JsonResponse(response_data, status=status.HTTP_200_OK)
 
 
 @api_view(['POST'])
@@ -110,3 +114,35 @@ def osi_stop_migration(request):
 @parser_classes([JSONParser])
 def osi_delete_migration(request):
     return osi_update_workflow(request=request, osi_action_func=delete_pipeline, action_name='Delete')
+
+
+@api_view(['GET'])
+@parser_classes([JSONParser])
+def osi_get_status_migration(request):
+    response_data = {
+        'timestamp': datetime.datetime.now(datetime.timezone.utc)
+    }
+    action_name = 'GetStatus'
+    try:
+        # Read the raw body data, necessary for GET requests with a body in Django
+        body_unicode = request.body.decode('utf-8')
+        request_data = json.loads(body_unicode)
+    except json.JSONDecodeError as jde:
+        logger.error(f'Error performing OSI {action_name} Pipeline API: {jde}')
+        response_data['error'] = f'Unable to parse JSON. Exception: {jde}'
+        return JsonResponse(data=response_data, status=status.HTTP_400_BAD_REQUEST)
+
+    logger.info(pretty_request(request, request_data))
+    osi_serializer = OpenSearchIngestionDefaultRequestSerializer(data=request_data)
+    osi_serializer.is_valid(raise_exception=True)
+    pipeline_name = request_data.get('PipelineName')
+    try:
+        status_dict = get_status(osi_client=determine_osi_client(request_data, action_name),
+                                 pipeline_name=pipeline_name)
+    except Exception as e:
+        logger.error(f'Error performing OSI {action_name} Pipeline API: {e}')
+        response_data['error'] = str(e)
+        return JsonResponse(response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    response_data.update(**status_dict)
+    return JsonResponse(response_data, status=status.HTTP_200_OK)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/osi_utils.py

@@ -200,6 +200,20 @@ def construct_pipeline_config(pipeline_config_file_path: str, source_endpoint: s
     return pipeline_config
 
 
+# TODO: Reconcile status with internal status (https://opensearch.atlassian.net/browse/MIGRATIONS-1958)
+def get_status(osi_client, pipeline_name: str):
+    name = pipeline_name if pipeline_name is not None else DEFAULT_PIPELINE_NAME
+    logger.info(f"Getting status of pipeline: {name}")
+    get_pipeline_response = osi_client.get_pipeline(
+        PipelineName=name
+    )
+
+    return {
+        'status': get_pipeline_response['Pipeline']['Status'],
+        'statusMessage': get_pipeline_response['Pipeline']['StatusReason']['Description']
+    }
+
+
 def start_pipeline(osi_client, pipeline_name: str):
     name = pipeline_name if pipeline_name is not None else DEFAULT_PIPELINE_NAME
     logger.info(f"Starting pipeline: {name}")

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_osi_utils.py

@@ -11,7 +11,8 @@
                                            delete_pipeline,
                                            get_assume_role_session,
                                            start_pipeline,
-                                           stop_pipeline)
+                                           stop_pipeline,
+                                           get_status)
 from console_link.models.cluster import AuthMethod
 from moto import mock_aws
 from tests.utils import create_valid_cluster
@@ -270,6 +271,26 @@ def test_valid_delete_pipeline(osi_client_stubber):
     osi_client_stubber.assert_no_pending_responses()
 
 
+def test_valid_get_status_pipeline(osi_client_stubber):
+    expected_request_body = {'PipelineName': f'{PIPELINE_NAME}'}
+    response_status = 'UPDATING'
+    response_status_reason = {'Description': 'Pipeline is being updated'}
+    service_response_body = {
+        'Pipeline':
+            {
+                'PipelineName': PIPELINE_NAME,
+                'Status': response_status,
+                'StatusReason': response_status_reason
+            }
+    }
+    osi_client_stubber.add_response("get_pipeline", service_response_body, expected_request_body)
+    osi_client_stubber.activate()
+
+    get_status(osi_client=osi_client_stubber.client, pipeline_name=PIPELINE_NAME)
+
+    osi_client_stubber.assert_no_pending_responses()
+
+
 @mock_aws
 def test_valid_get_assume_role_session():
     session_name = 'unittest-session'