Pipeline: add transfer methods

Clean up the code a bit, mainly `api/sword/helpers.py` which was not using the
Pipeline model to interact with the API. Other minor changes have been added.

storage_service/locations/api/sword/helpers.py

@@ -101,8 +101,8 @@ def download_resource(url, destination_path, filename=None, username=None, passw
     if username is not None and password is not None:
         auth = (username, password)
 
-    response = requests.get(url, auth=auth,
-                            verify=not settings.INSECURE_SKIP_VERIFY)
+    verify = not settings.INSECURE_SKIP_VERIFY
+    response = requests.get(url, auth=auth, verify=verify)
     if filename is None:
         if 'content-disposition' in response.headers:
             filename = parse_filename_from_content_disposition(response.headers['content-disposition'])
@@ -334,49 +334,37 @@ def activate_transfer_and_request_approval_from_pipeline(deposit, pipeline):
     destination_path = pad_destination_filepath_if_it_already_exists(destination_path)
     shutil.move(deposit.full_path, destination_path)
 
-    params = {
-        'username': pipeline.api_username,
-        'api_key': pipeline.api_key
-    }
-    url = pipeline.parse_url()
-    url = url._replace(path='api/transfer/unapproved/').geturl()
+    # Find the path of the transfer that we want to approve.
     while True:
-        response = requests.get(url, params=params,
-                                verify=not settings.INSECURE_SKIP_VERIFY)
-        if response.status_code == 200:
-            results = response.json()
+        try:
+            results = pipeline.list_unapproved_transfers()
+        except Exception:
+            LOGGER.exception('Retrieval of unapproved transfers failed')
         else:
-            raise Exception(_("Dashboard returned %(status_code)s: %(text)s") % {'status_code': response.status_code, 'text': response.text})
-
-        directories = [result['directory'] for result in results['results'] if result['type'] == 'standard']
-        if deposit.current_path in directories:
-            break
+            directories = [
+                result['directory'] for result in results['results']
+                if result['type'] == 'standard']
+            if deposit.current_path in directories:
+                break
         time.sleep(5)
 
-    # make request to pipeline's transfer approval API
-    data = {
-        'username': pipeline.api_username,
-        'api_key': pipeline.api_key,
-        'directory': deposit.current_path,
-        'type': 'standard'
-    }
-
-    url = pipeline.parse_url()
-    url = url._replace(path='api/transfer/approve/').geturl()
+    # Approve transfer.
     try:
-        response = requests.post(url, data=data,
-                                 verify=not settings.INSECURE_SKIP_VERIFY)
+        results = pipeline.approve_transfer(deposit.current_path, type='standard')
     except Exception:
-        LOGGER.exception('Automatic approval of transfer for deposit %s failed', deposit.uuid)
-        # move back to deposit directory
-        # FIXME moving the files out from under Archivematica leaves a transfer that will always error out - leave it?
+        LOGGER.exception(
+            'Automatic approval of transfer for deposit %s failed',
+            deposit.uuid)
+        # Move back to deposit directory. FIXME: moving the files out form under
+        # Archivematica leaves a transfer that will always error out - leave it?
         shutil.move(destination_path, deposit.full_path)
         return {
             'error': True,
-            'message': _('Request to pipeline %(uuid)s transfer approval API failed: check credentials and REST API IP whitelist.') % {'uuid': pipeline.uuid},
+            'message': _('Request to pipeline %(uuid)s transfer approval API '
+                         'failed: check credentials and REST API IP '
+                         'whitelist.') % {'uuid': pipeline.uuid},
         }
-    result = response.json()
-    return result
+    return results
 
 
 def sword_error_response(request, status, summary):

storage_service/locations/fixtures/pipelines.json

@@ -23,6 +23,18 @@
         "description": "Test"
     }
 },
+{
+    "pk": 3,
+    "model": "locations.pipeline",
+    "fields": {
+        "api_key": "test",
+        "uuid": "1b6de7e3-0a72-4b23-8451-3ac858cc4ce4",
+        "enabled": true,
+        "api_username": "test",
+        "remote_name": "http://archivematica-dashboard:8000",
+        "description": "Test"
+    }
+},
 {
     "pk": 1, 
     "model": "locations.locationpipeline", 

storage_service/locations/fixtures/vcr_cassettes/pipeline_approve_transfer.yaml

@@ -0,0 +1,24 @@
+interactions:
+- request:
+    body: !!python/unicode directory=Foobar1&type=standard
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Authorization: ['ApiKey test:test']
+      Connection: [keep-alive]
+      Content-Length: ['31']
+      Content-Type: [application/x-www-form-urlencoded]
+      User-Agent: [python-requests/2.14.2]
+    method: POST
+    uri: http://archivematica-dashboard:8000/api/transfer/approve/
+  response:
+    body: {string: !!python/unicode '{"message": "Approval successful.", "uuid": "090b7f5b-637b-400b-9014-3eb58986fe8f"}'}
+    headers:
+      connection: [keep-alive]
+      content-language: [en]
+      content-type: [application/json]
+      date: ['Wed, 29 Nov 2017 22:36:35 GMT']
+      server: [gunicorn/19.7.1]
+      vary: ['Accept-Language, Cookie']
+    status: {code: 200, message: OK}
+version: 1

storage_service/locations/fixtures/vcr_cassettes/pipeline_list_unapproved_transfers.yaml

@@ -0,0 +1,23 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Authorization: ['ApiKey test:test']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.14.2]
+    method: GET
+    uri: http://archivematica-dashboard:8000/api/transfer/unapproved/
+  response:
+    body: {string: !!python/unicode '{"message": "Fetched unapproved transfers successfully.",
+        "results": [{"directory": "Foobar1", "type": "standard", "uuid": "090b7f5b-637b-400b-9014-3eb58986fe8f"}]}'}
+    headers:
+      connection: [keep-alive]
+      content-language: [en]
+      content-type: [application/json]
+      date: ['Wed, 29 Nov 2017 22:28:33 GMT']
+      server: [gunicorn/19.7.1]
+      vary: ['Accept-Language, Cookie']
+    status: {code: 200, message: OK}
+version: 1

storage_service/locations/models/pipeline.py

@@ -73,7 +73,7 @@ def save(self, create_default_locations=False, shared_path=None, *args, **kwargs
         if create_default_locations:
             self.create_default_locations(shared_path)
 
-    def parse_url(self):
+    def parse_and_fix_url(self):
         """Returns a ParseResult object based on the remote_name field.
 
         We've always made the assumption that the value of the remote_name field
@@ -161,19 +161,18 @@ def create_default_locations(self, shared_path=None):
     # HTTP API CALLS
 
     def _request_api(self, method, path, fields=None):
-        url = self.parse_url()
-        url = url._replace(path='api/{}'.format(path)).geturl()
-        data = {
-            'username': self.api_username,
-            'api_key': self.api_key
-        }
-        if fields:
-            data.update(fields)
-        LOGGER.debug('URL: %s; data: %s', url, data)
+        api_url = self.parse_and_fix_url()
+        api_url = api_url._replace(path='api/{}'.format(path)).geturl()
+        headers = {'Authorization': 'ApiKey {}:{}'.format(
+            self.api_username,
+            self.api_key,
+        )}
+        LOGGER.debug('URL: %s; headers %s; data: %s', api_url, headers, fields)
         try:
-            resp = requests.request(method, url,
-                                    data=data, allow_redirects=True,
-                                    verify=not settings.INSECURE_SKIP_VERIFY)
+            verify = not settings.INSECURE_SKIP_VERIFY
+            resp = requests.request(method, api_url, headers=headers,
+                                    data=fields, allow_redirects=True,
+                                    verify=verify)
         except requests.exceptions.RequestException:
             LOGGER.exception('Unable to connect to pipeline %s.', self)
             raise
@@ -209,3 +208,30 @@ def reingest(self, name, uuid, target='transfer'):
             except ValueError:  # Failed to decode JSON
                 raise requests.exceptions.RequestException(_('Pipeline %(pipeline)s returned an unexpected status code: %(status_code)s') % {'pipeline': self, 'status_code': resp.status_code})
         return resp.json()
+
+    def approve_transfer(self, directory, transfer_type):
+        """Approve a transfer in the pipeline."""
+        url = 'transfer/approve/'
+        fields = {'directory': directory, 'type': transfer_type}
+        resp = self._request_api('POST', url, fields=fields)
+        if resp.status_code == requests.codes.ok:
+            return resp.json()
+        raise requests.exceptions.RequestException(
+            _('Pipeline %(pipeline)s could not approve the transfer: '
+              '%(status_code)s (%(text)s)') % {
+                'pipeline': self,
+                'status_code': resp.status_code,
+                'text': resp.text})
+
+    def list_unapproved_transfers(self):
+        """List the existing unapproved transfers."""
+        url = 'transfer/unapproved/'
+        resp = self._request_api('GET', url)
+        if resp.status_code == requests.codes.ok:
+            return resp.json()
+        raise requests.exceptions.RequestException(
+            _('Pipeline %(pipeline)s could not list unapproved transfers: '
+              '%(status_code)s (%(text)s)') % {
+                'pipeline': self,
+                'status_code': resp.status_code,
+                'text': resp.text})

storage_service/locations/tests/test_api.py

@@ -346,7 +346,7 @@ def test_pipeline_create(self):
         assert response.status_code == 201
 
         pipeline = models.Pipeline.objects.get(uuid=data['uuid'])
-        pipeline.parse_url() == urlparse(data['remote_name'])
+        pipeline.parse_and_fix_url() == urlparse(data['remote_name'])
 
         # When undefined the remote_name field should be populated after the
         # REMOTE_ADDR header.
@@ -358,4 +358,4 @@ def test_pipeline_create(self):
                                     REMOTE_ADDR='192.168.0.10')
         assert response.status_code == 201
         pipeline = models.Pipeline.objects.get(uuid=data['uuid'])
-        pipeline.parse_url() == urlparse('http://192.168.0.10')
+        pipeline.parse_and_fix_url() == urlparse('http://192.168.0.10')

storage_service/locations/tests/test_pipeline.py

@@ -1,33 +1,40 @@
+import os
+
 from django.test import TestCase
 from django.utils.six.moves.urllib.parse import ParseResult, urlparse
 
 from locations import models
 
 import mock
+import vcr
+
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+FIXTURES_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', 'fixtures'))
 
 
 class TestPipeline(TestCase):
 
     fixtures = ['base.json', 'pipelines.json']
 
-    def test_parse_url(self):
+    def test_parse_and_fix_url(self):
         pipeline = models.Pipeline.objects.get(pk=1)
-        res = pipeline.parse_url()
+        res = pipeline.parse_and_fix_url()
         assert isinstance(res, ParseResult)
         assert res.geturl() == 'http://127.0.0.1'
 
         pipeline = models.Pipeline.objects.get(pk=2)
-        res = pipeline.parse_url()
+        res = pipeline.parse_and_fix_url()
         assert res == urlparse('')
 
         url = 'https://archivematica-dashboard'
         pipeline.remote_name = url
-        assert pipeline.parse_url() == \
+        assert pipeline.parse_and_fix_url() == \
             urlparse(url)
 
         url = 'https://foo@bar:ss.qa.usip.tld:1234/dev/'
         pipeline.remote_name = url
-        assert pipeline.parse_url() == \
+        assert pipeline.parse_and_fix_url() == \
             urlparse(url)
 
     @mock.patch('requests.request')
@@ -36,17 +43,44 @@ def test_request_api(self, request):
 
         method = 'GET'
         url = 'http://127.0.0.1/api/processing-configuration/default'
-        data = {'username': None, 'api_key': None}
+        headers = {'Authorization': 'ApiKey None:None'}
 
         pipeline._request_api(method, 'processing-configuration/default')
         request.assert_called_with(method, url,
                                    allow_redirects=True,
-                                   data=data,
+                                   data=None,
+                                   headers=headers,
                                    verify=True)
 
         with self.settings(INSECURE_SKIP_VERIFY=True):
             pipeline._request_api(method, 'processing-configuration/default')
             request.assert_called_with(method, url,
                                        allow_redirects=True,
-                                       data=data,
+                                       data=None,
+                                       headers=headers,
                                        verify=False)
+
+    @vcr.use_cassette(os.path.join(
+        FIXTURES_DIR,
+        'vcr_cassettes', 'pipeline_list_unapproved_transfers.yaml'))
+    def test_list_unapproved_transfers(self):
+        pipeline = models.Pipeline.objects.get(pk=3)
+        result = pipeline.list_unapproved_transfers()
+
+        assert isinstance(result, dict) is True
+        assert result['message'] == 'Fetched unapproved transfers successfully.'
+        assert len(result['results']) == 1
+        assert result['results'][0]['directory'] == 'Foobar1'
+        assert result['results'][0]['type'] == 'standard'
+        assert result['results'][0]['uuid'] == \
+            '090b7f5b-637b-400b-9014-3eb58986fe8f'
+
+    @vcr.use_cassette(os.path.join(
+        FIXTURES_DIR,
+        'vcr_cassettes', 'pipeline_approve_transfer.yaml'))
+    def test_approve_transfer(self):
+        pipeline = models.Pipeline.objects.get(pk=3)
+        result = pipeline.approve_transfer('Foobar1', 'standard')
+
+        assert result['message'] == 'Approval successful.'
+        assert result['uuid'] == '090b7f5b-637b-400b-9014-3eb58986fe8f'