Logs project api endpoint (#1021)

* Create log api endpoints Signed-off-by: Mairi Dulaney <[email protected]> * Change TypeError to ValueError Signed-off-by: Mairi Dulaney <[email protected]> * Change to using setting rather than database for log api permission Signed-off-by: Mairi Dulaney <[email protected]> * switch to using arrow Signed-off-by: Mairi Dulaney <[email protected]> * Updates to api output Signed-off-by: Mairi Dulaney <[email protected]> * Tweak filter Signed-off-by: Mairi Dulaney <[email protected]> * Better permissions Signed-off-by: Mairi Dulaney <[email protected]> * change LOG_API_ACCESS_PROJECT_ID to OHLOG_PROJECT_ID Signed-off-by: Mairi Dulaney <[email protected]> * Updat docstrings Signed-off-by: Mairi Dulaney <[email protected]>
OpenHumans · Apr 10, 2019 · 36c2bc5 · 36c2bc5
1 parent a4d6656
commit 36c2bc5
Show file tree

Hide file tree

Showing 7 changed files with 183 additions and 6 deletions.
diff --git a/data_import/filters.py b/data_import/filters.py
@@ -0,0 +1,53 @@
+import datetime
+
+import arrow
+
+from rest_framework.filters import BaseFilterBackend
+
+
+class AccessLogFilter(BaseFilterBackend):
+    """
+    Used for filtering data returned by the custom API for OHLOG_PROJECT_ID.
+    """
+
+    def filter_queryset(self, request, queryset, view):
+
+        start_date = request.query_params.get("start_date", None)
+        end_date = request.query_params.get("end_date", None)
+        if start_date:
+            try:
+                start_date = arrow.get(start_date).datetime
+            except (TypeError, ValueError):
+                start_date = None
+        if end_date:
+            try:
+                end_date = arrow.get(end_date).datetime
+                # Special check if start_date and end_date is the same
+                # If this is the case, assume that a 24 hour period is meant, and set end_time accordingly
+                if start_date == end_date:
+                    end_date = end_date + datetime.timedelta(
+                        hours=23, minutes=59, seconds=59
+                    )
+            except (TypeError, ValueError):
+                end_date = None
+        if queryset.model.__name__ == "AWSDataFileAccessLog":
+            # AWS uses 'time' for the timestamp rather than 'date'
+            if start_date:
+                queryset = queryset.filter(time__gte=start_date)
+            if end_date:
+                queryset = queryset.filter(time__lte=end_date)
+        else:
+            if start_date:
+                queryset = queryset.filter(date__gte=start_date)
+            if end_date:
+                queryset = queryset.filter(date__lte=end_date)
+
+        datafile_id = request.query_params.get("datafile_id", None)
+        if datafile_id:
+            try:
+                datafile_id = int(datafile_id)
+                queryset = queryset.filter(serialized_data_file__id=datafile_id)
+            except ValueError:
+                pass
+
+        return queryset
diff --git a/data_import/permissions.py b/data_import/permissions.py
@@ -0,0 +1,15 @@
+from django.conf import settings
+
+from rest_framework.permissions import BasePermission
+
+
+class LogAPIAccessAllowed(BasePermission):
+    """
+    Return True if the request is from OHLOG_PROJECT_ID.
+    """
+
+    def has_permission(self, request, view):
+        if settings.OHLOG_PROJECT_ID:
+            if request.auth.id == int(settings.OHLOG_PROJECT_ID):
+                return True
+        return False
diff --git a/data_import/serializers.py b/data_import/serializers.py
@@ -1,8 +1,9 @@
 from collections import OrderedDict
+from urllib.parse import urlparse, parse_qs
 
 from rest_framework import serializers
 
-from .models import DataFile
+from .models import AWSDataFileAccessLog, DataFile, NewDataFileAccessLog
 
 
 def serialize_datafile_to_dict(datafile):
@@ -43,3 +44,46 @@ def to_representation(self, instance):
         ret["source"] = instance.source
 
         return ret
+
+
+class NewDataFileAccessLogSerializer(serializers.ModelSerializer):
+    """
+    Serialize logs of file access requests for custom API endpoint for OHLOG_PROJECT_ID
+    """
+
+    user = serializers.IntegerField(source="user.id", allow_null=True, default=None)
+    datafile = serializers.JSONField(source="serialized_data_file")
+    key = serializers.JSONField(source="data_file_key")
+
+    class Meta:  # noqa: D101
+        model = NewDataFileAccessLog
+        fields = ["date", "ip_address", "user", "datafile", "key", "aws_url"]
+
+
+class AWSDataFileAccessLogSerializer(serializers.ModelSerializer):
+    """
+    Serialize logs of AWS file access events for custom API endpoint for OHLOG_PROJECT_ID
+    """
+
+    datafile = serializers.JSONField(source="serialized_data_file")
+
+    class Meta:  # noqa: D101
+        model = AWSDataFileAccessLog
+        fields = [
+            "time",
+            "remote_ip",
+            "request_id",
+            "operation",
+            "bucket_key",
+            "request_uri",
+            "status",
+            "bytes_sent",
+            "object_size",
+            "total_time",
+            "turn_around_time",
+            "referrer",
+            "user_agent",
+            "cipher_suite",
+            "host_header",
+            "datafile",
+        ]
diff --git a/data_import/urls.py b/data_import/urls.py
@@ -1,6 +1,10 @@
-from django.urls import re_path
+from django.urls import path, re_path
 
-from .views import DataFileDownloadView
+from .views import (
+    AWSDataFileAccessLogView,
+    DataFileDownloadView,
+    NewDataFileAccessLogView,
+)
 
 app_name = "data-management"
 
@@ -9,5 +13,16 @@
         r"^datafile-download/(?P<pk>[0-9]+)/",
         DataFileDownloadView.as_view(),
         name="datafile-download",
-    )
+    ),
+    # Custom API endpoints for OHLOG_PROJECT_ID
+    path(
+        "awsdatafileaccesslog/",
+        AWSDataFileAccessLogView.as_view(),
+        name="awsdatafileaccesslog",
+    ),
+    path(
+        "newdatafileaccesslog/",
+        NewDataFileAccessLogView.as_view(),
+        name="newdatafileaccesslog",
+    ),
 ]
diff --git a/data_import/views.py b/data_import/views.py
@@ -4,10 +4,21 @@
 from django.http import HttpResponseForbidden, HttpResponseRedirect
 from django.views.generic import View
 
+from django_filters.rest_framework import DjangoFilterBackend
 from ipware.ip import get_ip
+from rest_framework.generics import ListAPIView
 
-from .models import DataFile, DataFileKey, NewDataFileAccessLog
-from data_import.serializers import serialize_datafile_to_dict
+from .filters import AccessLogFilter
+from .models import AWSDataFileAccessLog, DataFile, DataFileKey, NewDataFileAccessLog
+from .permissions import LogAPIAccessAllowed
+from common.mixins import NeverCacheMixin
+from data_import.serializers import (
+    AWSDataFileAccessLogSerializer,
+    NewDataFileAccessLogSerializer,
+    serialize_datafile_to_dict,
+)
+from private_sharing.api_authentication import CustomOAuth2Authentication
+from private_sharing.api_permissions import HasValidProjectToken
 
 UserModel = get_user_model()
 
@@ -75,3 +86,39 @@ def get(self, request, *args, **kwargs):
         return HttpResponseForbidden(
             "<h1>You are not authorized to view this file.</h1>"
         )
+
+
+class NewDataFileAccessLogView(NeverCacheMixin, ListAPIView):
+    """
+    Custom API endpoint returning logs of file access requests for OHLOG_PROJECT_ID
+    """
+
+    authentication_classes = (CustomOAuth2Authentication,)
+    filter_backends = (AccessLogFilter, DjangoFilterBackend)
+    filterset_fields = ("date",)
+    permission_classes = (HasValidProjectToken, LogAPIAccessAllowed)
+    serializer_class = NewDataFileAccessLogSerializer
+
+    def get_queryset(self):
+        queryset = NewDataFileAccessLog.objects.filter(
+            serialized_data_file__user_id=self.request.user.id
+        )
+        return queryset
+
+
+class AWSDataFileAccessLogView(NeverCacheMixin, ListAPIView):
+    """
+    Custom API endpoint returning logs of AWS file access events for OHLOG_PROJECT_ID
+    """
+
+    authentication_classes = (CustomOAuth2Authentication,)
+    filter_backends = (AccessLogFilter, DjangoFilterBackend)
+    filterset_fields = ("time",)
+    permission_classes = (HasValidProjectToken, LogAPIAccessAllowed)
+    serializer_class = AWSDataFileAccessLogSerializer
+
+    def get_queryset(self):
+        queryset = AWSDataFileAccessLog.objects.filter(
+            serialized_data_file__user_id=self.request.user.id
+        )
+        return queryset
diff --git a/open_humans/settings.py b/open_humans/settings.py
@@ -474,6 +474,8 @@ def to_bool(env, default="false"):
 RECAPTCHA_PUBLIC_KEY = os.getenv("RECAPTCHA_PUBLIC_KEY", "")
 RECAPTCHA_PRIVATE_KEY = os.getenv("RECAPTCHA_PRIVATE_KEY", "")
 
+OHLOG_PROJECT_ID = os.getenv("OHLOG_PROJECT_ID", None)
+
 ZAPIER_WEBHOOK_URL = os.getenv("ZAPIER_WEBHOOK_URL")
 
 MAX_UNAPPROVED_MEMBERS = int(os.getenv("MAX_UNAPPROVED_MEMBERS", "20"))

diff --git a/requirements.in b/requirements.in
@@ -37,6 +37,7 @@ Markdown
 mock
 Pillow  # for sorl-thumbnail
 pyparsing
+python-dateutil
 raven
 redis
 requests