Merge pull request #231 from EBI-Metagenomics/develop

Develop
EBI-Metagenomics · Sep 28, 2021 · 3081bcc · 3081bcc
2 parents f3f4567 + 0cf61b6
commit 3081bcc
Show file tree

Hide file tree

Showing 103 changed files with 1,734 additions and 416 deletions.
diff --git a/ci/configuration.yaml b/ci/configuration.yaml
@@ -12,3 +12,8 @@ emg:
   mongodb:
       db: 'testdb'
       host: '127.0.0.1'
+  sourmash:
+    signatures_path: 'fixtures/'
+    results_path: 'fixtures/'
+    celery_broker: 'redis://localhost:6379/0'
+    celery_backend: 'redis://localhost:6379/1'
diff --git a/config/local.yml b/config/local.yml
@@ -8,26 +8,32 @@ emg:
       PORT: 3306
 
   admin: True
+  downloads_bypass_nginx: True
   auth_backends: 
     - 'emgapi.backends.EMGBackend'
     - 'django.contrib.auth.backends.ModelBackend'
 
   cors_origin_allow_all: true
   debug: true
-  results_dir: '/opt/results'
-  static_root: '/opt/staticfiles'
+  results_dir: '/opt/emgapi/results'
+  static_root: '/opt/emgapi/staticfiles'
   emg_backend_auth: 'https://www.ebi.ac.uk/ena/auth/login'
   secure_cookies: false
 
   mongodb:
     db: emg
     host: mongodb
 
+  sourmash:
+    signatures_path: '/tmp/signatures'
+    results_path: '/tmp/results'
+    celery_broker: 'redis://localhost:6379/0'
+    celery_backend: 'redis://localhost:6379/1'
   documentation:
     title: 'MGnify API'
     external_docs_url: 'https://emg-docs.readthedocs.io/en/latest/api.html'
     external_docs_description: 'MGnify documentation – API section'
     description:
       >
         General MGnify documentation is available on Read the Docs:
-        https://emg-docs.readthedocs.io/
+        https://emg-docs.readthedocs.io/
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -12,9 +12,9 @@ RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results
 
 COPY requirements* /opt/emgapi/
 
-RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" && \
-    pip3 install -r /opt/emgapi/requirements-dev.txt && \
-    pip3 install -r /opt/emgapi/requirements-admin.txt
+RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
+RUN pip3 install -r /opt/emgapi/requirements-dev.txt
+RUN pip3 install -r /opt/emgapi/requirements-admin.txt
 
 ENV PYTHONPATH="${PYTHONPATH}:/opt/emgapi/emgcli"
 

diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile
@@ -12,10 +12,10 @@ RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results
 
 COPY requirements* /opt/emgapi/
 
-RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" && \
-    pip3 install -r /opt/emgapi/requirements-dev.txt && \
-    pip3 install -r /opt/emgapi/requirements-test.txt && \
-    pip3 install -r /opt/emgapi/requirements-admin.txt
+RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
+RUN pip3 install -r /opt/emgapi/requirements-dev.txt
+RUN pip3 install -r /opt/emgapi/requirements-test.txt
+RUN pip3 install -r /opt/emgapi/requirements-admin.txt
 
 ENV PYTHONPATH="${PYTHONPATH}:/opt/emgapi/emgcli"
 

diff --git a/emgapi/admin.py b/emgapi/admin.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from django.contrib import admin
+from django.contrib import admin, messages
 from django import forms
 
 from .widgets import BiomePredictionWidget
@@ -493,10 +493,6 @@ class GenomeSetAdmin(admin.ModelAdmin):
     ]
 
 
-class GenomeReleasesInline(admin.TabularInline):
-    model = emg_models.Genome.releases.through
-
-
 class GenomeDownloads(admin.TabularInline):
     model = emg_models.GenomeDownload
     raw_id_fields = [
@@ -507,6 +503,7 @@ class GenomeDownloads(admin.TabularInline):
         'description',
         'file_format'
     ]
+    extra = 0
 
 
 @admin.register(emg_models.Genome)
@@ -527,6 +524,7 @@ class GenomeAdmin(admin.ModelAdmin):
     ]
     search_fields = [
         'accession',
+        'catalogue__name',
         'type',
         'ena_genome_accession',
         'ena_sample_accession',
@@ -536,11 +534,11 @@ class GenomeAdmin(admin.ModelAdmin):
         'ncbi_study_accession',
         'img_genome_accession',
         'patric_genome_accession',
-        'biome'
+        'biome__lineage'
     ]
     list_filter = [
         'genome_set',
-        'releases',
+        'catalogue__catalogue_id',
         'type',
     ]
     raw_id_fields = [
@@ -553,24 +551,49 @@ class GenomeAdmin(admin.ModelAdmin):
         'antismash_geneclusters'
     ]
     inlines = [
-        GenomeReleasesInline,
         GenomeDownloads
     ]
 
 
-@admin.register(emg_models.Release)
-class GenomeReleaseAdmin(admin.ModelAdmin):
-    readonly_fields = [
-        'id'
+class GenomeCatalogueDownloads(admin.TabularInline):
+    model = emg_models.GenomeCatalogueDownload
+    raw_id_fields = [
+        'genome_catalogue',
+        'parent_id',
+        'group_type',
+        'subdir',
+        'description',
+        'file_format'
     ]
+    extra = 0
+
+
+def recalculate_genome_count(modeladmin, request, queryset):
+    for catalogue in queryset:
+        catalogue.calculate_genome_count()
+        messages.add_message(request, messages.SUCCESS, f'Catalogue {catalogue.catalogue_id} saved '
+                                                        f'with genome_count={catalogue.genome_count}')
+
+
+recalculate_genome_count.short_description = 'Recalculate genome count'
+
+
+@admin.register(emg_models.GenomeCatalogue)
+class GenomeCatalogueAdmin(admin.ModelAdmin):
     list_display = [
-        'version',
-        'first_created',
-        'last_update'
+        'catalogue_id',
+        'name',
+        'biome',
+        'last_update',
     ]
-    exclude = [
-        'genomes'
+    raw_id_fields = [
+        'biome',
+    ]
+    inlines = [
+        GenomeCatalogueDownloads
     ]
+    readonly_fields = ['genome_count']
+    actions = [recalculate_genome_count, ]
 
 
 @admin.register(emg_models.KeggClass)
@@ -756,8 +779,8 @@ class GenomeDownloadAdmin(admin.ModelAdmin, BaseDownloadAdmin):
     ]
 
 
-@admin.register(emg_models.ReleaseDownload)
-class ReleaseDownloadAdmin(admin.ModelAdmin, BaseDownloadAdmin):
+@admin.register(emg_models.GenomeCatalogueDownload)
+class GenomeCatalogueDownloadAdmin(admin.ModelAdmin, BaseDownloadAdmin):
     list_display = BaseDownloadAdmin.list_display + [
-        'release'
+        'genome_catalogue'
     ]
diff --git a/emgapi/fields.py b/emgapi/fields.py
@@ -40,9 +40,9 @@ def get_url(self, obj, view_name, request, format):
             kwargs = {
                 'accession': obj.genome.accession
             }
-        elif hasattr(obj, 'release'):
+        elif hasattr(obj, 'genome_catalogue'):
             kwargs = {
-                'version': obj.release.version
+                'catalogue_id': obj.genome_catalogue.catalogue_id,
             }
         kwargs['alias'] = obj.alias
 

diff --git a/emgapi/filters.py b/emgapi/filters.py
@@ -1063,6 +1063,39 @@ class Meta:
             "pangenome_accessory_size__lte",
         )
 
+
+class GenomeCatalogueFilter(django_filters.FilterSet):
+
+    lineage = filters.ModelChoiceFilter(
+        queryset=emg_models.Biome.objects.all(),
+        method='filter_lineage', distinct=True,
+        to_field_name='lineage',
+        label='Biome lineage',
+        help_text='Biome lineage')
+
+    def filter_lineage(self, qs, name, value):
+        try:
+            b = emg_models.Biome.objects.get(lineage=value)
+            catalogues = emg_models.GenomeCatalogue.objects\
+                .filter(
+                biome__lft__gte=b.lft,
+                biome__rgt__lte=b.rgt)
+            qs = qs.filter(pk__in=catalogues)
+        except emg_models.Biome.DoesNotExist:
+            pass
+        return qs
+
+    class Meta:
+        model = emg_models.GenomeCatalogue
+        fields = {
+            'catalogue_id': ['exact'],
+            'name': ['exact', 'icontains'],
+            'description': ['exact', 'icontains'],
+            'biome__biome_name': ['exact', 'icontains'],
+            'last_update': ['exact', 'lt', 'gt'],
+        }
+
+
 def getUnambiguousOrderingFilterByField(field):
     class UnambiguousOrderingFilter(drf_filters.OrderingFilter):
         def filter_queryset(self, request, queryset, view):