Merge pull request #7 from smaht-dac/tests

Add More Tests

.github/workflows/main.yml

@@ -22,6 +22,11 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
 
+    # Build matrix
+    strategy:
+      matrix:
+        test_type: [ 'UNIT', 'DOCKER' ]
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -31,6 +36,7 @@ jobs:
           python-version: 3.9
 
       - name: Install/Link Postgres
+        if: ${{ matrix.test_type == 'UNIT' }}
         run: |
           sudo apt-get install curl ca-certificates gnupg
           curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
@@ -41,6 +47,7 @@ jobs:
           sudo ln -s /usr/lib/postgresql/11/bin/initdb /usr/local/bin/initdb
 
       - name: Install Deps
+        if: ${{ matrix.test_type == 'UNIT' }}
         run: |
           make build
 
@@ -51,23 +58,30 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-      - name: QA
+      - name: QA (UNIT)
+        if: ${{ matrix.test_type == 'UNIT' }}
         env:
           # S3_ENCRYPT_KEY: ${{ secrets.S3_ENCRYPT_KEY }}
           # # The need for this old environment variable name will go away soon.
           # TRAVIS_JOB_ID: sno-x-test-${{ github.run_number }}-
           # This will be the new environment variable name.
           TEST_JOB_ID: smaht-test-indexing-${{ github.run_number }}-${{ matrix.python_version }}-
         run: |
-          make test
+          make remote-test
+
+      - name: Docker Build
+        if: ${{ matrix.test_type == 'DOCKER' }}
+        run: |
+          touch deploy/docker/local/docker_development.ini  # cheap substitute for prepare-docker to make ignored file
+          docker build .
 
       - name: Cleanup
-        if: ${{ always() }}
+        if: ${{ always() && matrix.test_type == 'UNIT' }}
         env:
           S3_ENCRYPT_KEY: ${{ secrets.S3_ENCRYPT_KEY }}
           # The need for this old environment variable name will go away soon.
           # TRAVIS_JOB_ID: sno-x-test-${{ github.run_number }}-
           # This will be the new environment variable name.
-          TEST_JOB_ID: sno-test-indexing-${{ github.run_number }}-${{ matrix.python_version }}-
+          TEST_JOB_ID: smaht-test-indexing-${{ github.run_number }}-${{ matrix.python_version }}-
         run: |
-          poetry run wipe-test-indices $TEST_JOB_ID search-fourfront-testing-opensearch-kqm7pliix4wgiu4druk2indorq.us-east-1.es.amazonaws.com:443
+          poetry run wipe-test-indices $TEST_JOB_ID search-opensearch-smaht-testing-ykavtw57jz4cx4f2gqewhu4b44.us-east-1.es.amazonaws.com:443

CHANGELOG.rst

@@ -7,6 +7,14 @@ smaht-portal
 Change Log
 ----------
 
+0.0.5
+=====
+
+* Improve testing by porting relevant tests as needed
+* Changes to accommodate working search/other tests
+* Allow testing with ES in GA with smaht-development credentials
+* Build Docker as part of GA
+
 0.0.4
 =====
 

Makefile

@@ -169,9 +169,8 @@ test-static:
 	poetry run python -m pytest -vv -m static
 	make lint
 
-remote-test:  # Actually, we don't normally use this. Instead the GA workflow sets up two parallel tests.
-	make remote-test-unit
-	make remote-test-npm
+remote-test:  # smaht-portal uses this make target for now as tests are not that burdensome
+	pytest -vv --aws-auth --durations=20 --es search-opensearch-smaht-testing-ykavtw57jz4cx4f2gqewhu4b44.us-east-1.es.amazonaws.com:443
 
 remote-test-npm:  # Note this only does the 'not indexing' tests
 	poetry run python -m pytest -xvv -r w --instafail --force-flaky --max-runs=2 --timeout=600 -m "not manual and not integratedx and not performance and not broken and not broken_remotely and not sloppy and not indexing and not static" --aws-auth --durations=20 --cov src/encoded --es search-cgap-unit-testing-opensearch-tcs45cjpwgdzoi7pafr6oewq6u.us-east-1.es.amazonaws.com:443

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "encoded"
-version = "0.0.4"
+version = "0.0.5"
 description = "SMaHT Data Analysis Portal"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -42,7 +42,7 @@ certifi = ">=2021.5.30"
 chardet = "3.0.4"
 codeguru-profiler-agent = "^1.2.4"
 colorama = "0.3.3"
-dcicsnovault = "8.2.0.1b42"
+dcicsnovault = "8.2.0.1b45"
 dcicutils = "7.5.1.1b0"
 encoded-core = "^0.0.2"
 elasticsearch = "7.13.4"

src/encoded/__init__.py

@@ -84,8 +84,8 @@ def include_snovault(config: Configurator) -> None:
     config.include('snovault.resource_views')
     config.include('snovault.settings')
     config.include('snovault.server_defaults')
-    # Renderers is giving problems at the moment - Will 6/1/23
     config.include('snovault.renderers')
+    config.include('snovault.ingestion.ingestion_listener')
     # configure redis server in production.ini
     if 'redis.server' in config.registry.settings:
         config.include('snovault.redis')

src/encoded/appdefs.py

@@ -1 +1,8 @@
+import os
+
+
 APP_VERSION_REGISTRY_KEY = 'snovault.app_version'
+
+
+ENCODED_ROOT_DIR = os.path.dirname(__file__)
+PROJECT_DIR = os.path.dirname(os.path.dirname(ENCODED_ROOT_DIR))  # two levels of hierarchy up

src/encoded/project/loadxl.py

@@ -3,9 +3,7 @@
 
 class SMaHTProjectLoadxl(SnovaultProjectLoadxl):
 
-    def loadxl_order(self):
-        """ Defines any hard orderings that must happen when reindexing types """
-        return [
+    order = [
             'AccessKey',
             'User',
             'Consortium',
@@ -29,4 +27,11 @@ def loadxl_order(self):
             'FilterSet',
             'HiglassViewConfig',
             'IngestionSubmission'
-        ]
+    ]
+
+    def loadxl_order(self):
+        """ Defines any hard orderings that must happen when reindexing types """
+        return self.order
+
+
+ITEM_INDEX_ORDER = SMaHTProjectLoadxl.order

src/encoded/schemas/consortium.json

@@ -1,6 +1,6 @@
 {
     "title": "Consortium",
-    "id": "/profiles/Consortium.json",
+    "id": "/profiles/consortium.json",
     "$schema": "http://json-schema.org/draft-04/schema#",
     "required": [
         "name",

src/encoded/schemas/submission_center.json

@@ -1,6 +1,6 @@
 {
     "title": "SubmissionCenter",
-    "id": "/profiles/SubmissionCenter.json",
+    "id": "/profiles/submission_center.json",
     "$schema": "http://json-schema.org/draft-04/schema#",
     "required": [
         "name",

src/encoded/tests/conftest.py

@@ -95,7 +95,7 @@ def app_settings(request, wsgi_server_host_port, conn, DBSession):  # noQA - We
     return settings
 
 
-INDEXER_NAMESPACE_FOR_TESTING = generate_indexer_namespace_for_testing('cgap')
+INDEXER_NAMESPACE_FOR_TESTING = generate_indexer_namespace_for_testing('smaht')
 
 
 @pytest.fixture(scope='session')

src/encoded/tests/test_authentication.py

@@ -0,0 +1,103 @@
+import pytest
+import unittest
+
+from pyramid.interfaces import IAuthenticationPolicy
+from pyramid.security import Authenticated, Everyone
+from pyramid.testing import DummyRequest
+from zope.interface.verify import verifyClass, verifyObject
+from ..authentication import NamespacedAuthenticationPolicy
+
+
+pytestmark = [pytest.mark.setone, pytest.mark.working]
+
+
+class TestNamespacedAuthenticationPolicy(unittest.TestCase):
+    """ This is a modified version of TestRemoteUserAuthenticationPolicy
+    """
+    def _getTargetClass(self):
+        return NamespacedAuthenticationPolicy
+
+    def _makeOne(self, namespace='user',
+                base='pyramid.authentication.RemoteUserAuthenticationPolicy',
+                *args, **kw):
+        return self._getTargetClass()(namespace, base, *args, **kw)
+
+    def test_class_implements_IAuthenticationPolicy(self):
+        klass = self._makeOne().__class__
+        verifyClass(IAuthenticationPolicy, klass)
+
+    def test_instance_implements_IAuthenticationPolicy(self):
+        verifyObject(IAuthenticationPolicy, self._makeOne())
+
+    def test_unauthenticated_userid_returns_None(self):
+        request = DummyRequest(environ={})
+        policy = self._makeOne()
+        self.assertEqual(policy.unauthenticated_userid(request), None)
+
+    def test_unauthenticated_userid(self):
+        request = DummyRequest(environ={'REMOTE_USER':'fred'})
+        policy = self._makeOne()
+        self.assertEqual(policy.unauthenticated_userid(request), 'user.fred')
+
+    def test_authenticated_userid_None(self):
+        request = DummyRequest(environ={})
+        policy = self._makeOne()
+        self.assertEqual(policy.authenticated_userid(request), None)
+
+    def test_authenticated_userid(self):
+        request = DummyRequest(environ={'REMOTE_USER':'fred'})
+        policy = self._makeOne()
+        self.assertEqual(policy.authenticated_userid(request), 'user.fred')
+
+    def test_effective_principals_None(self):
+        request = DummyRequest(environ={})
+        policy = self._makeOne()
+        self.assertEqual(policy.effective_principals(request), [Everyone])
+
+    def test_effective_principals(self):
+        request = DummyRequest(environ={'REMOTE_USER':'fred'})
+        policy = self._makeOne()
+        self.assertEqual(policy.effective_principals(request),
+                         [Everyone, Authenticated, 'user.fred'])
+
+    def test_remember(self):
+        request = DummyRequest(environ={'REMOTE_USER':'fred'})
+        policy = self._makeOne()
+        result = policy.remember(request, 'fred')
+        self.assertEqual(result, [])
+
+    def test_forget(self):
+        request = DummyRequest(environ={'REMOTE_USER':'fred'})
+        policy = self._makeOne()
+        result = policy.forget(request)
+        self.assertEqual(result, [])
+
+    # From TestSessionAuthenticationPolicy
+
+    def test_session_remember(self):
+        request = DummyRequest()
+        policy = self._makeOne(
+                    base='pyramid.authentication.SessionAuthenticationPolicy',
+                    prefix='')
+        result = policy.remember(request, 'user.fred')
+        self.assertEqual(request.session.get('userid'), 'fred')
+        self.assertEqual(result, [])
+        self.assertEqual(policy.unauthenticated_userid(request), 'user.fred')
+
+    def test_session_forget(self):
+        request = DummyRequest(session={'userid':'fred'})
+        policy = self._makeOne(
+                    base='pyramid.authentication.SessionAuthenticationPolicy',
+                    prefix='')
+        result = policy.forget(request)
+        self.assertEqual(request.session.get('userid'), None)
+        self.assertEqual(result, [])
+
+    def test_session_forget_no_identity(self):
+        request = DummyRequest()
+        policy = self._makeOne(
+                    base='pyramid.authentication.SessionAuthenticationPolicy',
+                    prefix='')
+        result = policy.forget(request)
+        self.assertEqual(request.session.get('userid'), None)
+        self.assertEqual(result, [])

src/encoded/tests/test_authorization.py

@@ -0,0 +1,28 @@
+import pytest
+
+
+from .test_search import MockedRequest
+from snovault.util import is_admin_request
+
+
+pytestmark = [pytest.mark.working]
+
+
+@pytest.mark.parametrize('mock_request, expected', [
+    [MockedRequest(), False],
+    [MockedRequest(principals_allowed=[
+        'group.admin'
+    ]), True],
+    [MockedRequest(principals_allowed=[
+        'group.read-only-admin'
+    ]), False],  # read only admin is not admin technically
+    [MockedRequest(principals_allowed=[
+        'group.something-else'
+    ]), False],
+    [MockedRequest(principals_allowed=[
+        'lots', 'of', 'other', 'perms', 'but', 'not', 'group', 'dot', 'admin'
+    ]), False]
+])
+def test_authorization_is_admin_request(mock_request, expected):
+    """ Checks that the request tests correctly resolve. """
+    assert is_admin_request(mock_request) is expected