Merge pull request #252 from 4dn-dcic/core

Core Modules

.github/workflows/main-publish.yml

@@ -27,6 +27,8 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.8
+      - name: Install Python dependencies for publish
+        run: python -m pip install dcicutils==7.5.0
       - name: Publish
         env:
           PYPI_USER: ${{ secrets.PYPI_USER }}

.github/workflows/main.yml

@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install/Link Postgres
         run: |

.gitignore

@@ -63,3 +63,7 @@
 
 # Vim
 *.swp
+
+# These are generated by prepare-local-dev from corresponding .template files
+development.ini
+test.ini

CHANGELOG.rst

@@ -6,6 +6,50 @@ snovault
 Change Log
 ----------
 
+9.0.0
+=====
+* Merge/unify ingestion and other code from cgap-portal and fourfront.
+
+
+8.1.0
+=====
+
+* Add several modules/commands from upstream portals that are generic enough to live in
+  this repository (to reduce code/library maintenace overhead)
+
+* Port support for ``make deploy1`` from the portals:
+
+  * In ``Makefile``:
+
+    * Support for ``make deploy1``
+
+    * Support for ``make psql-dev``
+
+    * Support for ``make psql-test``
+
+    * Support for ``make kibana-start`` (commented out for now, pending testing)
+
+    * Support for ``make kibana-start-test`` (commented out)
+
+    * Support for ``make kibana-stop`` (commented out)
+
+  * In ``pyproject.toml``:
+
+    * Template file ``development.ini.template``
+
+    * Template file ``test.ini.template``
+
+    * Support for ``prepare-local-dev`` script,
+      which creates ``development.ini`` from ``development.ini.template``
+      and ``test.ini`` from ``test.ini.template``.
+
+ * Port the ``dev_servers.py`` support from CGAP.
+
+ * In the ``scripts/`` dir:
+
+   * Add ``scripts/psql-start``
+     in support of ``make psql-dev`` and ``make psql-test``.
+
 
 8.0.1
 =====
@@ -33,6 +77,7 @@ Change Log
 * Change ``pytest.yield_fixture`` to ``pytest.yield``. This is techinically incompatible since it would break downstream portals if they were below ``pytest`` 6, but they are both at ``pytest 7`` now, so they should be unaffected.
 * Address some places involving ``.execute(raw_string)`` that should be ``.execute(text(raw_string))``.
 
+
 7.3.0
 =====
 

Makefile

@@ -1,59 +1,99 @@
+SHELL=/bin/bash
+
 clean:
-	rm -rf *.egg-info
+	make clean-python-caches
 
-configure:  # does any pre-requisite installs
-	@#pip install --upgrade pip==21.0.1
-	pip install --upgrade pip
-	@#pip install poetry==1.1.9  # this version is known to work. -kmp 5-Oct-2021
-	pip install wheel
-	pip install poetry
+clean-python-caches:
+	rm -rf *.egg-info
 
 clear-poetry-cache:  # clear poetry/pypi cache. for user to do explicitly, never automatic
 	poetry cache clear pypi --all
 
+aws-ip-ranges:
+	curl -o aws-ip-ranges.json https://ip-ranges.amazonaws.com/ip-ranges.json
+
 macpoetry-install:
 	scripts/macpoetry-install
 
-lint:
-	poetry run flake8 snovault
+configure:  # does any pre-requisite installs
+	pip install --upgrade pip
+	pip install wheel
+	pip install poetry==1.4.2
 
-macbuild:
+build-poetry:
+	make configure
+	poetry install
+
+macbuild-poetry:
 	make configure
 	make macpoetry-install
-	make build-after-poetry
 
 build:
-	make configure
-	make build-configured
+ifeq ($(shell uname -s), Darwin)
+	@echo "Looks like this is Mac so executing: make macbuild"
+	make macbuild
+else
+	make build-poetry
+	make build-after-poetry
+endif
 
-build-configured:
-	poetry install
+macbuild:
+	make macbuild-poetry
+	make build-after-poetry
+
+build-after-poetry:  # continuation of build after poetry install
+	make aws-ip-ranges
+	poetry run python setup_eb.py develop
+	make fix-dist-info
+	poetry run prepare-local-dev
+
+fix-dist-info:
+	@scripts/fix-dist-info
 
 build-for-ga:
 	make configure
 	poetry config --local virtualenvs.create true
-	make build-configured
+	poetry install
+
+deploy1:  # starts postgres/ES locally and loads inserts, and also starts ingestion engine
+	@DEBUGLOG=`pwd` SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers-snovault development.ini --app-name app --clear --init --load
+
+psql-dev:  # starts psql with the url after 'sqlalchemy.url =' in development.ini
+	@scripts/psql-start dev
+
+psql-test:  # starts psql with a url constructed from data in 'ps aux'.
+	@scripts/psql-start test
+
+#kibana-start:  # starts a dev version of kibana (default port)
+#	scripts/kibana-start
+#
+#kibana-start-test:  # starts a test version of kibana (port chosen for active tests)
+#	scripts/kibana-start test
+#
+#kibana-stop:
+#	scripts/kibana-stop
 
 ES_URL = search-fourfront-testing-opensearch-kqm7pliix4wgiu4druk2indorq.us-east-1.es.amazonaws.com:443
 
-LOCAL_INSTAFAIL_OPTIONS = --timeout=200 -xvv --instafail
+LOCAL_INSTAFAIL_OPTIONS = --timeout=400 -xvv --instafail
 LOCAL_MULTIFAIL_OPTIONS = --timeout=200 -vv
 GA_CICD_TESTING_OPTIONS = --timeout=400 -xvvv --durations=100 --aws-auth --es ${ES_URL}
 STATIC_ANALYSIS_OPTIONS =  -vv
 
-TEST_NAME ?= missing_TEST_NAME
-
-test-one:
-
-	SQLALCHEMY_WARN_20=1 pytest ${LOCAL_MULTIFAIL_OPTIONS} -k ${TEST_NAME}
-
 test:
 	@git log -1 --decorate | head -1
 	@date
 	make test-unit && make test-indexing && make test-static
 	@git log -1 --decorate | head -1
 	@date
 
+ES_URL = search-fourfront-testing-opensearch-kqm7pliix4wgiu4druk2indorq.us-east-1.es.amazonaws.com:443
+
+LOCAL_INSTAFAIL_OPTIONS = --timeout=400 -xvv --instafail
+LOCAL_MULTIFAIL_OPTIONS = --timeout=200 -vv
+GA_CICD_TESTING_OPTIONS = --timeout=400 -xvvv --durations=100 --aws-auth --es ${ES_URL}
+STATIC_ANALYSIS_OPTIONS =  -vv
+
 test-full:
 	@git log -1 --decorate | head -1
 	@date
@@ -63,48 +103,98 @@ test-full:
 	@git log -1 --decorate | head -1
 	@date
 
-test-indexing:
-	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_INSTAFAIL_OPTIONS} -m "indexing"
-
 test-unit:
 	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_INSTAFAIL_OPTIONS} -m "not indexing"
 
-test-indexing-full:
-	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_MULTIFAIL_OPTIONS} -m "indexing"
-
 test-unit-full:
 	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_MULTIFAIL_OPTIONS} -m "not indexing"
 
+test-indexing-full:
+	make test-indexing-not-es-full
+	make test-indexing-es-full
+
+test-indexing-es-full:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_MULTIFAIL_OPTIONS} -m "indexing and es"
+
+test-indexing-not-es-full:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_MULTIFAIL_OPTIONS} -m "indexing and not es"
+
+test-indexing:
+	make test-indexing-not-es && make test-indexing-es
+
+test-indexing-es:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_INSTAFAIL_OPTIONS} -m "indexing and es"
+
+test-indexing-not-es:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${LOCAL_INSTAFAIL_OPTIONS} -m "indexing and not es"
+
+test-performance:
+	@echo "snovault has no performance tests right now, but it could."
+
+test-integrated:
+	@echo "snovault has no integrated tests right now, but it could."
+
 test-static:
 	NO_SERVER_FIXTURES=TRUE USE_SAMPLE_ENVUTILS=TRUE poetry run python -m pytest -vv -m static
 	make lint
 
-remote-test-indexing:
-	SQLALCHEMY_WARN_20=1 poetry run pytest ${GA_CICD_TESTING_OPTIONS} -m "indexing"
+TEST_NAME ?= missing_TEST_NAME_parameter
+
+test-one:
+	SQLALCHEMY_WARN_20=1 poetry run python -m pytest ${LOCAL_MULTIFAIL_OPTIONS} -k ${TEST_NAME}
+
+remote-test:  # Actually, we don't normally use this. Instead the GA workflow sets up two parallel tests.
+	make remote-test-indexing && make remote-test-unit
 
 remote-test-unit:
+	make remote-test-not-indexing
+
+remote-test-not-indexing:
 	SQLALCHEMY_WARN_20=1 poetry run pytest ${GA_CICD_TESTING_OPTIONS} -m "not indexing"
 
+remote-test-indexing:
+	 make remote-test-indexing-not-es && make remote-test-indexing-es
+
+remote-test-indexing-es:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${GA_CICD_TESTING_OPTIONS} -m "indexing and es"
+
+remote-test-indexing-not-es:
+	SQLALCHEMY_WARN_20=1 poetry run pytest ${GA_CICD_TESTING_OPTIONS} -m "indexing and not es"
+
 update:
 	poetry update
 
 publish:
-	scripts/publish
+	poetry run publish-to-pypi
 
 publish-for-ga:
-	scripts/publish --noconfirm
+	poetry run publish-to-pypi --noconfirm
 
-kill:
+kill:  # kills back-end processes associated with the application. Use with care.
 	pkill -f postgres &
-	pkill -f elasticsearch &
+	pkill -f opensearch &
+
+
+lint-full:
+	poetry run flake8 deploy/ || echo "flake8 failed for deploy/"
+	poetry run flake8 snovault/ || echo "flake8 failed for snovault/"
+
+lint:
+	poetry run flake8 deploy/ && poetry run flake8 snovault/
 
 help:
 	@make info
 
 info:
 	@: $(info Here are some 'make' options:)
+	   $(info - Use 'make aws-ip-ranges' to download latest ip range information. Invoked automatically when needed.)
+	   $(info - Use 'make build' to build only application dependencies (or 'make macbuild' on OSX Catalina))
 	   $(info - Use 'make clean' to clear out (non-python) dependencies)
+	   $(info - Use 'make clear-poetry-cache' to clear the poetry pypi cache if in a bad state. (Safe, but later recaching can be slow.))
 	   $(info - Use 'make configure' to install poetry, though 'make build' will do it automatically.)
-	   $(info - Use 'make build' to build only application dependencies (or 'make macbuild' on OSX Catalina))
+	   $(info - Use 'make deploy1' to spin up postgres/elasticsearch and load inserts.)
+	   $(info - Use 'make kill' to kill postgres and opensearch proccesses. Please use with care.)
+	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
+	   $(info - Use 'make psql-test' to start psql on data associated with an active test.)
 	   $(info - Use 'make test' to run tests with the normal options we use on travis)
 	   $(info - Use 'make update' to update dependencies (and the lock file))

base.ini

@@ -0,0 +1,62 @@
+[app:app]
+use = egg:snovault
+create_tables = true
+sqlalchemy.url = postgresql:///encoded
+retry.attempts = 3
+file_wfout_bucket = encoded-4dn-files
+file_upload_profile_name = encoded-4dn-files
+system_bucket = elasticbeanstalk-encoded-4dn-system
+elasticsearch.server = 127.0.0.1:9200
+ontology_path = %(here)s/ontology.json
+aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
+#this is to reroute downloads to something other than aws s3
+#download_proxy = https://download.encodeproject.org/
+
+# # Only run ec2metadata on ec2 instances
+# # XXX really need to reorganise ini files for more reuse
+# hostname_command = command -v ec2metadata > /dev/null && ec2metadata --public-hostname || hostname
+#
+# multiauth.policies = auth0 session remoteuser accesskey
+# multiauth.groupfinder = encoded.authorization.groupfinder
+#
+# multiauth.policy.session.namespace = mailto
+# multiauth.policy.session.use = encoded.authentication.NamespacedAuthenticationPolicy
+# multiauth.policy.session.base = pyramid.authentication.SessionAuthenticationPolicy
+#
+# multiauth.policy.remoteuser.namespace = remoteuser
+# multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticationPolicy
+# multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy
+#
+# multiauth.policy.accesskey.namespace = accesskey
+# multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy
+# multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy
+# multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check
+
+# multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy
+# multiauth.policy.auth0.namespace = auth0
+# multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy
+
+auth0.siteName = 4DN DCC Submission
+
+postgresql.statement_timeout = 120
+pyramid.default_locale_name = en
+# Google analytics config
+ga_config_location = ./src/encoded/static/ga_config.json
+
+# [composite:indexer]
+# use = egg:encoded#indexer
+# app = app
+# path = /index
+# timeout = 60
+# set embed_cache.capacity = 5000
+# set indexer = true
+
+# [composite:ingester]
+# use = egg:encoded#ingester
+# app = app
+# path = /ingest
+# timeout = 60
+
+# [filter:memlimit]
+# use = egg:encoded#memlimit
+# rss_limit = 450MB