From 8c3700fe19c65d324c8e8de3ab9af46979824730 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:05:32 -0700 Subject: [PATCH 01/33] make dev-requirements.txt --- Makefile | 17 ++++++-- dev-requirements.in | 5 +++ dev-requirements.txt | 102 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 dev-requirements.in create mode 100644 dev-requirements.txt diff --git a/Makefile b/Makefile index 2a76e71..e55c4de 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ upload: open $(PKG_URL) install: venv requirements.txt - . $(ACTIVATE) && python3 -m pip install -r requirements.txt +# . $(ACTIVATE) && python3 -m pip install -r requirements.txt + . $(ACTIVATE) && pip-sync requirements.txt $(TARGET): install make.py lambdas/lambda.py . $(ACTIVATE) && python3 make.py > $(TARGET) @@ -20,9 +21,17 @@ $(TARGET): install make.py lambdas/lambda.py venv: python3 -m venv venv +tools: venv + . $(ACTIVATE) && python3 -m pip install pip-tools pip-compile: requirements.txt -requirements.txt: requirements.in - . $(ACTIVATE) && python3 -m pip install pip-tools - . $(ACTIVATE) && pip-compile requirements.in \ No newline at end of file +requirements.txt: tools requirements.in + . $(ACTIVATE) && pip-compile requirements.in + +pip-dev: venv dev-requirements.txt + . $(ACTIVATE) && pip-sync dev-requirements.txt + +dev-requirements.txt: tools dev-requirements.in + . $(ACTIVATE) && pip-compile dev-requirements.in + diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 0000000..c835376 --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,5 @@ +# https://suyojtamrakar.medium.com/managing-your-requirements-txt-with-pip-tools-in-python-8d07d9dfa464 +aws-lambda-powertools ~= 2.15 +benchling-sdk ~= 1.6 +jinja2 ~= 3.1 +pytest diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..6325dd2 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,102 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile dev-requirements.in +# +anyio==3.7.1 + # via httpcore +attrs==22.2.0 + # via + # benchling-api-client + # benchling-sdk +aws-lambda-powertools==2.22.0 + # via -r dev-requirements.in +backoff==1.11.1 + # via + # benchling-api-client + # benchling-sdk +benchling-api-client==2.0.167 + # via benchling-sdk +benchling-sdk==1.7.0 + # via -r dev-requirements.in +certifi==2023.7.22 + # via + # benchling-sdk + # httpcore + # httpx +cffi==1.15.1 + # via cryptography +cryptography==41.0.3 + # via jwcrypto +dataclasses-json==0.5.14 + # via + # benchling-api-client + # benchling-sdk +deprecated==1.2.14 + # via jwcrypto +exceptiongroup==1.1.3 + # via + # anyio + # pytest +h11==0.14.0 + # via httpcore +httpcore==0.17.3 + # via httpx +httpx==0.24.1 + # via + # benchling-api-client + # benchling-sdk +idna==3.4 + # via + # anyio + # httpx +iniconfig==2.0.0 + # via pytest +jinja2==3.1.2 + # via -r dev-requirements.in +jwcrypto==1.5.0 + # via benchling-sdk +markupsafe==2.1.3 + # via jinja2 +marshmallow==3.20.1 + # via dataclasses-json +mypy-extensions==1.0.0 + # via typing-inspect +ordered-set==4.1.0 + # via benchling-sdk +packaging==23.1 + # via + # marshmallow + # pytest +pluggy==1.2.0 + # via pytest +pycparser==2.21 + # via cffi +pytest==7.4.0 + # via -r dev-requirements.in +python-dateutil==2.8.2 + # via + # benchling-api-client + # benchling-sdk +pyyaml==6.0.1 + # via benchling-sdk +six==1.16.0 + # via python-dateutil +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +tomli==2.0.1 + # via pytest +typing-extensions==4.7.1 + # via + # aws-lambda-powertools + # benchling-api-client + # benchling-sdk + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +wrapt==1.15.0 + # via deprecated From dab72c1c39b0de93d30c0b1a419b98be2b3aad18 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:10:00 -0700 Subject: [PATCH 02/33] make test --- Makefile | 7 ++++++- test/test_lambda.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 test/test_lambda.py diff --git a/Makefile b/Makefile index e55c4de..8550c7f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" -.PHONY: all template install pip-compile upload +.PHONY: all install pip-compile pip-dev template test upload all: template upload @@ -18,6 +18,11 @@ install: venv requirements.txt $(TARGET): install make.py lambdas/lambda.py . $(ACTIVATE) && python3 make.py > $(TARGET) +test: pip-dev run-test + +run-test: venv + . $(ACTIVATE) && python3 -m pytest + venv: python3 -m venv venv diff --git a/test/test_lambda.py b/test/test_lambda.py new file mode 100644 index 0000000..0468354 --- /dev/null +++ b/test/test_lambda.py @@ -0,0 +1,2 @@ +def test_lambda(): + assert True \ No newline at end of file From ff0052a0694919b767b792f0978b9de2491f38c2 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:10:56 -0700 Subject: [PATCH 03/33] Create mega-linter.yml --- .github/workflows/mega-linter.yml | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/mega-linter.yml diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml new file mode 100644 index 0000000..570e0be --- /dev/null +++ b/.github/workflows/mega-linter.yml @@ -0,0 +1,56 @@ +# MegaLinter GitHub Action configuration file +# More info at https://megalinter.github.io +name: MegaLinter + +on: + # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) +permissions: read-all + +env: # Comment env block if you do not want to apply fixes + # Apply linter fixes configuration + APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool) + #APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all) + #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request) + DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,PYTHON_BANDIT,PYTHON_MYPY,PYTHON_PYRIGHT,PYTHON_PYLINT + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + name: MegaLinter + runs-on: ubuntu-latest + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances + + # MegaLinter + - name: MegaLinter + id: ml + # You can override MegaLinter flavor used to have faster performances + # More info at https://megalinter.github.io/flavors/ + uses: oxsecurity/megalinter/flavors/python@v6.22.2 + env: + # All available variables are described in documentation + # https://megalinter.github.io/configuration/ + VALIDATE_ALL_CODEBASE: true + # VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Validates all source when push on main, else just the git diff with main. Override with true if you always want to lint all sources + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY + DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks + + # Upload MegaLinter artifacts + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log From d0d9025dea7fd302fd8324a97ccfad54e353377b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:17:09 -0700 Subject: [PATCH 04/33] fix megalinting --- .github/workflows/mega-linter.yml | 2 +- Makefile | 10 ++++++++-- layer/deploy.sh | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 570e0be..70b0494 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -12,7 +12,7 @@ env: # Comment env block if you do not want to apply fixes APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool) #APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all) #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request) - DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,PYTHON_BANDIT,PYTHON_MYPY,PYTHON_PYRIGHT,PYTHON_PYLINT + DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,PYTHON_BANDIT,PYTHON_MYPY,PYTHON_PYRIGHT,PYTHON_PYLINT,MARKDOWN_MARKDOWN_LINK_CHECK,REPOSITORY_TRIVY concurrency: group: ${{ github.ref }}-${{ github.workflow }} diff --git a/Makefile b/Makefile index 8550c7f..41f1d3a 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" -.PHONY: all install pip-compile pip-dev template test upload +.PHONY: all clean install pip-compile pip-dev template test upload -all: template upload +all: clean template upload + +clean: + rm -rf build + rm -rf venv + rm -f requirements.txt + rm -f dev-requirements.txt template: $(TARGET) diff --git a/layer/deploy.sh b/layer/deploy.sh index e4fb942..8547aab 100755 --- a/layer/deploy.sh +++ b/layer/deploy.sh @@ -3,7 +3,7 @@ set -euo pipefail error() { - echo $@ 2>&1 + echo "$@" 2>&1 exit 1 } @@ -23,7 +23,7 @@ python3 -m pip install \ --python 3.9 \ --no-deps \ --no-compile \ - -r $exec_dir/requirements.txt + -r "$exec_dir/requirements.txt" echo "Compressing..." zip -9 -r "$zip_file" "." From 674e6b8222055578687772585791da5004b7da31 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:21:51 -0700 Subject: [PATCH 05/33] fix line lengths --- lambdas/lambda.py | 17 ++++++++++------- make.py | 6 ++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lambdas/lambda.py b/lambdas/lambda.py index 9834a3a..7a26e4b 100644 --- a/lambdas/lambda.py +++ b/lambdas/lambda.py @@ -5,13 +5,9 @@ import tempfile import urllib import zipfile - -# Must be done before importing quilt3 -os.environ["QUILT_DISABLE_CACHE"] = "true" - import botocore import jinja2 -import quilt3 + from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities import parameters from benchling_sdk import models as benchling_models @@ -19,6 +15,11 @@ from benchling_sdk.benchling import Benchling from benchling_sdk.helpers import serialization_helpers +# Must be done before importing quilt3 +os.environ["QUILT_DISABLE_CACHE"] = "true" +import quilt3 # noqa: E402 + + logger = Logger() BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] @@ -122,14 +123,16 @@ def lambda_handler(event, context): try: pkg = quilt3.Package.browse(pkg_name, registry=registry) except botocore.exceptions.ClientError as e: - # XXX: quilt3 should raise some specific exception when package doesn't exist. + # XXX: quilt3 should raise some specific exception + # when package doesn't exist. if e.response["Error"]["Code"] not in ("NoSuchKey", "404"): raise pkg = quilt3.Package() pkg.set_dir( ".", tmpdir_path, - # This shouldn't hit 1 MB limit on metadata, because max size of EventBridge is 256 KiB. + # This shouldn't hit 1 MB limit on metadata, + # because max size of EventBridge is 256 KiB. meta=entry, ).push( pkg_name, diff --git a/make.py b/make.py index 393e309..16930d9 100644 --- a/make.py +++ b/make.py @@ -76,7 +76,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: Type="String", AllowedPattern=r"^aws\.partner(/[\.\-_A-Za-z0-9]+){2,}$", Description=( - "Name of event bus where Benchling events are emitted, e.g aws.partner/benchling.com/tenant/app-name" + "Name of event bus where Benchling events are emitted, "+\ + "e.g aws.partner/benchling.com/tenant/app-name" ), ) benchling_tenant = troposphere.Parameter( @@ -84,7 +85,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: template=cft, Type="String", AllowedPattern=r"^[^/]+$", - Description="Benchling tenant name, i.e. $BenchlingTenant in https://$BenchlingTenant.benchling.com", + Description="Benchling tenant name, i.e. $BenchlingTenant in "+\ + "https://$BenchlingTenant.benchling.com", ) benchling_client_id = troposphere.Parameter( "BenchlingClientId", From 73afb5e6ab7f4397ecf4876c192b3c36fc6a2cbb Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 12:28:22 -0700 Subject: [PATCH 06/33] still too long --- lambdas/lambda.py | 6 +++--- make.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lambdas/lambda.py b/lambdas/lambda.py index 7a26e4b..81192a5 100644 --- a/lambdas/lambda.py +++ b/lambdas/lambda.py @@ -28,7 +28,7 @@ DST_BUCKET = os.environ["DST_BUCKET"] PKG_PREFIX = os.environ["PKG_PREFIX"] QUILT_CATALOG_DOMAIN = os.environ["QUILT_CATALOG_DOMAIN"] - +QUILT_PREFIX = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages" benchling = Benchling( url=f"https://{BENCHLING_TENANT}.benchling.com", @@ -145,11 +145,11 @@ def lambda_handler(event, context): if "Quilt Catalog URL" in entry["fields"]: fields_values[ "Quilt Catalog URL" - ] = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages/{pkg_name}" + ] = f"{QUILT_PREFIX}/{pkg_name}" if "Quilt Revise URL" in entry["fields"]: fields_values[ "Quilt Revise URL" - ] = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages/{pkg_name}?action=revisePackage" + ] = f"{QUILT_PREFIX}/{pkg_name}?action=revisePackage" if fields_values: benchling.entries.update_entry( diff --git a/make.py b/make.py index 16930d9..cf47d48 100644 --- a/make.py +++ b/make.py @@ -53,7 +53,7 @@ def make_layer(cft: troposphere.Template): template=cft, Content=awslambda.Content( S3Bucket=troposphere.Sub("quilt-lambda-${AWS::Region}"), - S3Key="benchling-packager/benchling-packager-layer.4bcb4369305e6dca4ec2cec50d2891ad138adfc1f3833293d32a999bd1295770.zip", + S3Key="benchling-packager/benchling-packager-layer.4bcb4369305e6dca4ec2cec50d2891ad138adfc1f3833293d32a999bd1295770.zip", # noqa ), ) @@ -116,7 +116,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: Default="benchling/", AllowedPattern=r".+/.*$", Description=( - "Prefix for package names i.e. package names will be $PackageNamePrefix$ExperimentDisplayID," + "Prefix for package names i.e. package names will be"+ + " $PackageNamePrefix$ExperimentDisplayID,"+ " must contain, but not start with '/'" ), ) From c32839d4f89bc6d33fe655955335dad44b951e01 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 14:13:38 -0700 Subject: [PATCH 07/33] test depedencies --- .github/workflows/python-package.yml | 36 ++++++++++++++ Makefile | 7 ++- dev-requirements.in | 2 + dev-requirements.txt | 73 ++++++++++++++++++++++++---- requirements.in | 1 + requirements.txt | 29 +++++------ test/test_lambda.py | 5 +- 7 files changed, 127 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..0bfe961 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] +permissions: read-all + +jobs: + build: + + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] #, "3.10", "3.11" + os: [ubuntu-latest] # , macos-latest, windows-latest + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Test with pytest + run: | + make test TEST_OS=${{ matrix.os }} + env: + BENCHLING_API_KEY: ${{ secrets.BENCHLING_API_KEY }} + BENCHLING_AUTHOR_ID: ${{ secrets.BENCHLING_AUTHOR_ID }} + BENCHLING_ENTRY_ID: ${{ secrets.BENCHLING_ENTRY_ID }} + BENCHLING_TENANT_DNS: ${{ vars.BENCHLING_TENANT_DNS }} diff --git a/Makefile b/Makefile index 41f1d3a..8fc65bf 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ install: venv requirements.txt # . $(ACTIVATE) && python3 -m pip install -r requirements.txt . $(ACTIVATE) && pip-sync requirements.txt -$(TARGET): install make.py lambdas/lambda.py +$(TARGET): build install make.py lambdas/lambda.py . $(ACTIVATE) && python3 make.py > $(TARGET) test: pip-dev run-test @@ -32,6 +32,9 @@ run-test: venv venv: python3 -m venv venv +build: + mkdir build + tools: venv . $(ACTIVATE) && python3 -m pip install pip-tools @@ -43,6 +46,6 @@ requirements.txt: tools requirements.in pip-dev: venv dev-requirements.txt . $(ACTIVATE) && pip-sync dev-requirements.txt -dev-requirements.txt: tools dev-requirements.in +dev-requirements.txt: tools dev-requirements.in . $(ACTIVATE) && pip-compile dev-requirements.in diff --git a/dev-requirements.in b/dev-requirements.in index c835376..a6f0457 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,5 +1,7 @@ # https://suyojtamrakar.medium.com/managing-your-requirements-txt-with-pip-tools-in-python-8d07d9dfa464 aws-lambda-powertools ~= 2.15 benchling-sdk ~= 1.6 +botocore ~= 1.31 jinja2 ~= 3.1 +quilt3 ~=5.3 pytest diff --git a/dev-requirements.txt b/dev-requirements.txt index 6325dd2..298bd90 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile dev-requirements.in @@ -10,8 +10,12 @@ attrs==22.2.0 # via # benchling-api-client # benchling-sdk + # jsonschema + # referencing aws-lambda-powertools==2.22.0 # via -r dev-requirements.in +aws-requests-auth==0.4.3 + # via quilt3 backoff==1.11.1 # via # benchling-api-client @@ -20,13 +24,23 @@ benchling-api-client==2.0.167 # via benchling-sdk benchling-sdk==1.7.0 # via -r dev-requirements.in +boto3==1.28.26 + # via quilt3 +botocore==1.31.26 + # via + # -r dev-requirements.in + # boto3 + # s3transfer certifi==2023.7.22 # via # benchling-sdk # httpcore # httpx + # requests cffi==1.15.1 # via cryptography +charset-normalizer==3.2.0 + # via requests cryptography==41.0.3 # via jwcrypto dataclasses-json==0.5.14 @@ -35,10 +49,6 @@ dataclasses-json==0.5.14 # benchling-sdk deprecated==1.2.14 # via jwcrypto -exceptiongroup==1.1.3 - # via - # anyio - # pytest h11==0.14.0 # via httpcore httpcore==0.17.3 @@ -51,10 +61,21 @@ idna==3.4 # via # anyio # httpx + # requests iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via -r dev-requirements.in +jmespath==1.0.1 + # via + # boto3 + # botocore +jsonlines==1.2.0 + # via quilt3 +jsonschema==4.19.0 + # via quilt3 +jsonschema-specifications==2023.7.1 + # via jsonschema jwcrypto==1.5.0 # via benchling-sdk markupsafe==2.1.3 @@ -69,8 +90,12 @@ packaging==23.1 # via # marshmallow # pytest +platformdirs==3.10.0 + # via quilt3 pluggy==1.2.0 # via pytest +psutil==5.9.5 + # via benchling-sdk pycparser==2.21 # via cffi pytest==7.4.0 @@ -79,17 +104,43 @@ python-dateutil==2.8.2 # via # benchling-api-client # benchling-sdk + # botocore pyyaml==6.0.1 - # via benchling-sdk + # via + # benchling-sdk + # quilt3 +quilt3==5.3.1 + # via -r dev-requirements.in +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # aws-requests-auth + # quilt3 + # requests-futures +requests-futures==1.0.0 + # via quilt3 +rpds-py==0.9.2 + # via + # jsonschema + # referencing +s3transfer==0.6.1 + # via boto3 six==1.16.0 - # via python-dateutil + # via + # jsonlines + # python-dateutil sniffio==1.3.0 # via # anyio # httpcore # httpx -tomli==2.0.1 - # via pytest +tenacity==8.2.3 + # via quilt3 +tqdm==4.66.1 + # via quilt3 typing-extensions==4.7.1 # via # aws-lambda-powertools @@ -98,5 +149,9 @@ typing-extensions==4.7.1 # typing-inspect typing-inspect==0.9.0 # via dataclasses-json +urllib3==1.26.16 + # via + # botocore + # requests wrapt==1.15.0 # via deprecated diff --git a/requirements.in b/requirements.in index a89b443..6772766 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,4 @@ +botocore ~= 1.31 troposphere ~=4.1 quilt3 ~=5.3 cfn-lint diff --git a/requirements.txt b/requirements.txt index b438e23..b6aecaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements.in @@ -11,25 +11,26 @@ attrs==23.1.0 # sarif-om aws-requests-auth==0.4.3 # via quilt3 -aws-sam-translator==1.68.0 +aws-sam-translator==1.73.0 # via cfn-lint -boto3==1.26.151 +boto3==1.28.26 # via # aws-sam-translator # quilt3 -botocore==1.29.151 +botocore==1.31.26 # via + # -r requirements.in # boto3 # s3transfer certifi==2023.7.22 # via requests cfn-flip==1.3.0 # via troposphere -cfn-lint==0.77.7 +cfn-lint==0.79.7 # via -r requirements.in charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.6 # via cfn-flip idna==3.4 # via requests @@ -41,11 +42,11 @@ jschema-to-python==1.2.3 # via cfn-lint jsonlines==1.2.0 # via quilt3 -jsonpatch==1.32 +jsonpatch==1.33 # via cfn-lint -jsonpickle==3.0.1 +jsonpickle==3.0.2 # via jschema-to-python -jsonpointer==2.3 +jsonpointer==2.4 # via jsonpatch jsonschema==4.17.3 # via @@ -64,20 +65,20 @@ pbr==5.11.1 # sarif-om platformdirs==3.10.0 # via quilt3 -pydantic==1.10.9 +pydantic==1.10.12 # via aws-sam-translator pyrsistent==0.19.3 # via jsonschema python-dateutil==2.8.2 # via botocore -pyyaml==6.0 +pyyaml==6.0.1 # via # cfn-flip # cfn-lint # quilt3 quilt3==5.3.1 # via -r requirements.in -regex==2023.6.3 +regex==2023.8.8 # via cfn-lint requests==2.31.0 # via @@ -102,9 +103,9 @@ tenacity==8.2.3 # via quilt3 tqdm==4.66.1 # via quilt3 -troposphere==4.3.2 +troposphere==4.4.0 # via -r requirements.in -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via # aws-sam-translator # pydantic diff --git a/test/test_lambda.py b/test/test_lambda.py index 0468354..04ab212 100644 --- a/test/test_lambda.py +++ b/test/test_lambda.py @@ -1,2 +1,5 @@ def test_lambda(): - assert True \ No newline at end of file + assert True + +def test_env(): + pass From 6d149c921b8ca0bee35a0447f95b5b7589a8da3f Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 17:40:36 -0700 Subject: [PATCH 08/33] refactor lambda to classes --- lambdas/lambda.py | 210 +++++++++++++++++++++++++++------------------- 1 file changed, 125 insertions(+), 85 deletions(-) diff --git a/lambdas/lambda.py b/lambdas/lambda.py index 81192a5..55921b4 100644 --- a/lambdas/lambda.py +++ b/lambdas/lambda.py @@ -3,9 +3,7 @@ import os import pathlib import tempfile -import urllib import zipfile -import botocore import jinja2 from aws_lambda_powertools import Logger @@ -14,12 +12,13 @@ from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2 from benchling_sdk.benchling import Benchling from benchling_sdk.helpers import serialization_helpers +from botocore import exceptions as botocore_exceptions +from urllib import request as urllib_request # Must be done before importing quilt3 os.environ["QUILT_DISABLE_CACHE"] = "true" import quilt3 # noqa: E402 - logger = Logger() BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] @@ -28,19 +27,70 @@ DST_BUCKET = os.environ["DST_BUCKET"] PKG_PREFIX = os.environ["PKG_PREFIX"] QUILT_CATALOG_DOMAIN = os.environ["QUILT_CATALOG_DOMAIN"] -QUILT_PREFIX = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages" -benchling = Benchling( - url=f"https://{BENCHLING_TENANT}.benchling.com", - auth_method=ClientCredentialsOAuth2( - client_id=BENCHLING_CLIENT_ID, - client_secret=parameters.get_secret(BENCHLING_CLIENT_SECRET_ARN), - ), -) +FIELD_URI = "Quilt+ URI" +FIELD_CATALOG = "Quilt Catalog URL" +FIELD_REVISE = "Quilt Revise URL" + +class BenchlingClient(): + @classmethod + def MakeProxy(cls): + BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] + BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] + BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] + return cls(BENCHLING_TENANT, BENCHLING_CLIENT_ID, BENCHLING_CLIENT_SECRET_ARN) + + def __init__(self, tenant, id, arn): + secret = parameters.get_secret(arn) + if not isinstance(secret, str): + raise Exception(f"Failed to fetch secret: {arn!r}") + self.benchling = Benchling( + url=f"https://{tenant}.benchling.com", + auth_method=ClientCredentialsOAuth2( + client_id=id, + client_secret=secret, + ), + ) + def get_task(self, entry_id): + self.task = self.benchling.tasks.wait_for_task( + self.benchling.exports.export( + benchling_models.ExportItemRequest(_id=entry_id) + ).task_id + ) + if self.task.status != benchling_models.AsyncTaskStatus.SUCCEEDED: + raise Exception(f"Notes export failed: {self.task!r}") + return self.task + + def update_entry(self, entry_id, fields_values): + values = {k: {"value": v} for k, v in fields_values.items()} + self.benchling.entries.update_entry( + entry_id, + benchling_models.EntryUpdate( + _fields=serialization_helpers.fields(values) + ), + ) -template = jinja2.Template( - """# [{{ entry.name }}]({{ entry.webURL }}) +class BenchlingEntry: + QUILT_SUMMARIZE = json.dumps( + [ + [ + { + "path": "entry.md", + "width": "calc(40% - 16px)", + "expand": True, + }, + { + "path": "notes.pdf", + "width": "calc(60% - 16px)", + "expand": True, + }, + ] + ] + ) + + ENTRY_FMT = """ +# [{{ entry.name }}]({{ entry.webURL }}) * id: {{ entry.id }} * displayId: {{ entry.displayId }} @@ -70,45 +120,27 @@ * {{ name }}: {{ value.value }} {%- endfor %} """ -) - - -QUILT_SUMMARIZE = json.dumps( - [ - [ - { - "path": "entry.md", - "width": "calc(40% - 16px)", - "expand": True, - }, - { - "path": "notes.pdf", - "width": "calc(60% - 16px)", - "expand": True, - }, - ] - ] -) + def __init__(self, entry): + self.client = BenchlingClient.MakeProxy() + self.entry = entry + self.entry_id = entry["id"] + self.fields = entry["fields"] + self.pkg_name = PKG_PREFIX + entry["displayId"] -@logger.inject_lambda_context -def lambda_handler(event, context): - entry = event["detail"]["entry"] - task = benchling.tasks.wait_for_task( - benchling.exports.export( - benchling_models.ExportItemRequest(id=entry["id"]) - ).task_id - ) - if task.status != benchling_models.AsyncTaskStatus.SUCCEEDED: - raise Exception(f"Notes export failed: {task!r}") + def format(self): + template = jinja2.Template(self.ENTRY_FMT) + return template.render({"entry": self.entry}) + + def dump(self): + return json.dumps(self.entry) - with urllib.request.urlopen(task.response["downloadURL"]) as src: - buf = io.BytesIO(src.read()) - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir_path = pathlib.Path(tmpdir) + def write_notes(self, tmpdir_path): + task = self.client.get_task(self.entry_id) - (tmpdir_path / "entry.md").write_text(template.render({"entry": entry})) + with urllib_request.urlopen(task.response["downloadURL"]) as src: + buf = io.BytesIO(src.read()) with zipfile.ZipFile(buf) as zip_file: with zip_file.open(zip_file.namelist()[0]) as src: @@ -116,50 +148,58 @@ def lambda_handler(event, context): while data := src.read(4096): dst.write(data) - (tmpdir_path / "quilt_summarize.json").write_text(QUILT_SUMMARIZE) + def write(self, tmpdir_path): + self.write_notes(tmpdir_path) + (tmpdir_path / "entry.md").write_text(self.format()) + (tmpdir_path / "entry.json").write_text(self.dump()) + (tmpdir_path / "quilt_summarize.json").write_text(self.QUILT_SUMMARIZE) - pkg_name = PKG_PREFIX + entry["displayId"] + def make_package(self, tmpdir_path): registry = f"s3://{DST_BUCKET}" try: - pkg = quilt3.Package.browse(pkg_name, registry=registry) - except botocore.exceptions.ClientError as e: + pkg = quilt3.Package.browse(self.pkg_name, registry=registry) + except botocore_exceptions.ClientError as e: # XXX: quilt3 should raise some specific exception # when package doesn't exist. if e.response["Error"]["Code"] not in ("NoSuchKey", "404"): raise pkg = quilt3.Package() - pkg.set_dir( - ".", - tmpdir_path, - # This shouldn't hit 1 MB limit on metadata, - # because max size of EventBridge is 256 KiB. - meta=entry, - ).push( - pkg_name, - registry=registry, - ) + pkg.set_dir(".", tmpdir_path, meta=self.entry) + # This shouldn't hit 1 MB limit on metadata, + # because max size of EventBridge is 256 KiB. + pkg.push(self.pkg_name, registry=registry) + + def field_values(self): + QUILT_PREFIX = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages" + REVISE="action=revisePackage" + values = {} + if FIELD_URI in self.fields: + values[FIELD_URI] = f"quilt+s3://{DST_BUCKET}#package={self.pkg_name}" + if FIELD_CATALOG in self.fields: + values[FIELD_CATALOG] = f"{QUILT_PREFIX}/{self.pkg_name}" + if FIELD_REVISE in self.fields: + values[FIELD_REVISE] = f"{QUILT_PREFIX}/{self.pkg_name}?{REVISE}" + return values + + def update_benchling_notebook(self): + values = self.field_values() + if values: + self.client.update_entry(self.entry_id, values) + logger.debug(f"Updated entry {self.entry_id} with package {self.pkg_name}") + else: + logger.warning(f"Quilt schema fields not found for entry {self.entry_id!r}") - fields_values = {} - if "Quilt+ URI" in entry["fields"]: - fields_values["Quilt+ URI"] = f"quilt+s3://{DST_BUCKET}#package={pkg_name}" - if "Quilt Catalog URL" in entry["fields"]: - fields_values[ - "Quilt Catalog URL" - ] = f"{QUILT_PREFIX}/{pkg_name}" - if "Quilt Revise URL" in entry["fields"]: - fields_values[ - "Quilt Revise URL" - ] = f"{QUILT_PREFIX}/{pkg_name}?action=revisePackage" - - if fields_values: - benchling.entries.update_entry( - event["detail"]["entry"]["id"], - benchling_models.EntryUpdate( - fields=serialization_helpers.fields( - { - name: {"value": value} - for name, value in fields_values.items() - } - ) - ), - ) +@logger.inject_lambda_context +def lambda_handler(event, context): + entry = BenchlingEntry(event["detail"]["entry"]) + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = pathlib.Path(tmpdir) + + entry.write(tmpdir_path) + entry.make_package(tmpdir_path) + entry.update_benchling_notebook() + + return { + "statusCode": 200, + } \ No newline at end of file From 30da752b2fd735b3a0ede0fc2dfb3fafe526e505 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 18:14:02 -0700 Subject: [PATCH 09/33] hack iimports --- lambdas/__init__.py | 1 + lambdas/client.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ test/test_lambda.py | 5 ----- tests/__init__.py | 0 tests/test_lambda.py | 7 +++++++ 5 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 lambdas/__init__.py create mode 100644 lambdas/client.py delete mode 100644 test/test_lambda.py create mode 100644 tests/__init__.py create mode 100644 tests/test_lambda.py diff --git a/lambdas/__init__.py b/lambdas/__init__.py new file mode 100644 index 0000000..d2c3759 --- /dev/null +++ b/lambdas/__init__.py @@ -0,0 +1 @@ +from .client import BenchlingClient diff --git a/lambdas/client.py b/lambdas/client.py new file mode 100644 index 0000000..d9960f5 --- /dev/null +++ b/lambdas/client.py @@ -0,0 +1,46 @@ +import os + +from aws_lambda_powertools.utilities import parameters +from benchling_sdk import models as benchling_models +from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2 +from benchling_sdk.benchling import Benchling +from benchling_sdk.helpers import serialization_helpers + +class BenchlingClient(): + @classmethod + def MakeProxy(cls): + BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] + BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] + BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] + return cls(BENCHLING_TENANT, BENCHLING_CLIENT_ID, BENCHLING_CLIENT_SECRET_ARN) + + def __init__(self, tenant, id, arn): + secret = parameters.get_secret(arn) + if not isinstance(secret, str): + raise Exception(f"Failed to fetch secret: {arn!r}") + self.benchling = Benchling( + url=f"https://{tenant}.benchling.com", + auth_method=ClientCredentialsOAuth2( + client_id=id, + client_secret=secret, + ), + ) + + def get_task(self, entry_id): + self.task = self.benchling.tasks.wait_for_task( + self.benchling.exports.export( + benchling_models.ExportItemRequest(_id=entry_id) + ).task_id + ) + if self.task.status != benchling_models.AsyncTaskStatus.SUCCEEDED: + raise Exception(f"Notes export failed: {self.task!r}") + return self.task + + def update_entry(self, entry_id, fields_values): + values = {k: {"value": v} for k, v in fields_values.items()} + self.benchling.entries.update_entry( + entry_id, + benchling_models.EntryUpdate( + _fields=serialization_helpers.fields(values) + ), + ) diff --git a/test/test_lambda.py b/test/test_lambda.py deleted file mode 100644 index 04ab212..0000000 --- a/test/test_lambda.py +++ /dev/null @@ -1,5 +0,0 @@ -def test_lambda(): - assert True - -def test_env(): - pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_lambda.py b/tests/test_lambda.py new file mode 100644 index 0000000..c0c0567 --- /dev/null +++ b/tests/test_lambda.py @@ -0,0 +1,7 @@ +from lambdas import BenchlingClient + +def test_import(): + assert BenchlingClient + +def test_env(): + pass From 47dbef3a2401737aab663bb4e6eab8c4b4cb92a1 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 18:23:29 -0700 Subject: [PATCH 10/33] rename lambda.py to allow import --- Makefile | 1 + lambdas/__init__.py | 2 +- lambdas/client.py | 46 ---------------------------------- lambdas/{lambda.py => main.py} | 0 make.py | 2 +- 5 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 lambdas/client.py rename lambdas/{lambda.py => main.py} (100%) diff --git a/Makefile b/Makefile index 8fc65bf..087d068 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +sinclude .env TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" diff --git a/lambdas/__init__.py b/lambdas/__init__.py index d2c3759..2d23ee1 100644 --- a/lambdas/__init__.py +++ b/lambdas/__init__.py @@ -1 +1 @@ -from .client import BenchlingClient +from .main import BenchlingClient diff --git a/lambdas/client.py b/lambdas/client.py deleted file mode 100644 index d9960f5..0000000 --- a/lambdas/client.py +++ /dev/null @@ -1,46 +0,0 @@ -import os - -from aws_lambda_powertools.utilities import parameters -from benchling_sdk import models as benchling_models -from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2 -from benchling_sdk.benchling import Benchling -from benchling_sdk.helpers import serialization_helpers - -class BenchlingClient(): - @classmethod - def MakeProxy(cls): - BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] - BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] - BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] - return cls(BENCHLING_TENANT, BENCHLING_CLIENT_ID, BENCHLING_CLIENT_SECRET_ARN) - - def __init__(self, tenant, id, arn): - secret = parameters.get_secret(arn) - if not isinstance(secret, str): - raise Exception(f"Failed to fetch secret: {arn!r}") - self.benchling = Benchling( - url=f"https://{tenant}.benchling.com", - auth_method=ClientCredentialsOAuth2( - client_id=id, - client_secret=secret, - ), - ) - - def get_task(self, entry_id): - self.task = self.benchling.tasks.wait_for_task( - self.benchling.exports.export( - benchling_models.ExportItemRequest(_id=entry_id) - ).task_id - ) - if self.task.status != benchling_models.AsyncTaskStatus.SUCCEEDED: - raise Exception(f"Notes export failed: {self.task!r}") - return self.task - - def update_entry(self, entry_id, fields_values): - values = {k: {"value": v} for k, v in fields_values.items()} - self.benchling.entries.update_entry( - entry_id, - benchling_models.EntryUpdate( - _fields=serialization_helpers.fields(values) - ), - ) diff --git a/lambdas/lambda.py b/lambdas/main.py similarity index 100% rename from lambdas/lambda.py rename to lambdas/main.py diff --git a/make.py b/make.py index cf47d48..08bd445 100644 --- a/make.py +++ b/make.py @@ -183,7 +183,7 @@ def make_template(*, metadata: dict) -> troposphere.Template: QUILT_CATALOG_DOMAIN=quilt_domain.ref(), ), Handler="index.lambda_handler", - Code=awslambda.Code(ZipFile=(LAMBDAS_DIR / "lambda.py").read_text()), + Code=awslambda.Code(ZipFile=(LAMBDAS_DIR / "main.py").read_text()), ReservedConcurrentExecutions=1, # FIXME MemorySize=512, ) From acad4f49df4558b82d38bef2e6ca5c65c8b1e884 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 20:28:44 -0700 Subject: [PATCH 11/33] test_client --- lambdas/__init__.py | 2 +- lambdas/main.py | 50 +++++++++++++++++++++++--------------------- tests/test_lambda.py | 10 +++++++-- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/lambdas/__init__.py b/lambdas/__init__.py index 2d23ee1..22de419 100644 --- a/lambdas/__init__.py +++ b/lambdas/__init__.py @@ -1 +1 @@ -from .main import BenchlingClient +from .main import BenchlingClient, BenchlingEntry diff --git a/lambdas/main.py b/lambdas/main.py index 55921b4..1e12568 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -21,24 +21,18 @@ logger = Logger() -BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] -BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] -BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] -DST_BUCKET = os.environ["DST_BUCKET"] -PKG_PREFIX = os.environ["PKG_PREFIX"] -QUILT_CATALOG_DOMAIN = os.environ["QUILT_CATALOG_DOMAIN"] - -FIELD_URI = "Quilt+ URI" -FIELD_CATALOG = "Quilt Catalog URL" -FIELD_REVISE = "Quilt Revise URL" - class BenchlingClient(): + BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] + BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] + BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] + @classmethod - def MakeProxy(cls): - BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] - BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] - BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] - return cls(BENCHLING_TENANT, BENCHLING_CLIENT_ID, BENCHLING_CLIENT_SECRET_ARN) + def Default(cls): + return cls( + cls.BENCHLING_TENANT, + cls.BENCHLING_CLIENT_ID, + cls.BENCHLING_CLIENT_SECRET_ARN, + ) def __init__(self, tenant, id, arn): secret = parameters.get_secret(arn) @@ -121,12 +115,17 @@ class BenchlingEntry: {%- endfor %} """ + DST_BUCKET = os.environ["DST_BUCKET"] + PKG_PREFIX = os.environ["PKG_PREFIX"] + QUILT_CATALOG_DOMAIN = os.environ["QUILT_CATALOG_DOMAIN"] + QUILT_PREFIX = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages" + def __init__(self, entry): - self.client = BenchlingClient.MakeProxy() + self.client = BenchlingClient.Default() self.entry = entry self.entry_id = entry["id"] self.fields = entry["fields"] - self.pkg_name = PKG_PREFIX + entry["displayId"] + self.pkg_name = self.PKG_PREFIX + entry["displayId"] def format(self): template = jinja2.Template(self.ENTRY_FMT) @@ -155,7 +154,8 @@ def write(self, tmpdir_path): (tmpdir_path / "quilt_summarize.json").write_text(self.QUILT_SUMMARIZE) def make_package(self, tmpdir_path): - registry = f"s3://{DST_BUCKET}" + registry = f"s3://{self.DST_BUCKET}" + pkg = quilt3.Package() try: pkg = quilt3.Package.browse(self.pkg_name, registry=registry) except botocore_exceptions.ClientError as e: @@ -163,22 +163,24 @@ def make_package(self, tmpdir_path): # when package doesn't exist. if e.response["Error"]["Code"] not in ("NoSuchKey", "404"): raise - pkg = quilt3.Package() + pkg.set_dir(".", tmpdir_path, meta=self.entry) # This shouldn't hit 1 MB limit on metadata, # because max size of EventBridge is 256 KiB. pkg.push(self.pkg_name, registry=registry) def field_values(self): - QUILT_PREFIX = f"https://{QUILT_CATALOG_DOMAIN}/b/{DST_BUCKET}/packages" + FIELD_URI = "Quilt+ URI" + FIELD_CATALOG = "Quilt Catalog URL" + FIELD_REVISE = "Quilt Revise URL" REVISE="action=revisePackage" values = {} if FIELD_URI in self.fields: - values[FIELD_URI] = f"quilt+s3://{DST_BUCKET}#package={self.pkg_name}" + values[FIELD_URI] = f"quilt+s3://{self.DST_BUCKET}#package={self.pkg_name}" if FIELD_CATALOG in self.fields: - values[FIELD_CATALOG] = f"{QUILT_PREFIX}/{self.pkg_name}" + values[FIELD_CATALOG] = f"{self.QUILT_PREFIX}/{self.pkg_name}" if FIELD_REVISE in self.fields: - values[FIELD_REVISE] = f"{QUILT_PREFIX}/{self.pkg_name}?{REVISE}" + values[FIELD_REVISE] = f"{self.QUILT_PREFIX}/{self.pkg_name}?{REVISE}" return values def update_benchling_notebook(self): diff --git a/tests/test_lambda.py b/tests/test_lambda.py index c0c0567..d044e62 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -1,7 +1,13 @@ -from lambdas import BenchlingClient +from lambdas import BenchlingClient, BenchlingEntry def test_import(): assert BenchlingClient + assert BenchlingEntry def test_env(): - pass + assert BenchlingClient.BENCHLING_TENANT + assert BenchlingEntry.DST_BUCKET + +def test_client(): + client = BenchlingClient.Default() + assert client From faf235aec3c713c382bd9ca31114d653f6734c85 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 20:32:42 -0700 Subject: [PATCH 12/33] secrets --- .github/workflows/python-package.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0bfe961..520e49a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,9 @@ jobs: run: | make test TEST_OS=${{ matrix.os }} env: - BENCHLING_API_KEY: ${{ secrets.BENCHLING_API_KEY }} - BENCHLING_AUTHOR_ID: ${{ secrets.BENCHLING_AUTHOR_ID }} - BENCHLING_ENTRY_ID: ${{ secrets.BENCHLING_ENTRY_ID }} - BENCHLING_TENANT_DNS: ${{ vars.BENCHLING_TENANT_DNS }} + BENCHLING_TENANT: ${{ secrets.BENCHLING_TENANT }} + BENCHLING_CLIENT_ID: ${{ secrets.BENCHLING_CLIENT_ID }} + BENCHLING_CLIENT_SECRET_ARN: ${{ secrets.BENCHLING_CLIENT_SECRET_ARN }} + DST_BUCKET: ${{ secrets.DST_BUCKET }} + PKG_PREFIX: ${{ secrets.PKG_PREFIX }} + QUILT_CATALOG_DOMAIN: ${{ secrets.QUILT_CATALOG_DOMAIN }} From c8ca054f9597f1f94b950ced05a4d9019c1ccffa Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 14 Aug 2023 20:44:27 -0700 Subject: [PATCH 13/33] debug arn error --- lambdas/main.py | 2 ++ tests/test_lambda.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/lambdas/main.py b/lambdas/main.py index 1e12568..1bee26a 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -35,6 +35,8 @@ def Default(cls): ) def __init__(self, tenant, id, arn): + if not isinstance(arn, str): + raise Exception(f"Failed to fetch CLIENT_SECRET_ARN") secret = parameters.get_secret(arn) if not isinstance(secret, str): raise Exception(f"Failed to fetch secret: {arn!r}") diff --git a/tests/test_lambda.py b/tests/test_lambda.py index d044e62..ce1419d 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -1,13 +1,25 @@ +from aws_lambda_powertools.utilities import parameters from lambdas import BenchlingClient, BenchlingEntry + def test_import(): assert BenchlingClient assert BenchlingEntry + def test_env(): assert BenchlingClient.BENCHLING_TENANT assert BenchlingEntry.DST_BUCKET + +def test_secret(): + arn = BenchlingClient.BENCHLING_CLIENT_SECRET_ARN + assert arn + assert "us-east-1" in arn + secret = parameters.get_secret(arn) + assert secret + + def test_client(): client = BenchlingClient.Default() assert client From 2187e37128d911b7a2feb1eb19c749922b5417d9 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 06:20:20 -0700 Subject: [PATCH 14/33] Configure AWS Credentials --- .github/workflows/python-package.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 520e49a..230e24a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,6 +26,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Deployment + aws-region: us-east-1 - name: Test with pytest run: | make test TEST_OS=${{ matrix.os }} From b866f57614662f6e411cd2c6fdd41c07bde8f3bc Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 06:24:52 -0700 Subject: [PATCH 15/33] Update README.md sync with main --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6065d16..76d5034 100644 --- a/README.md +++ b/README.md @@ -5,20 +5,25 @@ This repository generates a CloudFormation template for processing (and link, if possible) a [Quilt](https://quiltdata.com/) package for every Benchling notebook. -## Template generation and upload +## Template generation Requires a recent version of Python 3. -```bash -make all +```shell +python3 -m venv venv +. ./venv/bin/activate +python3 -m pip install -r requirements.txt +python3 make.py > build/benchling_packager.yaml ``` -This will: +## Template upload -- setup the Python environment -- generate the template in the `build` directory -- upload the template to a Quilt package (if you have appropriate permissions) -- open the package URL: +Currently it's distributed as a Quilt [package](https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager) +which is this way: + +```python +quilt3.Package().set('README.md', 'Install.md').set('benchling_packager.yaml', 'build/benchling_packager.yaml').push('examples/benchling-packager', 's3://quilt-example') +``` ## Installation From 5918412e073da327dc6039148cb9b65b72f983db Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 06:34:34 -0700 Subject: [PATCH 16/33] use new Role --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 230e24a..d0259a9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,7 +29,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Deployment + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-BenchlingPackager aws-region: us-east-1 - name: Test with pytest run: | From a9aa42cb5aea44ec1e007226c221293bce773e0a Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 07:32:42 -0700 Subject: [PATCH 17/33] permissions --- .github/workflows/python-package.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d0259a9..55bbbe8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,7 +12,11 @@ permissions: read-all jobs: build: - + permissions: + contents: read + id-token: write + issues: write + pull-requests: write strategy: fail-fast: false matrix: From 9f5f9b3746faefac937e9f4031c8515e7f0870f0 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 11:03:26 -0700 Subject: [PATCH 18/33] match add-makefile --- Makefile | 24 +++++++++++++----------- dev-requirements.txt | 10 +++++++--- requirements.txt | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 087d068..fe27bfa 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ sinclude .env TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" -.PHONY: all clean install pip-compile pip-dev template test upload +.PHONY: all check-python39 clean install pip-compile pip-dev template upload -all: clean template upload +all: template upload clean: rm -rf build @@ -14,28 +14,30 @@ clean: template: $(TARGET) +$(TARGET): build install make.py lambdas/main.py + . $(ACTIVATE) && python3 make.py > $(TARGET) + upload: . $(ACTIVATE) && python3 upload.py open $(PKG_URL) +build: + mkdir -p build + install: venv requirements.txt -# . $(ACTIVATE) && python3 -m pip install -r requirements.txt . $(ACTIVATE) && pip-sync requirements.txt -$(TARGET): build install make.py lambdas/lambda.py - . $(ACTIVATE) && python3 make.py > $(TARGET) +venv: check-python39 + python3.9 -m venv venv + +check-python39: + @python3.9 --version|| (echo "Python 3.9 required" && exit 1) test: pip-dev run-test run-test: venv . $(ACTIVATE) && python3 -m pytest -venv: - python3 -m venv venv - -build: - mkdir build - tools: venv . $(ACTIVATE) && python3 -m pip install pip-tools diff --git a/dev-requirements.txt b/dev-requirements.txt index 298bd90..860c0eb 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile dev-requirements.in @@ -49,6 +49,10 @@ dataclasses-json==0.5.14 # benchling-sdk deprecated==1.2.14 # via jwcrypto +exceptiongroup==1.1.3 + # via + # anyio + # pytest h11==0.14.0 # via httpcore httpcore==0.17.3 @@ -94,8 +98,6 @@ platformdirs==3.10.0 # via quilt3 pluggy==1.2.0 # via pytest -psutil==5.9.5 - # via benchling-sdk pycparser==2.21 # via cffi pytest==7.4.0 @@ -139,6 +141,8 @@ sniffio==1.3.0 # httpx tenacity==8.2.3 # via quilt3 +tomli==2.0.1 + # via pytest tqdm==4.66.1 # via quilt3 typing-extensions==4.7.1 diff --git a/requirements.txt b/requirements.txt index b6aecaa..c7a63c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile requirements.in From 8962e87e545b405d76f147bea96ea35d8482895f Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 11:14:43 -0700 Subject: [PATCH 19/33] autolint --- Makefile | 2 +- lambdas/__init__.py | 2 +- lambdas/main.py | 28 ++++++++++++++-------------- layer/deploy.sh | 30 ++++++++++++++---------------- tests/test_lambda.py | 4 ++-- 5 files changed, 32 insertions(+), 34 deletions(-) mode change 100755 => 100644 layer/deploy.sh diff --git a/Makefile b/Makefile index fe27bfa..9b07284 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ sinclude .env TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" -.PHONY: all check-python39 clean install pip-compile pip-dev template upload +.PHONY: all check-python39 clean install pip-compile pip-dev template test upload all: template upload diff --git a/lambdas/__init__.py b/lambdas/__init__.py index 22de419..272d1aa 100644 --- a/lambdas/__init__.py +++ b/lambdas/__init__.py @@ -1 +1 @@ -from .main import BenchlingClient, BenchlingEntry +from .main import BenchlingClient, BenchlingEntry # noqa: F401 diff --git a/lambdas/main.py b/lambdas/main.py index 1bee26a..32e75f9 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -4,8 +4,9 @@ import pathlib import tempfile import zipfile -import jinja2 +from urllib import request as urllib_request +import jinja2 from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities import parameters from benchling_sdk import models as benchling_models @@ -13,7 +14,6 @@ from benchling_sdk.benchling import Benchling from benchling_sdk.helpers import serialization_helpers from botocore import exceptions as botocore_exceptions -from urllib import request as urllib_request # Must be done before importing quilt3 os.environ["QUILT_DISABLE_CACHE"] = "true" @@ -21,7 +21,8 @@ logger = Logger() -class BenchlingClient(): + +class BenchlingClient: BENCHLING_TENANT = os.environ["BENCHLING_TENANT"] BENCHLING_CLIENT_ID = os.environ["BENCHLING_CLIENT_ID"] BENCHLING_CLIENT_SECRET_ARN = os.environ["BENCHLING_CLIENT_SECRET_ARN"] @@ -36,7 +37,7 @@ def Default(cls): def __init__(self, tenant, id, arn): if not isinstance(arn, str): - raise Exception(f"Failed to fetch CLIENT_SECRET_ARN") + raise Exception("Failed to fetch CLIENT_SECRET_ARN") secret = parameters.get_secret(arn) if not isinstance(secret, str): raise Exception(f"Failed to fetch secret: {arn!r}") @@ -62,11 +63,10 @@ def update_entry(self, entry_id, fields_values): values = {k: {"value": v} for k, v in fields_values.items()} self.benchling.entries.update_entry( entry_id, - benchling_models.EntryUpdate( - _fields=serialization_helpers.fields(values) - ), + benchling_models.EntryUpdate(_fields=serialization_helpers.fields(values)), ) + class BenchlingEntry: QUILT_SUMMARIZE = json.dumps( [ @@ -132,11 +132,10 @@ def __init__(self, entry): def format(self): template = jinja2.Template(self.ENTRY_FMT) return template.render({"entry": self.entry}) - + def dump(self): return json.dumps(self.entry) - def write_notes(self, tmpdir_path): task = self.client.get_task(self.entry_id) @@ -161,7 +160,7 @@ def make_package(self, tmpdir_path): try: pkg = quilt3.Package.browse(self.pkg_name, registry=registry) except botocore_exceptions.ClientError as e: - # XXX: quilt3 should raise some specific exception + # XXX: quilt3 should raise some specific exception # when package doesn't exist. if e.response["Error"]["Code"] not in ("NoSuchKey", "404"): raise @@ -175,7 +174,7 @@ def field_values(self): FIELD_URI = "Quilt+ URI" FIELD_CATALOG = "Quilt Catalog URL" FIELD_REVISE = "Quilt Revise URL" - REVISE="action=revisePackage" + REVISE = "action=revisePackage" values = {} if FIELD_URI in self.fields: values[FIELD_URI] = f"quilt+s3://{self.DST_BUCKET}#package={self.pkg_name}" @@ -193,17 +192,18 @@ def update_benchling_notebook(self): else: logger.warning(f"Quilt schema fields not found for entry {self.entry_id!r}") + @logger.inject_lambda_context def lambda_handler(event, context): entry = BenchlingEntry(event["detail"]["entry"]) - + with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = pathlib.Path(tmpdir) - + entry.write(tmpdir_path) entry.make_package(tmpdir_path) entry.update_benchling_notebook() return { "statusCode": 200, - } \ No newline at end of file + } diff --git a/layer/deploy.sh b/layer/deploy.sh old mode 100755 new mode 100644 index 8547aab..8efc999 --- a/layer/deploy.sh +++ b/layer/deploy.sh @@ -3,8 +3,8 @@ set -euo pipefail error() { - echo "$@" 2>&1 - exit 1 + echo "$@" 2>&1 + exit 1 } [ "$#" -eq 0 ] || error "Usage: $0" @@ -17,13 +17,13 @@ cd "$work_dir" echo "Installing packages..." python3 -m pip install \ - --platform manylinux2014_x86_64 \ - --target=./python/lib/python3.9/site-packages \ - --implementation cp \ - --python 3.9 \ - --no-deps \ - --no-compile \ - -r "$exec_dir/requirements.txt" + --platform manylinux2014_x86_64 \ + --target=./python/lib/python3.9/site-packages \ + --implementation cp \ + --python 3.9 \ + --no-deps \ + --no-compile \ + -r "$exec_dir/requirements.txt" echo "Compressing..." zip -9 -r "$zip_file" "." @@ -39,13 +39,11 @@ aws s3 cp --acl public-read "$zip_file" "s3://quilt-lambda-$primary_region/$s3_k cd .. rm -rf "$work_dir" -for region in $regions -do - if [ "$region" != "$primary_region" ] - then - echo "Copying to $region..." - aws s3 cp --acl public-read "s3://quilt-lambda-$primary_region/$s3_key" "s3://quilt-lambda-$region/$s3_key" --region "$region" --source-region "$primary_region" - fi +for region in $regions; do + if [ "$region" != "$primary_region" ]; then + echo "Copying to $region..." + aws s3 cp --acl public-read "s3://quilt-lambda-$primary_region/$s3_key" "s3://quilt-lambda-$region/$s3_key" --region "$region" --source-region "$primary_region" + fi done echo "Deployed $s3_key" diff --git a/tests/test_lambda.py b/tests/test_lambda.py index ce1419d..2c61d3d 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -1,5 +1,5 @@ from aws_lambda_powertools.utilities import parameters -from lambdas import BenchlingClient, BenchlingEntry +from lambdas import BenchlingClient, BenchlingEntry def test_import(): @@ -18,7 +18,7 @@ def test_secret(): assert "us-east-1" in arn secret = parameters.get_secret(arn) assert secret - + def test_client(): client = BenchlingClient.Default() From e294b2141e85f19afd55faba08004df14c26b941 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 11:21:07 -0700 Subject: [PATCH 20/33] try to enable pytest --- .github/workflows/python-package.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 55bbbe8..1d0f8d4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -23,7 +23,6 @@ jobs: python-version: ["3.9"] #, "3.10", "3.11" os: [ubuntu-latest] # , macos-latest, windows-latest runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 4163855d68aea0bc70c04c47ba8dcfc3423ecdc8 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 12:14:54 -0700 Subject: [PATCH 21/33] streamline Makefile --- Makefile | 36 ++++++++++++++++-------------------- dev-requirements.txt | 6 +++--- requirements.txt | 1 + 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 9b07284..984eab4 100644 --- a/Makefile +++ b/Makefile @@ -2,19 +2,18 @@ sinclude .env TARGET = build/benchling_packager.yaml ACTIVATE = ./venv/bin/activate PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchling-packager" -.PHONY: all check-python39 clean install pip-compile pip-dev template test upload +.PHONY: all clean install install-dev template test upload all: template upload clean: rm -rf build rm -rf venv - rm -f requirements.txt - rm -f dev-requirements.txt + rm -f *requirements.txt template: $(TARGET) -$(TARGET): build install make.py lambdas/main.py +$(TARGET): build venv install make.py lambdas/main.py . $(ACTIVATE) && python3 make.py > $(TARGET) upload: @@ -24,31 +23,28 @@ upload: build: mkdir -p build -install: venv requirements.txt - . $(ACTIVATE) && pip-sync requirements.txt - -venv: check-python39 - python3.9 -m venv venv - -check-python39: +venv: @python3.9 --version|| (echo "Python 3.9 required" && exit 1) + python3.9 -m venv venv -test: pip-dev run-test - -run-test: venv - . $(ACTIVATE) && python3 -m pytest +venv/bin/pip-compile: + . $(ACTIVATE) && python3 -m pip install pip-tools -tools: venv +venv/bin/pip-sync: . $(ACTIVATE) && python3 -m pip install pip-tools -pip-compile: requirements.txt +install: venv/bin/pip-sync requirements.txt + . $(ACTIVATE) && pip-sync requirements.txt -requirements.txt: tools requirements.in +requirements.txt: venv/bin/pip-compile requirements.in . $(ACTIVATE) && pip-compile requirements.in -pip-dev: venv dev-requirements.txt +test: venv install-dev + . $(ACTIVATE) && python3 -m pytest + +install-dev: venv/bin/pip-sync dev-requirements.txt . $(ACTIVATE) && pip-sync dev-requirements.txt -dev-requirements.txt: tools dev-requirements.in +dev-requirements.txt: venv/bin/pip-sync dev-requirements.in . $(ACTIVATE) && pip-compile dev-requirements.in diff --git a/dev-requirements.txt b/dev-requirements.txt index 860c0eb..8a871b2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -24,9 +24,9 @@ benchling-api-client==2.0.167 # via benchling-sdk benchling-sdk==1.7.0 # via -r dev-requirements.in -boto3==1.28.26 +boto3==1.28.27 # via quilt3 -botocore==1.31.26 +botocore==1.31.27 # via # -r dev-requirements.in # boto3 @@ -128,7 +128,7 @@ rpds-py==0.9.2 # via # jsonschema # referencing -s3transfer==0.6.1 +s3transfer==0.6.2 # via boto3 six==1.16.0 # via diff --git a/requirements.txt b/requirements.txt index 8d61f79..8cc8394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ boto3==1.28.27 # quilt3 botocore==1.31.27 # via + # -r requirements.in # boto3 # s3transfer certifi==2023.7.22 From 6ca3528940712dfb565056ad00dbfd5c847cc021 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 12:27:54 -0700 Subject: [PATCH 22/33] add coverage --- Makefile | 7 +++---- dev-requirements.in | 2 ++ dev-requirements.txt | 19 ++++++++++++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 984eab4..f07ae01 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,8 @@ PKG_URL = "https://open.quiltdata.com/b/quilt-example/packages/examples/benchlin all: template upload clean: - rm -rf build - rm -rf venv - rm -f *requirements.txt + rm -rf build venv + rm -f *requirements.txt .pytest_cache .DS_Store template: $(TARGET) @@ -40,7 +39,7 @@ requirements.txt: venv/bin/pip-compile requirements.in . $(ACTIVATE) && pip-compile requirements.in test: venv install-dev - . $(ACTIVATE) && python3 -m pytest + . $(ACTIVATE) && python3 -m pytest --cov --cov-report xml:coverage.xml install-dev: venv/bin/pip-sync dev-requirements.txt . $(ACTIVATE) && pip-sync dev-requirements.txt diff --git a/dev-requirements.in b/dev-requirements.in index a6f0457..a342f7d 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -5,3 +5,5 @@ botocore ~= 1.31 jinja2 ~= 3.1 quilt3 ~=5.3 pytest +pytest-coverage +pytest-watcher diff --git a/dev-requirements.txt b/dev-requirements.txt index 8a871b2..8509eaa 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -41,6 +41,8 @@ cffi==1.15.1 # via cryptography charset-normalizer==3.2.0 # via requests +coverage[toml]==7.3.0 + # via pytest-cov cryptography==41.0.3 # via jwcrypto dataclasses-json==0.5.14 @@ -101,6 +103,16 @@ pluggy==1.2.0 pycparser==2.21 # via cffi pytest==7.4.0 + # via + # -r dev-requirements.in + # pytest-cov +pytest-cov==4.1.0 + # via pytest-cover +pytest-cover==3.0.0 + # via pytest-coverage +pytest-coverage==0.0 + # via -r dev-requirements.in +pytest-watcher==0.3.4 # via -r dev-requirements.in python-dateutil==2.8.2 # via @@ -142,7 +154,10 @@ sniffio==1.3.0 tenacity==8.2.3 # via quilt3 tomli==2.0.1 - # via pytest + # via + # coverage + # pytest + # pytest-watcher tqdm==4.66.1 # via quilt3 typing-extensions==4.7.1 @@ -157,5 +172,7 @@ urllib3==1.26.16 # via # botocore # requests +watchdog==3.0.0 + # via pytest-watcher wrapt==1.15.0 # via deprecated From 1a321612877f3d7f7c29468e9f69a64ba464b180 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 12:27:59 -0700 Subject: [PATCH 23/33] Update python-package.yml --- .github/workflows/python-package.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1d0f8d4..a2cde4e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,8 +20,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9"] #, "3.10", "3.11" - os: [ubuntu-latest] # , macos-latest, windows-latest + python-version: ["3.9"] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -32,7 +32,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-BenchlingPackager + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Deployment aws-region: us-east-1 - name: Test with pytest run: | @@ -44,3 +44,10 @@ jobs: DST_BUCKET: ${{ secrets.DST_BUCKET }} PKG_PREFIX: ${{ secrets.PKG_PREFIX }} QUILT_CATALOG_DOMAIN: ${{ secrets.QUILT_CATALOG_DOMAIN }} + - name: Get Coverage Report + uses: orgoro/coverage@v3.1 + with: + coverageFile: coverage.xml + token: ${{ secrets.GITHUB_TOKEN }} + thresholdAll: 0.8 + if: github.event_name == 'pull_request' From 8c9c02eb29b973428d21ffb4aa19cc7ec56431d6 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 12:30:21 -0700 Subject: [PATCH 24/33] run on all push --- .github/workflows/python-package.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a2cde4e..3b59f71 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,13 +1,8 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - +# This workflow will install Python dependencies and run tests name: Python package on: push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] permissions: read-all jobs: From 200259c590481ee2c62c414d0b29b78b4e489591 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 12:38:29 -0700 Subject: [PATCH 25/33] test correctly --- .github/workflows/python-package.yml | 2 +- Makefile | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3b59f71..05bd687 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,7 +27,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Deployment + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-BenchlingPackager aws-region: us-east-1 - name: Test with pytest run: | diff --git a/Makefile b/Makefile index f07ae01..59aba64 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ all: template upload clean: rm -rf build venv rm -f *requirements.txt .pytest_cache .DS_Store + rm -f .coverage coverage.xml template: $(TARGET) @@ -41,6 +42,13 @@ requirements.txt: venv/bin/pip-compile requirements.in test: venv install-dev . $(ACTIVATE) && python3 -m pytest --cov --cov-report xml:coverage.xml +coverage: venv install-dev + . $(ACTIVATE) && python3 -m --cov --cov-report html:coverage.html + open coverage.html/index.html + +watch: venv install-dev + . $(ACTIVATE) && ptw . --now + install-dev: venv/bin/pip-sync dev-requirements.txt . $(ACTIVATE) && pip-sync dev-requirements.txt From 7c908b9033f5b9bb82414fd5011ed2ccc4e5065a Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 14:42:03 -0700 Subject: [PATCH 26/33] stub test_entry --- Makefile | 3 ++- lambdas/main.py | 30 ++++++++++++++++-------------- tests/test_entry.py | 24 ++++++++++++++++++++++++ tests/test_lambda.py | 2 +- 4 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 tests/test_entry.py diff --git a/Makefile b/Makefile index 59aba64..c60607a 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,8 @@ test: venv install-dev . $(ACTIVATE) && python3 -m pytest --cov --cov-report xml:coverage.xml coverage: venv install-dev - . $(ACTIVATE) && python3 -m --cov --cov-report html:coverage.html + printenv BENCHLING_ENTRY_ID + BENCHLING_ENTRY_ID=$(BENCHLING_ENTRY_ID) . $(ACTIVATE) && python3 -m pytest --cov --cov-report html:coverage.html open coverage.html/index.html watch: venv install-dev diff --git a/lambdas/main.py b/lambdas/main.py index 32e75f9..c001ea2 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -68,6 +68,8 @@ def update_entry(self, entry_id, fields_values): class BenchlingEntry: + REVISE = "action=revisePackage" + QUILT_SUMMARIZE = json.dumps( [ [ @@ -85,6 +87,12 @@ class BenchlingEntry: ] ) + FLD = { + "URI": "Quilt+ URI", + "CAT": "Quilt Catalog URL", + "REV": "Quilt Revise URL", + } + ENTRY_FMT = """ # [{{ entry.name }}]({{ entry.webURL }}) @@ -126,8 +134,8 @@ def __init__(self, entry): self.client = BenchlingClient.Default() self.entry = entry self.entry_id = entry["id"] - self.fields = entry["fields"] - self.pkg_name = self.PKG_PREFIX + entry["displayId"] + self.fields = entry.get("fields", {}) + self.pkg_name = self.PKG_PREFIX + entry.get("displayId", self.entry_id) def format(self): template = jinja2.Template(self.ENTRY_FMT) @@ -171,18 +179,12 @@ def make_package(self, tmpdir_path): pkg.push(self.pkg_name, registry=registry) def field_values(self): - FIELD_URI = "Quilt+ URI" - FIELD_CATALOG = "Quilt Catalog URL" - FIELD_REVISE = "Quilt Revise URL" - REVISE = "action=revisePackage" - values = {} - if FIELD_URI in self.fields: - values[FIELD_URI] = f"quilt+s3://{self.DST_BUCKET}#package={self.pkg_name}" - if FIELD_CATALOG in self.fields: - values[FIELD_CATALOG] = f"{self.QUILT_PREFIX}/{self.pkg_name}" - if FIELD_REVISE in self.fields: - values[FIELD_REVISE] = f"{self.QUILT_PREFIX}/{self.pkg_name}?{REVISE}" - return values + values = { + "URI": f"quilt+s3://{self.DST_BUCKET}#package={self.pkg_name}", + "CAT": f"{self.QUILT_PREFIX}/{self.pkg_name}", + "REV": f"{self.QUILT_PREFIX}/{self.pkg_name}?{self.REVISE}", + } + return {f: values.get(k) for k, f in self.FLD.items()} def update_benchling_notebook(self): values = self.field_values() diff --git a/tests/test_entry.py b/tests/test_entry.py new file mode 100644 index 0000000..c0b9f5a --- /dev/null +++ b/tests/test_entry.py @@ -0,0 +1,24 @@ +import os +import pytest + +from lambdas import BenchlingEntry + +try: + BENCHLING_ENTRY_ID=os.environ["BENCHLING_ENTRY_ID"] +except KeyError: + pytest.skip(allow_module_level=True) + +@pytest.fixture +def entry(): + entry_dict = { + "id": BENCHLING_ENTRY_ID, + } + return BenchlingEntry(entry_dict) + +def test_entry(entry): + assert entry + assert entry.entry_id == BENCHLING_ENTRY_ID + assert entry.fields == {} + assert entry.pkg_name + assert BenchlingEntry.PKG_PREFIX in entry.pkg_name + diff --git a/tests/test_lambda.py b/tests/test_lambda.py index 2c61d3d..ddaedb3 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -1,4 +1,4 @@ -from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities import parameters # type: ignore from lambdas import BenchlingClient, BenchlingEntry From 46c4a3da6a5ed063cb391c48bca079ef965e59c0 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 15:45:10 -0700 Subject: [PATCH 27/33] test with ENTRY_DATA --- Makefile | 2 +- lambdas/__init__.py | 2 +- lambdas/main.py | 49 ++++++++++++++--------- tests/test_entry.py | 92 +++++++++++++++++++++++++++++++++++++++++--- tests/test_lambda.py | 4 +- 5 files changed, 123 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index c60607a..0971b6a 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ test: venv install-dev coverage: venv install-dev printenv BENCHLING_ENTRY_ID - BENCHLING_ENTRY_ID=$(BENCHLING_ENTRY_ID) . $(ACTIVATE) && python3 -m pytest --cov --cov-report html:coverage.html + . $(ACTIVATE) && python3 -m pytest --cov --cov-report html:coverage.html open coverage.html/index.html watch: venv install-dev diff --git a/lambdas/__init__.py b/lambdas/__init__.py index 272d1aa..610510b 100644 --- a/lambdas/__init__.py +++ b/lambdas/__init__.py @@ -1 +1 @@ -from .main import BenchlingClient, BenchlingEntry # noqa: F401 +from .main import BenchlingClient, BenchlingEntry, main # noqa: F401 diff --git a/lambdas/main.py b/lambdas/main.py index c001ea2..3f2d6b2 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -52,7 +52,7 @@ def __init__(self, tenant, id, arn): def get_task(self, entry_id): self.task = self.benchling.tasks.wait_for_task( self.benchling.exports.export( - benchling_models.ExportItemRequest(_id=entry_id) + benchling_models.ExportItemRequest(id=entry_id) # type: ignore ).task_id ) if self.task.status != benchling_models.AsyncTaskStatus.SUCCEEDED: @@ -61,9 +61,10 @@ def get_task(self, entry_id): def update_entry(self, entry_id, fields_values): values = {k: {"value": v} for k, v in fields_values.items()} + fields = serialization_helpers.fields(values) self.benchling.entries.update_entry( entry_id, - benchling_models.EntryUpdate(_fields=serialization_helpers.fields(values)), + benchling_models.EntryUpdate(fields=fields), # type: ignore ) @@ -135,7 +136,13 @@ def __init__(self, entry): self.entry = entry self.entry_id = entry["id"] self.fields = entry.get("fields", {}) - self.pkg_name = self.PKG_PREFIX + entry.get("displayId", self.entry_id) + self.pkg_name = self.name() + self.registry = f"s3://{self.DST_BUCKET}" + + def name(self): + SEP = "/" if self.PKG_PREFIX[-1] != "/" else "" + return self.PKG_PREFIX + SEP + self.entry.get("displayId", self.entry_id) + def format(self): template = jinja2.Template(self.ENTRY_FMT) @@ -145,6 +152,7 @@ def dump(self): return json.dumps(self.entry) def write_notes(self, tmpdir_path): + outfile = tmpdir_path / "notes.pdf" task = self.client.get_task(self.entry_id) with urllib_request.urlopen(task.response["downloadURL"]) as src: @@ -152,21 +160,21 @@ def write_notes(self, tmpdir_path): with zipfile.ZipFile(buf) as zip_file: with zip_file.open(zip_file.namelist()[0]) as src: - with (tmpdir_path / "notes.pdf").open("wb") as dst: + with outfile.open("wb") as dst: while data := src.read(4096): dst.write(data) - - def write(self, tmpdir_path): + return outfile + + def write_files(self, tmpdir_path): self.write_notes(tmpdir_path) (tmpdir_path / "entry.md").write_text(self.format()) (tmpdir_path / "entry.json").write_text(self.dump()) (tmpdir_path / "quilt_summarize.json").write_text(self.QUILT_SUMMARIZE) - def make_package(self, tmpdir_path): - registry = f"s3://{self.DST_BUCKET}" + def push_package(self, tmpdir_path): pkg = quilt3.Package() try: - pkg = quilt3.Package.browse(self.pkg_name, registry=registry) + pkg = quilt3.Package.browse(self.pkg_name, registry=self.registry) except botocore_exceptions.ClientError as e: # XXX: quilt3 should raise some specific exception # when package doesn't exist. @@ -176,7 +184,8 @@ def make_package(self, tmpdir_path): pkg.set_dir(".", tmpdir_path, meta=self.entry) # This shouldn't hit 1 MB limit on metadata, # because max size of EventBridge is 256 KiB. - pkg.push(self.pkg_name, registry=registry) + return pkg.push(self.pkg_name, registry=self.registry) + def field_values(self): values = { @@ -186,25 +195,29 @@ def field_values(self): } return {f: values.get(k) for k, f in self.FLD.items()} - def update_benchling_notebook(self): + def update_benchling_notebook(self) -> bool: values = self.field_values() if values: self.client.update_entry(self.entry_id, values) logger.debug(f"Updated entry {self.entry_id} with package {self.pkg_name}") + return True else: logger.warning(f"Quilt schema fields not found for entry {self.entry_id!r}") + return False - -@logger.inject_lambda_context -def lambda_handler(event, context): - entry = BenchlingEntry(event["detail"]["entry"]) - +def main(entry_dict): + entry = BenchlingEntry(entry_dict) with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = pathlib.Path(tmpdir) - entry.write(tmpdir_path) - entry.make_package(tmpdir_path) + entry.write_files(tmpdir_path) + entry.push_package(tmpdir_path) entry.update_benchling_notebook() + return entry + +@logger.inject_lambda_context +def lambda_handler(event, context): + main(event["detail"]["entry"]) return { "statusCode": 200, diff --git a/tests/test_entry.py b/tests/test_entry.py index c0b9f5a..eaaf43a 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -1,19 +1,48 @@ import os import pytest +import quilt3 -from lambdas import BenchlingEntry +from lambdas import BenchlingEntry, main +from pathlib import Path +from tempfile import TemporaryDirectory + +LOCAL_ONLY = os.environ.get("LOCAL_ONLY", False) +SKIP_PARTIALS = os.environ.get("SKIP_PARTIALS", True) try: BENCHLING_ENTRY_ID=os.environ["BENCHLING_ENTRY_ID"] except KeyError: pytest.skip(allow_module_level=True) +ENTRY_DATA = { + "id": BENCHLING_ENTRY_ID, + "displayId": "test_entry", + "name": "test_entry", + "folderId": "test_folder", + "createdAt": "2021-01-01T00:00:00.000Z", + "modifiedAt": "2021-01-01T00:00:00.000Z", + "schema": { + "id": "test_schema", + "name": "test_schema", + }, + "fields": {}, + "customFields": {}, + "authors": [ + { + "id": "test_author", + "name": "test_author", + "handle": "test_author", + } + ], + "days": [], + "webURL": "https://example.com", +} + + @pytest.fixture def entry(): - entry_dict = { - "id": BENCHLING_ENTRY_ID, - } - return BenchlingEntry(entry_dict) + return BenchlingEntry(ENTRY_DATA) + def test_entry(entry): assert entry @@ -21,4 +50,57 @@ def test_entry(entry): assert entry.fields == {} assert entry.pkg_name assert BenchlingEntry.PKG_PREFIX in entry.pkg_name + assert "/" in entry.pkg_name + + +def test_format(entry): + fmt = entry.format() + assert fmt + assert "test_entry" in fmt + + +def test_dump(entry): + dmp = entry.dump() + assert dmp + assert "days" in dmp + + +@pytest.mark.skipif(SKIP_PARTIALS, reason="Only do end-to-end test") +def test_write(entry): + with TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + entry.write_files(tmpdir_path) + files = list(tmpdir_path.glob("*")) + fn = {f.name: f for f in files} + assert "entry.json" in fn + assert "notes.pdf" in fn + notes = fn["notes.pdf"] + assert isinstance(notes, Path) + assert notes.exists() + +@pytest.mark.skipif(SKIP_PARTIALS, reason="Only do end-to-end test") +def test_push(entry): + with TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + (tmpdir_path / "README.md").write_text("test_push") + rc = entry.push_package(tmpdir_path) + assert rc + + pkg = quilt3.Package.browse(entry.pkg_name, registry=entry.registry) + assert pkg + readme = pkg["README.md"] + assert readme + assert readme() == "test_push" + print(entry.pkg_name, entry.registry) + assert False + + +@pytest.mark.skipif(SKIP_PARTIALS, reason="Only do end-to-end test") +def test_update(entry): + rc = entry.update_benchling_notebook() + assert rc +@pytest.mark.skipif(SKIP_PARTIALS==False, reason="Only do partial tests") +def test_handler(): + entry = main(ENTRY_DATA) + assert isinstance(entry, BenchlingEntry) diff --git a/tests/test_lambda.py b/tests/test_lambda.py index ddaedb3..c439c23 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -1,10 +1,11 @@ from aws_lambda_powertools.utilities import parameters # type: ignore -from lambdas import BenchlingClient, BenchlingEntry +from lambdas import BenchlingClient, BenchlingEntry, main def test_import(): assert BenchlingClient assert BenchlingEntry + assert main def test_env(): @@ -23,3 +24,4 @@ def test_secret(): def test_client(): client = BenchlingClient.Default() assert client + From b4af8e665121efa8f69944de818fe4b6e0ca422f Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 16:02:09 -0700 Subject: [PATCH 28/33] autolint --- Install.md | 14 +++++++------- lambdas/main.py | 6 +++--- make.py | 14 +++++++------- tests/test_entry.py | 12 +++++++----- tests/test_lambda.py | 1 - upload.py | 2 ++ 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Install.md b/Install.md index 7fc4e9b..b27dcd3 100644 --- a/Install.md +++ b/Install.md @@ -124,7 +124,7 @@ since it was already created there. 1. Under `Parameters`: 1. Enter the name of the event bus created at step 1 as `BenchlingEventBusName`. 1. Enter the client ID from settings of app created at step 2 as `BenchlingClientId`. - 1. Enter the your Benchling tenant name (i.e. $BenchlingTenant in https://$BenchlingTenant.benchling.com) as `BenchlingTenant`. + 1. Enter the your Benchling tenant name (i.e. $BenchlingTenant in ) as `BenchlingTenant`. 1. Enter the name of the S3 bucket to use for storing packages as `DestinationBucket`. 1. Optional: change the `PackageNamePrefix` used when creating new packages (default: `benchling/`). 1. Specify the hostname of your Quilt Catalog as `QuiltWebHost` @@ -144,12 +144,12 @@ click on its Physical ID. In order for the lambda to update Benchling with the package information, the notebook must have a schema containing exactly the following fields: -| Name | Required | Multi-select | Definition | -| --------------------- | --------- | ------------- | ------------- | -| Quilt+ URI | | | Text | -| Quilt Revise URL | | | Text | -| Quilt Catalog URL | | | Text | -| Sentinel | | | Integer | +| Name | Required | Multi-select | Definition | +|-------------------|----------|--------------|------------| +| Quilt+ URI | | | Text | +| Quilt Revise URL | | | Text | +| Quilt Catalog URL | | | Text | +| Sentinel | | | Integer | You can either create a brand-new schema, or add these fields to an existing schema. Each new notebook will need to have this schema applied to it. diff --git a/lambdas/main.py b/lambdas/main.py index 3f2d6b2..74e849b 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -143,7 +143,6 @@ def name(self): SEP = "/" if self.PKG_PREFIX[-1] != "/" else "" return self.PKG_PREFIX + SEP + self.entry.get("displayId", self.entry_id) - def format(self): template = jinja2.Template(self.ENTRY_FMT) return template.render({"entry": self.entry}) @@ -164,7 +163,7 @@ def write_notes(self, tmpdir_path): while data := src.read(4096): dst.write(data) return outfile - + def write_files(self, tmpdir_path): self.write_notes(tmpdir_path) (tmpdir_path / "entry.md").write_text(self.format()) @@ -185,7 +184,6 @@ def push_package(self, tmpdir_path): # This shouldn't hit 1 MB limit on metadata, # because max size of EventBridge is 256 KiB. return pkg.push(self.pkg_name, registry=self.registry) - def field_values(self): values = { @@ -205,6 +203,7 @@ def update_benchling_notebook(self) -> bool: logger.warning(f"Quilt schema fields not found for entry {self.entry_id!r}") return False + def main(entry_dict): entry = BenchlingEntry(entry_dict) with tempfile.TemporaryDirectory() as tmpdir: @@ -215,6 +214,7 @@ def main(entry_dict): entry.update_benchling_notebook() return entry + @logger.inject_lambda_context def lambda_handler(event, context): main(event["detail"]["entry"]) diff --git a/make.py b/make.py index 08bd445..e10387b 100644 --- a/make.py +++ b/make.py @@ -76,8 +76,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: Type="String", AllowedPattern=r"^aws\.partner(/[\.\-_A-Za-z0-9]+){2,}$", Description=( - "Name of event bus where Benchling events are emitted, "+\ - "e.g aws.partner/benchling.com/tenant/app-name" + "Name of event bus where Benchling events are emitted, " + + "e.g aws.partner/benchling.com/tenant/app-name" ), ) benchling_tenant = troposphere.Parameter( @@ -85,8 +85,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: template=cft, Type="String", AllowedPattern=r"^[^/]+$", - Description="Benchling tenant name, i.e. $BenchlingTenant in "+\ - "https://$BenchlingTenant.benchling.com", + Description="Benchling tenant name, i.e. $BenchlingTenant in " + + "https://$BenchlingTenant.benchling.com", ) benchling_client_id = troposphere.Parameter( "BenchlingClientId", @@ -116,9 +116,9 @@ def make_template(*, metadata: dict) -> troposphere.Template: Default="benchling/", AllowedPattern=r".+/.*$", Description=( - "Prefix for package names i.e. package names will be"+ - " $PackageNamePrefix$ExperimentDisplayID,"+ - " must contain, but not start with '/'" + "Prefix for package names i.e. package names will be" + + " $PackageNamePrefix$ExperimentDisplayID," + + " must contain, but not start with '/'" ), ) diff --git a/tests/test_entry.py b/tests/test_entry.py index eaaf43a..f168dd7 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -1,16 +1,16 @@ import os +from pathlib import Path +from tempfile import TemporaryDirectory + import pytest import quilt3 - from lambdas import BenchlingEntry, main -from pathlib import Path -from tempfile import TemporaryDirectory LOCAL_ONLY = os.environ.get("LOCAL_ONLY", False) SKIP_PARTIALS = os.environ.get("SKIP_PARTIALS", True) try: - BENCHLING_ENTRY_ID=os.environ["BENCHLING_ENTRY_ID"] + BENCHLING_ENTRY_ID = os.environ["BENCHLING_ENTRY_ID"] except KeyError: pytest.skip(allow_module_level=True) @@ -78,6 +78,7 @@ def test_write(entry): assert isinstance(notes, Path) assert notes.exists() + @pytest.mark.skipif(SKIP_PARTIALS, reason="Only do end-to-end test") def test_push(entry): with TemporaryDirectory() as tmpdir: @@ -100,7 +101,8 @@ def test_update(entry): rc = entry.update_benchling_notebook() assert rc -@pytest.mark.skipif(SKIP_PARTIALS==False, reason="Only do partial tests") + +@pytest.mark.skipif(SKIP_PARTIALS is False, reason="Only do partial tests") def test_handler(): entry = main(ENTRY_DATA) assert isinstance(entry, BenchlingEntry) diff --git a/tests/test_lambda.py b/tests/test_lambda.py index c439c23..09f1293 100644 --- a/tests/test_lambda.py +++ b/tests/test_lambda.py @@ -24,4 +24,3 @@ def test_secret(): def test_client(): client = BenchlingClient.Default() assert client - diff --git a/upload.py b/upload.py index 3a655e2..a719535 100644 --- a/upload.py +++ b/upload.py @@ -5,11 +5,13 @@ TEMPLATE = "benchling_packager.yaml" TARGET = f"build/{TEMPLATE}" + def upload(): pkg = quilt3.Package() pkg.set("README.md", "Install.md") pkg.set(TEMPLATE, TARGET) pkg.push(PKG_NAME, registry=REGISTRY) + if __name__ == "__main__": upload() From c848c9bf6fb525b9497f15260ee655c07ecd473c Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 16:06:56 -0700 Subject: [PATCH 29/33] test-partials --- Makefile | 3 +++ tests/test_entry.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9435433..70990e7 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ requirements.txt: venv/bin/pip-compile requirements.in test: venv install-dev . $(ACTIVATE) && python3 -m pytest --cov --cov-report xml:coverage.xml +test-partials: venv install-dev + SKIP_PARTIALS=False . $(ACTIVATE) && python3 -m pytest + coverage: venv install-dev printenv BENCHLING_ENTRY_ID . $(ACTIVATE) && python3 -m pytest --cov --cov-report html:coverage.html diff --git a/tests/test_entry.py b/tests/test_entry.py index f168dd7..b5190ec 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -92,8 +92,6 @@ def test_push(entry): readme = pkg["README.md"] assert readme assert readme() == "test_push" - print(entry.pkg_name, entry.registry) - assert False @pytest.mark.skipif(SKIP_PARTIALS, reason="Only do end-to-end test") From 51c809f66c0f97f2d9a0a2398f22d1249e63d5ea Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Wed, 16 Aug 2023 16:17:53 -0700 Subject: [PATCH 30/33] Testing instructions --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 1f35f7f..6678f93 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,21 @@ python3 make.py > build/benchling_packager.yaml To install and configure the template, see [Install.md](Install.md). Note: this is the file that's distributed as `README.md` in the package. + +## Testing for Developers + +If you want to modify the actual lambda function, you can run automated tests via: + +```shell +make test +``` + +In order to run these tests, you'll need to set the following environment variables +(usually in the `.env` file, which is auto-included by the Makefile): + +- `BENCHLING_TENANT`: the part before ".benchling.com" in your Benchling URL (e.g. "mycompany" for "mycompany.benchling.com") +- `BENCHLING_CLIENT_ID`: the client ID for the Benchling API` +- `BENCHLING_CLIENT_SECRET_ARN`: the ARN of the AWS Secrets Manager secret containing the client secret for the Benchling API +- `DST_BUCKET`: the name of the S3 bucket (no prefix) where the generated packages should be stored +- `PKG_PREFIX`: the prefix to use for the generated packages, with a trailing "/" (e.g. "benchling/" to store packages in the "benchling" directory) +- `QUILT_CATALOG_DOMAIN`: the domain name of your Quilt catalog (if any) where the generated packages can be viewed From 7b5cd291fbb3f4a13acb9ed47b034e5742aab968 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Thu, 17 Aug 2023 15:56:49 -0700 Subject: [PATCH 31/33] Update make.py doc Co-authored-by: Sergey Fedoseev --- make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/make.py b/make.py index e10387b..3bde703 100644 --- a/make.py +++ b/make.py @@ -117,8 +117,8 @@ def make_template(*, metadata: dict) -> troposphere.Template: AllowedPattern=r".+/.*$", Description=( "Prefix for package names i.e. package names will be" - + " $PackageNamePrefix$ExperimentDisplayID," - + " must contain, but not start with '/'" + " $PackageNamePrefix$ExperimentDisplayID," + " must contain, but not start with '/'" ), ) From 57d4edc282544bf7233f02fdd86ab6f5423e57e4 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Thu, 17 Aug 2023 15:57:02 -0700 Subject: [PATCH 32/33] Update tests/test_entry.py iterator Co-authored-by: Sergey Fedoseev --- tests/test_entry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_entry.py b/tests/test_entry.py index b5190ec..12f631d 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -70,8 +70,7 @@ def test_write(entry): with TemporaryDirectory() as tmpdir: tmpdir_path = Path(tmpdir) entry.write_files(tmpdir_path) - files = list(tmpdir_path.glob("*")) - fn = {f.name: f for f in files} + fn = {f.name: f for f in tmpdir_path.glob("*")} assert "entry.json" in fn assert "notes.pdf" in fn notes = fn["notes.pdf"] From 62bfe2a2a03ac17d77b4b1e895fed4282c9ab193 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Thu, 17 Aug 2023 16:08:41 -0700 Subject: [PATCH 33/33] fix prefix --- lambdas/main.py | 6 ++++-- tests/test_entry.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lambdas/main.py b/lambdas/main.py index 74e849b..6affa27 100644 --- a/lambdas/main.py +++ b/lambdas/main.py @@ -140,8 +140,10 @@ def __init__(self, entry): self.registry = f"s3://{self.DST_BUCKET}" def name(self): - SEP = "/" if self.PKG_PREFIX[-1] != "/" else "" - return self.PKG_PREFIX + SEP + self.entry.get("displayId", self.entry_id) + SEP = "/" + if SEP not in self.PKG_PREFIX: + self.PKG_PREFIX += SEP + return self.PKG_PREFIX + self.entry.get("displayId", self.entry_id) def format(self): template = jinja2.Template(self.ENTRY_FMT) diff --git a/tests/test_entry.py b/tests/test_entry.py index 12f631d..2997379 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -6,7 +6,6 @@ import quilt3 from lambdas import BenchlingEntry, main -LOCAL_ONLY = os.environ.get("LOCAL_ONLY", False) SKIP_PARTIALS = os.environ.get("SKIP_PARTIALS", True) try: