From 0a12830e9d0d2972095c87ca0380f085349b3962 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 9 Aug 2023 09:35:02 -0400 Subject: [PATCH] repair date validation --- CHANGELOG.rst | 6 ++ poetry.lock | 128 +++++++++++++++++++++++++++- pyproject.toml | 4 +- snovault/schema_utils.py | 7 +- snovault/tests/test_schema_utils.py | 58 ++++++++++++- 5 files changed, 191 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c0c138622..09fba9884 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,12 @@ snovault Change Log ---------- +10.0.1 +====== + +* Extend ``FormatChecker`` to ensure date and date-time validation + + 10.0.0 ====== diff --git a/poetry.lock b/poetry.lock index c4f2f7dc2..cb032d95a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,20 @@ # This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +[[package]] +name = "arrow" +version = "1.2.3" +description = "Better dates & times for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "arrow-1.2.3-py3-none-any.whl", hash = "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2"}, + {file = "arrow-1.2.3.tar.gz", hash = "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1"}, +] + +[package.dependencies] +python-dateutil = ">=2.7.0" + [[package]] name = "async-timeout" version = "4.0.2" @@ -977,6 +992,18 @@ files = [ {file = "flaky-3.7.0.tar.gz", hash = "sha256:3ad100780721a1911f57a165809b7ea265a7863305acb66708220820caf8aa0d"}, ] +[[package]] +name = "fqdn" +version = "1.5.1" +description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" +files = [ + {file = "fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"}, + {file = "fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f"}, +] + [[package]] name = "future" version = "0.18.3" @@ -1200,6 +1227,21 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "isoduration" +version = "20.11.0" +description = "Operations with ISO 8601 durations" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"}, + {file = "isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9"}, +] + +[package.dependencies] +arrow = ">=0.15.0" + [[package]] name = "jinja2" version = "3.1.2" @@ -1230,25 +1272,45 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jsonpointer" +version = "2.4" +description = "Identify specific nodes in a JSON document (RFC 6901)" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, +] + [[package]] name = "jsonschema" -version = "4.18.4" +version = "4.19.0" description = "An implementation of JSON Schema validation for Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema-4.18.4-py3-none-any.whl", hash = "sha256:971be834317c22daaa9132340a51c01b50910724082c2c1a2ac87eeec153a3fe"}, - {file = "jsonschema-4.18.4.tar.gz", hash = "sha256:fb3642735399fa958c0d2aad7057901554596c63349f4f6b283c493cf692a25d"}, + {file = "jsonschema-4.19.0-py3-none-any.whl", hash = "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb"}, + {file = "jsonschema-4.19.0.tar.gz", hash = "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"}, ] [package.dependencies] attrs = ">=22.2.0" +fqdn = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +idna = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +isoduration = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format-nongpl\""} jsonschema-specifications = ">=2023.03.6" pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} referencing = ">=0.28.4" +rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} rpds-py = ">=0.7.1" +uri-template = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format-nongpl\""} [package.extras] format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] @@ -2302,6 +2364,21 @@ urllib3 = ">=1.25.10" [package.extras] tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +description = "A pure python RFC3339 validator" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, + {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "rfc3986" version = "1.5.0" @@ -2317,6 +2394,18 @@ files = [ [package.extras] idna2008 = ["idna"] +[[package]] +name = "rfc3986-validator" +version = "0.1.1" +description = "Pure python rfc3986 validator" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9"}, + {file = "rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"}, +] + [[package]] name = "rpds-py" version = "0.9.2" @@ -2868,6 +2957,21 @@ files = [ {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] +[[package]] +name = "uri-template" +version = "1.3.0" +description = "RFC 6570 URI Template Processor" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7"}, + {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"}, +] + +[package.extras] +dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake8-commas", "flake8-comprehensions", "flake8-continuation", "flake8-datetimez", "flake8-docstrings", "flake8-import-order", "flake8-literal", "flake8-modern-annotations", "flake8-noqa", "flake8-pyproject", "flake8-requirements", "flake8-typechecking-import", "flake8-use-fstring", "mypy", "pep8-naming", "types-PyYAML"] + [[package]] name = "urllib3" version = "1.26.16" @@ -2917,6 +3021,22 @@ files = [ docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.9)"] testing = ["coverage (>=5.0)", "pytest", "pytest-cover"] +[[package]] +name = "webcolors" +version = "1.13" +description = "A library for working with the color formats defined by HTML and CSS." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "webcolors-1.13-py3-none-any.whl", hash = "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf"}, + {file = "webcolors-1.13.tar.gz", hash = "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"}, +] + +[package.extras] +docs = ["furo", "sphinx", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinxext-opengraph"] +tests = ["pytest", "pytest-cov"] + [[package]] name = "webencodings" version = "0.5.1" @@ -3169,4 +3289,4 @@ test = ["zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.10" -content-hash = "e9ce5f87370edc1b363189bfedbe20d776d6ca8855ce148f181af20382a85c95" +content-hash = "65af2719f3cb55317c8fd8869d431277c1e5c7a49782e01c5e22d8088e57fbea" diff --git a/pyproject.toml b/pyproject.toml index 0211318d0..54bd096c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicsnovault" -version = "10.0.0" +version = "10.0.1" description = "Storage support for 4DN Data Portals." authors = ["4DN-DCIC Team "] license = "MIT" @@ -82,7 +82,7 @@ xlrd = "^1.0.0" "zope.deprecation" = "^4.4.0" "zope.interface" = ">=4.7.2,<6" "zope.sqlalchemy" = "1.6" -jsonschema = "^4.18.4" +jsonschema = {extras = ["format-nongpl"], version = "^4.19.0"} [tool.poetry.dev-dependencies] botocore-stubs = ">=1.29.119" # no particular version required, but this speeds up search diff --git a/snovault/schema_utils.py b/snovault/schema_utils.py index 426bb79a2..4a327eaf0 100644 --- a/snovault/schema_utils.py +++ b/snovault/schema_utils.py @@ -7,7 +7,7 @@ from datetime import datetime from dcicutils.misc_utils import ignored from snovault.schema_validation import SerializingSchemaValidator -from jsonschema import FormatChecker +from jsonschema import Draft202012Validator from jsonschema import RefResolver from jsonschema.exceptions import ValidationError import os @@ -355,9 +355,6 @@ class SchemaValidator(SerializingSchemaValidator): SERVER_DEFAULTS = SERVER_DEFAULTS -format_checker = FormatChecker() - - def load_schema(filename): filename = favor_app_specific_schema(filename) if isinstance(filename, dict): @@ -403,7 +400,7 @@ def validate(schema, data, current=None, validate_current=False): dict validated contents, list of errors """ resolver = NoRemoteResolver.from_schema(schema) - sv = SchemaValidator(schema, resolver=resolver, format_checker=format_checker) + sv = SchemaValidator(schema, resolver=resolver, format_checker=Draft202012Validator.FORMAT_CHECKER) validated, errors = sv.serialize(data) # validate against current contents if validate_current is set if current and validate_current: diff --git a/snovault/tests/test_schema_utils.py b/snovault/tests/test_schema_utils.py index 75a7c8cc5..ab09d7219 100644 --- a/snovault/tests/test_schema_utils.py +++ b/snovault/tests/test_schema_utils.py @@ -1,6 +1,7 @@ import pytest from snovault.schema_utils import ( - _update_resolved_data, _handle_list_or_string_value, resolve_merge_refs + _update_resolved_data, _handle_list_or_string_value, resolve_merge_refs, + validate ) @@ -364,3 +365,58 @@ def test_schema_utils_resolve_merge_ref_in_embedded_schema(testapp): embedded = testapp.get(f'/{atid}?frame=embedded', status=200).json assert embedded['link']['quality'] == 5 assert embedded['link']['linked_targets'][0]['test_description'] == 'test' + + +@pytest.mark.parametrize('invalid_date', [ + 'not a date', + 'also-not-a-date', + '10-5-2022', + '10-05-almost', + '10-05-2023f' +]) +def test_schema_utils_validates_dates(testapp, invalid_date): + """ Tests that our validator will validate dates """ + schema = { + "type": "object", + "properties": { + "date_property": { + "type": "string", + "format": "date" + } + } + } + _, errors = validate(schema, { + 'date_property': invalid_date + }) + date_error = str(errors[0]) + assert f"'{invalid_date}' is not a 'date'" in date_error + + +@pytest.mark.parametrize('invalid_date_time', [ + 'not a date', + 'also-not-a-date', + '10-5-2022', + '10-05-almost', + '10-05-2023f', + '1424-45-93T15:32:12.9023368Z', + '20015-10-23T15:32:12.9023368Z', + '2001-130-23T15:32:12.9023368Z', + '2001-10-233T15:32:12.9023368Z', + '2001-10-23T153:32:12.9023368Z' +]) +def test_schema_utils_validates_date_times(testapp, invalid_date_time): + """ Tests that our validator will validate date-time """ + schema = { + "type": "object", + "properties": { + "date_time_property": { + "type": "string", + "format": "date-time" + } + } + } + _, errors = validate(schema, { + 'date_time_property': invalid_date_time + }) + date_error = str(errors[0]) + assert f"'{invalid_date_time}' is not a 'date-time'" in date_error