diff --git a/poetry.lock b/poetry.lock index d01e5514..b8092273 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "cached-property" version = "1.5.2" description = "A decorator for caching properties in classes." +category = "dev" optional = false python-versions = "*" files = [ @@ -15,6 +16,7 @@ files = [ name = "chispa" version = "0.9.4" description = "Pyspark test helper library" +category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -26,6 +28,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -41,6 +44,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -52,6 +56,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -66,6 +71,7 @@ test = ["pytest (>=6)"] name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." +category = "dev" optional = false python-versions = "*" files = [ @@ -83,6 +89,7 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "griffe" version = "0.30.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -98,6 +105,7 @@ colorama = ">=0.4" name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -118,6 +126,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -129,6 +138,7 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -146,6 +156,7 @@ i18n = ["Babel (>=2.7)"] name = "markdown" version = "3.4.4" description = "Python implementation of John Gruber's Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -164,6 +175,7 @@ testing = ["coverage", "pyyaml"] name = "markdown-include" version = "0.8.1" description = "A Python-Markdown extension which provides an 'include' function" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -181,6 +193,7 @@ tests = ["pytest"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -250,6 +263,7 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -261,6 +275,7 @@ files = [ name = "mkdocs" version = "1.5.3" description = "Project documentation with Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -293,6 +308,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "0.4.1" description = "Automatically link across pages in MkDocs." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -308,6 +324,7 @@ mkdocs = ">=1.1" name = "mkdocs-gen-files" version = "0.4.0" description = "MkDocs plugin to programmatically generate documentation pages during the build" +category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -322,6 +339,7 @@ mkdocs = ">=1.0.3,<2.0.0" name = "mkdocs-literate-nav" version = "0.6.1" description = "MkDocs plugin to specify the navigation in Markdown instead of YAML" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -336,6 +354,7 @@ mkdocs = ">=1.0.3" name = "mkdocs-section-index" version = "0.3.8" description = "MkDocs plugin to allow clickable sections that lead to an index page" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -350,6 +369,7 @@ mkdocs = ">=1.2" name = "mkdocstrings" version = "0.22.0" description = "Automatic documentation from sources, for MkDocs." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -376,6 +396,7 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] name = "mkdocstrings-python" version = "0.8.3" description = "A Python handler for mkdocstrings." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -391,6 +412,7 @@ mkdocstrings = ">=0.19" name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -402,6 +424,7 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -411,13 +434,14 @@ files = [ [[package]] name = "platformdirs" -version = "3.11.0" +version = "4.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, - {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, + {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, + {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, ] [package.dependencies] @@ -431,6 +455,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -449,6 +474,7 @@ testing = ["pytest", "pytest-benchmark"] name = "py4j" version = "0.10.9.7" description = "Enables Python programs to dynamically access arbitrary Java objects" +category = "dev" optional = false python-versions = "*" files = [ @@ -460,6 +486,7 @@ files = [ name = "pymdown-extensions" version = "10.2.1" description = "Extension pack for Python Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -478,6 +505,7 @@ extra = ["pygments (>=2.12)"] name = "pyspark" version = "3.4.1" description = "Apache Spark Python API" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -496,13 +524,14 @@ sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] [[package]] name = "pytest" -version = "7.4.2" +version = "7.4.3" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, - {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] [package.dependencies] @@ -521,6 +550,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-describe" version = "2.1.0" description = "Describe-style plugin for pytest" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -535,6 +565,7 @@ pytest = ">=4.6,<8" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -549,6 +580,7 @@ six = ">=1.5" name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -608,6 +640,7 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -622,6 +655,7 @@ pyyaml = "*" name = "ruff" version = "0.0.291" description = "An extremely fast Python linter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -646,19 +680,21 @@ files = [ [[package]] name = "semver" -version = "3.0.1" +version = "3.0.2" description = "Python helper for Semantic Versioning (https://semver.org)" +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, - {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, ] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -670,6 +706,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -679,19 +716,21 @@ files = [ [[package]] name = "typing-extensions" -version = "4.8.0" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -731,6 +770,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -745,4 +785,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "5559a0b7175528b497b00d5e68b5841721e49230e371ad307c74c64e82a55d9a" +content-hash = "41791e6258914ac7f51b4ce6759ae8a03c8345313ea50c0c650293797e13c571" diff --git a/pyproject.toml b/pyproject.toml index 076f7aca..58b03671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ pytest-describe = "^2" [tool.poetry.group.linting.dependencies] ruff = "^0.0.291" -typing-extensions = {version = "^4.8.0", python = ">=3.8,<4.0"} +typing-extensions = {version = "^4", python = ">=3.6,<4.0"} [tool.poetry.group.docs.dependencies] mkdocstrings-python = "^0.8.3" @@ -55,7 +55,7 @@ mkdocs-gen-files = "^0.4.0" mkdocs-literate-nav = "^0.6.0" mkdocs-section-index = "^0.3.5" markdown-include = "^0.8.1" -mkdocs = "^1.4.2" +mkdocs = "^1" ########################################################################### # LINTING CONFIGURATION diff --git a/quinn/transformations.py b/quinn/transformations.py index c0214e51..a2eac362 100644 --- a/quinn/transformations.py +++ b/quinn/transformations.py @@ -241,7 +241,7 @@ def fix_nullability(field: StructField, result_dict: dict) -> None: if spark is None: spark = SparkSession.builder.getOrCreate() - return spark.sparkSession.createDataFrame(output.rdd, output.schema) + return spark.createDataFrame(output.rdd, output.schema) def flatten_struct(df: DataFrame, col_name: str, separator: str = ":") -> DataFrame: