From f2e685e584073ab8067cba741917c2cf414cced1 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Fri, 18 Aug 2023 10:53:07 +0100 Subject: [PATCH] align with framework setup --- kedro-datasets/setup.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py index cbfed429b..152fceaf5 100644 --- a/kedro-datasets/setup.py +++ b/kedro-datasets/setup.py @@ -3,8 +3,8 @@ from setuptools import setup # at least 1.3 to be able to use XMLDataSet and pandas integration with fsspec -PANDAS = "pandas>=1.3, <3.0" -SPARK = "pyspark>=2.2, <4.0" +PANDAS = "pandas~=1.3" +SPARK = "pyspark>=2.2, <3.4" HDFS = "hdfs>=2.5.8, <3.0" S3FS = "s3fs>=0.3.0, <0.5" POLARS = "polars~=0.17.0" @@ -76,7 +76,6 @@ def _collect_requirements(requires): "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'", ] } -video_require = {"video.VideoDataSet": ["opencv-python~=4.5.5.64"]} yaml_require = {"yaml.YAMLDataSet": [PANDAS, "PyYAML>=4.2, <7.0"]} extras_require = { @@ -139,21 +138,22 @@ def _collect_requirements(requires): "Jinja2<3.1.0", ] extras_require["test"] = [ - "adlfs>=2021.7.1, <=2022.2", + "adlfs>=2021.7.1, <=2022.2; python_version == '3.7'", + "adlfs~=2023.1; python_version >= '3.8'", "bandit>=1.6.2, <2.0", "behave==1.2.6", "biopython~=1.73", "blacken-docs==1.9.2", "black~=22.0", - "compress-pickle[lz4]~=1.2.0", + "compress-pickle[lz4]~=2.1.0", "coverage[toml]", - "dask[complete]", - "delta-spark~=1.2.1", - # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070 - "deltalake>=0.10.0", + "dask[complete]~=2021.10", # pinned by Snyk to avoid a vulnerability + "delta-spark>=1.2.1; python_version >= '3.11'", # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070 + "delta-spark~=1.2.1; python_version < '3.11'", "dill~=0.3.1", "filelock>=3.4.0, <4.0", - "gcsfs>=2021.4, <=2022.1", + "gcsfs>=2021.4, <=2023.1; python_version == '3.7'", + "gcsfs>=2023.1, <2023.3; python_version >= '3.8'", "geopandas>=0.6.0, <1.0", "hdfs>=2.5.8, <3.0", "holoviews~=1.13.0", @@ -174,7 +174,7 @@ def _collect_requirements(requires): "openpyxl>=3.0.3, <4.0", "pandas-gbq>=0.12.0, <0.18.0; python_version < '3.11'", "pandas-gbq>=0.18.0; python_version >= '3.11'", - "pandas>=1.3, <2", # 1.3 for read_xml/to_xml, <2 for compatibility with Spark < 3.4 + "pandas~=1.3 # 1.3 for read_xml/to_xml", "Pillow~=9.0", "plotly>=4.8.0, <6.0", "polars~=0.15.13", @@ -194,14 +194,15 @@ def _collect_requirements(requires): "requests-mock~=1.6", "requests~=2.20", "s3fs>=0.3.0, <0.5", # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem. - "scikit-learn~=1.0.2, <2; python_version < '3.11'", - "scikit-learn>=1.1; python_version >= '3.11'", - "scipy~=1.7.3", + "scikit-learn>=1.0.2,<2", + "scipy>=1.7.3", "semver", - "snowflake-snowpark-python~=1.0.0; python_version == '3.8'", - "SQLAlchemy>=1.4, <3.0", - # The `Inspector.has_table()` method replaces the `Engine.has_table()` method in version 1.4. - "tables~=3.6", + "SQLAlchemy~=1.2", + "tables~=3.6.0; platform_system == 'Windows' and python_version<'3.8'", + "tables~=3.8.0; platform_system == 'Windows' and python_version>='3.8'", # Import issues with python 3.8 with pytables pinning to 3.8.0 fixes this https://github.com/PyTables/PyTables/issues/933#issuecomment-1555917593 + "tables~=3.6; platform_system != 'Windows'", + "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'", + # https://developer.apple.com/metal/tensorflow-plugin/ "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'", "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'", "triad>=0.6.7, <1.0",