Use dynamic dependencies

Signed-off-by: Constantin M Adam <[email protected]>

transforms/universal/fdedup/python/Dockerfile

@@ -19,7 +19,7 @@ RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml
 COPY --chown=dpk:root README.md README.md
-#COPY --chown=dpk:root requirements.txt requirements.txt
+COPY --chown=dpk:root requirements.txt requirements.txt
 
 RUN pip install --no-cache-dir -e .
 

transforms/universal/fdedup/python/pyproject.toml

@@ -9,23 +9,13 @@ authors = [
     { name = "Nelson Bore", email = "[email protected]" },
     { name = "Constantin Adam", email = "[email protected]" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev2",
-    "pyarrow==16.1.0",
-    "pyyaml>=6.0.2",
-    "boto3>=1.34.69",
-    "kubernetes>=30.1.0",
-    "polars==1.9.0",
-    "disjoint-set>=0.8.0",
-    "scipy>=1.14.1, <2.0.0",
-    "numpy<1.29.0",
-    "sentencepiece>=0.2.0",
-    "mmh3>=4.1.0",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
 
 [project.optional-dependencies]
 dev = [

transforms/universal/fdedup/python/requirements.txt

@@ -0,0 +1,10 @@
+data-prep-toolkit==0.2.2.dev2
+pyyaml>=6.0.2
+boto3>=1.34.69
+kubernetes>=30.1.0
+polars==1.9.0
+disjoint-set>=0.8.0
+scipy>=1.14.1, <2.0.0
+numpy<1.29.0
+sentencepiece>=0.2.0
+mmh3>=4.1.0

transforms/universal/fdedup/ray/Dockerfile

@@ -20,6 +20,7 @@ RUN cd python-transform && pip install --no-cache-dir -e .
 COPY --chown=ray:users src/ src/
 COPY --chown=ray:users pyproject.toml pyproject.toml
 COPY --chown=ray:users README.md README.md
+COPY --chown=ray:users requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy source files needed by test-image

transforms/universal/fdedup/ray/pyproject.toml

@@ -9,18 +9,13 @@ authors = [
     { name = "Nelson Bore", email = "[email protected]" },
     { name = "Constantin Adam", email = "[email protected]" },
 ]
-dependencies = [
-    "data-prep-toolkit[ray]==0.2.2.dev2",
-    "dpk_fdedup_transform_python==0.2.2.dev2",
-    "mmh3>=4.1.0",
-    "xxhash==3.4.1",
-    "tqdm==4.66.3",
-    "scipy>=1.12.0, <2.0.0"
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
 
 [project.optional-dependencies]
 dev = [

transforms/universal/fdedup/ray/requirements.txt

@@ -0,0 +1,6 @@
+data-prep-toolkit[ray]==0.2.2.dev2
+dpk_fdedup_transform_python==0.2.2.dev2
+mmh3>=4.1.0
+xxhash==3.4.1
+tqdm==4.66.3
+scipy>=1.12.0, <2.0.0

transforms/universal/fdedup/spark/pyproject.toml

@@ -9,10 +9,13 @@ authors = [
     { name = "Nelson Bore", email = "[email protected]" },
     { name = "Constantin Adam", email = "[email protected]" },
 ]
-dependencies = [
-    "dpk_fdedup_transform_python==0.2.2.dev2",
-    "data-prep-toolkit-spark==0.2.2.dev2",
-]
+dynamic = ["dependencies"]
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
 
 [project.optional-dependencies]
 dev = [

transforms/universal/fdedup/spark/requirements.txt

@@ -1,4 +1,5 @@
-pyarrow==16.1.0
+dpk_fdedup_transform_python==0.2.2.dev2
+data-prep-toolkit[spark]==0.2.2.dev2
 pyyaml>=6.0.2
 boto3>=1.34.69
 kubernetes>=30.1.0