Merge pull request #253 from roytman/sharedLibs

Combine the common KFP support code in a shared library
IBM · Jun 13, 2024 · 87e37af · 87e37af
2 parents d468da0 + 1b77e51
commit 87e37af
Show file tree

Hide file tree

Showing 56 changed files with 238 additions and 976 deletions.
diff --git a/.make.versions b/.make.versions
@@ -14,6 +14,7 @@ RELEASE_VERSION_SUFFIX=.dev6
 DPK_LIB_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
 DPK_LIB_KFP_VERSION=0.2.0$(RELEASE_VERSION_SUFFIX)
 DPK_LIB_KFP_VERSION_v2=0.2.0$(RELEASE_VERSION_SUFFIX)
+DPK_LIB_KFP_SHARED=0.2.0$(RELEASE_VERSION_SUFFIX)
 
 # Begin transform versions/tags
 BLOCKLIST_VERSION=0.4.2$(RELEASE_VERSION_SUFFIX)

diff --git a/kfp/kfp_ray_components/Dockerfile b/kfp/kfp_ray_components/Dockerfile
@@ -11,8 +11,8 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 COPY --chown=ray:users data-processing-lib-ray/ data-processing-lib-ray/
 RUN cd data-processing-lib-ray    && pip install --no-cache-dir -e .
 
-COPY --chown=ray:users python_apiserver_client python_apiserver_client/
-RUN cd python_apiserver_client && pip install --no-cache-dir -e .
+COPY --chown=ray:users shared_workflow_support_lib shared_workflow_support_lib/
+RUN cd shared_workflow_support_lib && pip install --no-cache-dir -e .
 
 COPY --chown=ray:users workflow_support_lib workflow_support_lib/
 RUN cd workflow_support_lib && pip install --no-cache-dir -e .

diff --git a/kfp/kfp_ray_components/Makefile b/kfp/kfp_ray_components/Makefile
@@ -28,12 +28,12 @@ DOCKER_IMG=$(DOCKER_LOCAL_IMAGE)
 .lib-src-image:: 
 	$(MAKE) .defaults.copy-lib LIB_PATH=$(DPK_RAY_LIB_DIR) LIB_NAME=data-processing-lib-ray
 	$(MAKE) .defaults.copy-lib LIB_PATH=$(DPK_PYTHON_LIB_DIR) LIB_NAME=data-processing-lib-python
-	$(MAKE) .defaults.copy-lib LIB_PATH=$(REPOROOT)/kfp/kfp_support_lib/python_apiserver_client LIB_NAME=python_apiserver_client
+	$(MAKE) .defaults.copy-lib LIB_PATH=$(REPOROOT)/kfp/kfp_support_lib/shared_workflow_support LIB_NAME=shared_workflow_support_lib
 	$(MAKE) .defaults.copy-lib LIB_PATH=$(REPOROOT)/kfp/kfp_support_lib/$(WORKFLOW_SUPPORT_LIB) LIB_NAME=workflow_support_lib
 	$(MAKE) .defaults.image
 	rm -rf data-processing-lib-ray
 	rm -rf data-processing-lib-python
-	rm -rf python_apiserver_client
+	rm -rf shared_workflow_support_lib
 	rm -rf workflow_support_lib
 
 .PHONY: image

diff --git a/kfp/kfp_ray_components/src/create_ray_cluster.py b/kfp/kfp_ray_components/src/create_ray_cluster.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 ################################################################################
 import sys
-from workflow_support.runtime_utils import KFPUtils,  RayRemoteJobs
+from runtime_utils import KFPUtils,  RayRemoteJobs
 
 
 def start_ray_cluster(

diff --git a/kfp/kfp_ray_components/src/delete_ray_cluster.py b/kfp/kfp_ray_components/src/delete_ray_cluster.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 ################################################################################
 import sys
-from workflow_support.runtime_utils import KFPUtils, RayRemoteJobs
+from runtime_utils import KFPUtils, RayRemoteJobs
 
 # Cleans and shutdowns the Ray cluster
 def cleanup_ray_cluster(

diff --git a/kfp/kfp_ray_components/src/execute_ray_job.py b/kfp/kfp_ray_components/src/execute_ray_job.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
-from workflow_support.runtime_utils import KFPUtils, execute_ray_jobs
+from runtime_utils import KFPUtils, execute_ray_jobs
 
 if __name__ == "__main__":
     import argparse

diff --git a/kfp/kfp_ray_components/src/execute_ray_job_multi_s3.py b/kfp/kfp_ray_components/src/execute_ray_job_multi_s3.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 ################################################################################
 
-from workflow_support.runtime_utils import KFPUtils, execute_ray_jobs
+from runtime_utils import KFPUtils, execute_ray_jobs
 
 if __name__ == "__main__":
     import argparse

diff --git a/kfp/kfp_ray_components/src/subworkflow.py b/kfp/kfp_ray_components/src/subworkflow.py
@@ -1,6 +1,6 @@
 import sys
 
-from workflow_support.runtime_utils import KFPUtils
+from runtime_utils import KFPUtils
 from workflow_support.pipeline_utils import PipelinesUtils
 
 

diff --git a/kfp/kfp_support_lib/README.md b/kfp/kfp_support_lib/README.md
@@ -3,7 +3,7 @@
 This provides support for implementing KFP pipelines automating transform's execution.
 It comprises 3 main modules
 
-* [api server client](python_apiserver_client/README.md) 
+* [shared_workflow_support](shared_workflow_support/README.md) 
 * [kfp_v1_workflow_support](kfp_v1_workflow_support//README.md)
 * [kfp_v2_workflow_support](kfp_v2_workflow_support//README.md)
 

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/Makefile b/kfp/kfp_support_lib/kfp_v1_workflow_support/Makefile
@@ -27,7 +27,7 @@ clean::
 set-versions: .check-env
 	@# Help: Copy the Makefile distribution version into the pyproject.toml
 	sed -i.back 's/^version[ ]*=.*/version = "'${DPK_LIB_KFP_VERSION}'"/' pyproject.toml
-	sed -i.back 's/data_prep_toolkit_ray==[0-9].*/data_prep_toolkit_ray==${DPK_LIB_VERSION}",/' pyproject.toml
+	sed -i.back 's/data_prep_toolkit==[0-9].*/data_prep_toolkit==${DPK_LIB_VERSION}",/' pyproject.toml
 	sed -i.back 's/kfp==[0-9].*/kfp==${KFP_v1}",/' pyproject.toml
 	sed -i.back 's/ray=[0-9].*/ray==${RAY}",/' pyproject.toml
 
@@ -60,12 +60,11 @@ else
 	$(PYTHON) -m venv venv
 	. ${VENV_ACTIVATE};     \
 	cd ../../../data-processing-lib/python && make set-versions && cd -; \
+	pip install --upgrade pip; \
 	pip install -e ../../../data-processing-lib/python; \
-	cd ../../../data-prepossesing-lib/ray && make set-versions && cd -; \
-	pip install -e ../../../data-processing-lib/ray; \
-	cd ../python_apiserver_client && make set-versions && cd -; \
-	pip install -e ../python_apiserver_client;      \
-	pip install -e .;				\
+	cd ../shared_workflow_support && make set-versions && cd -; \
+	pip install -e ../shared_workflow_support;	\
+	pip install -e .;	\
 	pip install pytest pytest-cov
 endif
 
@@ -76,7 +75,6 @@ ifeq ($(KFPv2), 1)
 else
 	@# Help: Use the already-built virtual environment to run pytest on the test directory.
 ifeq ($(DEPLOY_KUBEFLOW),1)
-	. ${VENV_ACTIVATE}; export PYTHONPATH=../src; cd test;  $(PYTEST) ray_remote_jobs_test.py;
 	. ${VENV_ACTIVATE}; export PYTHONPATH=../src; cd test;  $(PYTEST) pipeline_utils_test.py;
 endif
 endif
diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml
@@ -15,8 +15,7 @@ dependencies = [
     "kfp==1.8.22",
     "ray==2.9.3",
     "requests",
-    "data_prep_toolkit_ray==0.2.0.dev6",
-    "python_apiserver_client==0.1.0",
+    "data_prep_toolkit==0.2.0.dev6",
 ]
 
 [build-system]
@@ -40,7 +39,6 @@ package_dir = ["src"]
 [options.packages.find]
 where = ["src/workflow_support"]
 
-
 [tool.pytest.ini_options]
 addopts = "--cov --cov-report term-missing --cov-fail-under 10"
 markers = ["unit: unit tests", "integration: integration tests"]

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/runtime_utils/__init__.py b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/runtime_utils/__init__.py
diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/runtime_utils/kfp_utils.py b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/runtime_utils/kfp_utils.py