more data-processing-lib refence changes in example notebook, kfp, Ma…

…kefile and README.md Signed-off-by: David Wood <[email protected]>
IBM · May 16, 2024 · 7c49c4c · 7c49c4c
1 parent 48bc74d
commit 7c49c4c
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 20 deletions.
diff --git a/Makefile b/Makefile
@@ -45,7 +45,7 @@ test::
 
 lib-release:
 	@# Help: Publish data-prep-kit $(DPK_LIB_VERSION) and data-prep-kit-kfp $(DPK_LIB_KFP_VERSION) libraries to pypi 
-	@$(MAKE) -C data-processing-lib build publish
+	@$(MAKE) -C $(DPK_RAY_LIB_DIR) build publish
 	@$(MAKE) -C kfp/kfp_support_lib build publish
 	@echo ""
 	@echo "This modified files in the repo. Please be sure to commit/push back to the repository."

diff --git a/README.md b/README.md
@@ -65,7 +65,7 @@ Features of the toolkit:
 - It aims to accelerate unstructured data prep for the "long tail" of LLM use cases.
 - It offers a growing set of module implementations across multiple runtimes, targeting laptop-scale to datacenter-scale processing.
 - It provides a growing set of sample pipelines developed for real enterprise use cases.
-- It provides the [Data processing library](data-processing-lib) to enable contribution of new custom modules targeting new use cases.
+- It provides the [Data processing library](data-processing-lib/ray) to enable contribution of new custom modules targeting new use cases.
 - It uses [Kube Flow Pipelines](https://www.kubeflow.org/docs/components/pipelines/v1/introduction/)-based [workflow automation](kfp/doc/simple_transform_pipeline.md) for no-code data prep.
 
 Data modalities supported: 

diff --git a/examples/demo_with_launcher.ipynb b/examples/demo_with_launcher.ipynb
@@ -257,20 +257,20 @@
      "output_type": "stream",
      "text": [
       " skipping data-prep-kit-dev/data-processing-lib/doc/processing-architecture.jpg No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/src/data_processing/__init__.py No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/src/data_processing/test_support/launch/__init__.py No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/daf/input/ds1/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/daf/input/ds1/sample2.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/daf/input/ds2/sample3.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/daf/output/ds1/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/input/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/input_multiple/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/input_multiple/sample2.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/input_multiple/sample3.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/ray/noop/expected/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/ray/noop/expected/subdir/test1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/ray/noop/input/sample1.parquet No contents decoded\n",
-      " skipping data-prep-kit-dev/data-processing-lib/test-data/data_processing/ray/noop/input/subdir/test1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/src/data_processing/__init__.py No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/src/data_processing/test_support/launch/__init__.py No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/daf/input/ds1/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/daf/input/ds1/sample2.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/daf/input/ds2/sample3.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/daf/output/ds1/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/input/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/input_multiple/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/input_multiple/sample2.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/input_multiple/sample3.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/ray/noop/expected/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/ray/noop/expected/subdir/test1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/ray/noop/input/sample1.parquet No contents decoded\n",
+      " skipping data-prep-kit-dev/data-processing-lib/ray/test-data/data_processing/ray/noop/input/subdir/test1.parquet No contents decoded\n",
       " skipping data-prep-kit-dev/doc/data-flow.jpg No contents decoded\n",
       " skipping data-prep-kit-dev/doc/data-transformation.jpg No contents decoded\n",
       " skipping data-prep-kit-dev/examples/requirements.txt No contents decoded\n",

diff --git a/examples/launch.sh b/examples/launch.sh
@@ -4,7 +4,7 @@ echo $REPO_ROOT
 
 
 # Set PYTHONPATH for `data_processing` library
-export PYTHONPATH=$REPO_ROOT/data-processing-lib/src/
+export PYTHONPATH=$REPO_ROOT/data-processing-lib/ray/src/
 
 # Set PYTHONAPATH for transforms
 export PYTHONPATH=$PYTHONPATH:$REPO_ROOT/transforms/code/malware/src

diff --git a/kfp/transform_workflows/.make.transforms_workflows b/kfp/transform_workflows/.make.transforms_workflows
@@ -26,7 +26,7 @@ endef
 .transforms_workflows.compile-pipeline:
 	. ${VENV_ACTIVATE} && ${PYTHON} ${WF_NAME}.py
 
-%.yaml: %.py ${REPOROOT}/.make.versions ${REPOROOT}/kfp/requirements.env ${REPOROOT}/data-processing-lib/ ${REPOROOT}/kfp/kfp_support_lib/
+%.yaml: %.py ${REPOROOT}/.make.versions ${REPOROOT}/kfp/requirements.env ${DPK_RAY_LIB_DIR} ${REPOROOT}/kfp/kfp_support_lib/
 	$(MAKE) .transforms_workflows.reconcile-requirement PIPELINE_FILE=$<
 	$(MAKE) .transforms_workflows.compile-pipeline WF_NAME=$(shell (basename $< .py))
 
@@ -43,13 +43,13 @@ ifeq ($(USE_DEV_IMAGES), 1)
 endif
 	. ${VENV_ACTIVATE}  && ${PYTHON} -m kfp_support.workflow_support.utils.pipelines_tests_utils -c "sanity-test" -p ${CURDIR}/${PIPELINE_FILE}
 
-${VENV_ACTIVATE}: ${REPOROOT}/.make.versions ${REPOROOT}/kfp/requirements.env ${REPOROOT}/kfp/kfp_ray_components/requirements.txt ${REPOROOT}/data-processing-lib/ ${REPOROOT}/kfp/kfp_support_lib/
+${VENV_ACTIVATE}: ${REPOROOT}/.make.versions ${REPOROOT}/kfp/requirements.env ${REPOROOT}/kfp/kfp_ray_components/requirements.txt ${DPK_RAY_LIB_DIR} ${REPOROOT}/kfp/kfp_support_lib/
 	@# Help: Create the virtual environment common to all workflows
 	rm -rf ${REPOROOT}/kfp/transform_workflows/venv
 	$(PYTHON) -m venv ${REPOROOT}/kfp/transform_workflows/venv
 	. ${VENV_ACTIVATE};     \
 	pip install kfp==${KFP} --extra-index-url https://pypi.org/simple; \
-	pip install -e $(REPOROOT)/data-processing-lib/; \
+	pip install -e $($DPK_RAY_LIB_DIR); \
 	pip install -e $(REPOROOT)/kfp/kfp_support_lib/;
 
 .PHONY: .transforms_workflows.upload-pipeline