Merge pull request #405 from IBM/make-mods

Enables configuration of makefile to use src or pypi for data-prep-kit library dependencies
IBM · Jul 12, 2024 · 240bfae · 240bfae
2 parents a7d3c42 + ef16e7f
commit 240bfae
Showing 1 changed file with 44 additions and 21 deletions.
diff --git a/.make.defaults b/.make.defaults
@@ -51,6 +51,9 @@ RAY_BASE_IMAGE?=docker.io/rayproject/ray:${RAY}-py310
 DOCKER_IMAGE?=$(DOCKER_REMOTE_IMAGE)
 KIND_CLUSTER_NAME=dataprep
 ARTIFACTS_DIR?=$(REPOROOT)/artifacts
+# Set this to 0 to disable installation of data-processing-lib/* into virtual environments
+# Should cause data-prep-kit dependencies to be loaded from pypi instead.
+USE_REPO_LIB_SRC?=1
 
 DPK_PYTHON_LIB_DIR=$(REPOROOT)/data-processing-lib/python
 DPK_RAY_LIB_DIR=$(REPOROOT)/data-processing-lib/ray
@@ -232,9 +235,11 @@ __check_defined = \
 .PHONY: .defaults.python-lib-src-image
 .defaults.python-lib-src-image:: # Must be called with a DOCKER_LOCAL_IMAGE= settings.
 	@# Help: Build the Python $(DOCKER_LOCAL_IMAGE) using the $(DOCKER_FILE), requirements.txt and data-processing-lib/python source
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	$(MAKE) LIB_PATH=$(DPK_PYTHON_LIB_DIR) LIB_NAME=data-processing-lib-python .defaults.copy-lib
+endif
 	$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) .defaults.image
-	rm -rf data-processing-lib-python
+	-rm -rf data-processing-lib-python
 
 # Build an image using the local Dockerfile and make the data-processing-lib/ray
 # available in the current directory for use by the Dockerfile (i.e. to install the library). 
@@ -243,15 +248,17 @@ __check_defined = \
 .PHONY: .defaults.ray-lib-src-image
 .defaults.ray-lib-src-image:: # Must be called with a DOCKER_LOCAL_IMAGE= settings.
 	@# Help: Build the Ray $(DOCKER_LOCAL_IMAGE) using the $(DOCKER_FILE), requirements.txt and data-processing-libs source
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	$(MAKE) LIB_PATH=$(DPK_PYTHON_LIB_DIR) LIB_NAME=data-processing-lib-python .defaults.copy-lib
 	$(MAKE) LIB_PATH=$(DPK_RAY_LIB_DIR) LIB_NAME=data-processing-lib-ray .defaults.copy-lib
+endif
 	if [ -e ../python ]; then								\
 		$(MAKE) LIB_PATH=../python LIB_NAME=python-transform .defaults.copy-lib;	\
 	fi
 	$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) .defaults.image
-	rm -rf data-processing-lib-python
-	rm -rf data-processing-lib-ray
-	rm -rf python-transform 
+	-rm -rf data-processing-lib-python
+	-rm -rf data-processing-lib-ray
+	-rm -rf python-transform 
 
 # Build the base spark image used by spark-based transforms
 .PHONY: .defaults.spark-lib-base-image
@@ -265,27 +272,30 @@ __check_defined = \
 .defaults.spark-lib-src-image:: .defaults.spark-lib-base-image
 	@# Help: Build the Spark $(DOCKER_LOCAL_IMAGE) using the $(DOCKER_FILE), requirements.txt and data-processing-libs source
 	$(MAKE) IMAGE_NAME_TO_VERIFY=$(DOCKER_SPARK_BASE_IMAGE_NAME) .defaults.verify-image-availability
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	$(MAKE) LIB_PATH=$(DPK_PYTHON_LIB_DIR) LIB_NAME=data-processing-lib-python .defaults.copy-lib
 	$(MAKE) LIB_PATH=$(DPK_SPARK_LIB_DIR) LIB_NAME=data-processing-lib-spark .defaults.copy-lib
+endif
 	if [ -e ../python ]; then								\
 		$(MAKE) LIB_PATH=../python LIB_NAME=python-transform .defaults.copy-lib;	\
 	fi
 	$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE) .defaults.image
-	rm -rf data-processing-lib-python
-	rm -rf data-processing-lib-spark
-	rm -rf python-transform 
+	-rm -rf data-processing-lib-python
+	-rm -rf data-processing-lib-spark
+	-rm -rf python-transform 
 
 # Install the source from the given directory into an existing venv
 # Expected PYTHON_PROJECT_DIR and uses EXTRA_INDEX_URL if set.
 # PYTHON_PROJECT_DIR is expected to have src and pyproject.toml
 .PHONY: .defaults.install-src-venv
 .defaults.install-src-venv:: 
+	@echo Begin installing source from $(PYTHON_PROJECT_DIR) into venv
 	@source venv/bin/activate;				      	\
 	if [ ! -z "$(EXTRA_INDEX_URL)" ]; then				\
 		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
 	fi;								\
-	pip install $${extra_url}  -e $(PYTHON_PROJECT_DIR);		
-	@echo Installed python project source in $(PYTHON_PROJECT_DIR) into venv
+	pip install --force-reinstall $${extra_url}  -e $(PYTHON_PROJECT_DIR);		
+	@echo Done installing source from $(PYTHON_PROJECT_DIR) into venv
 
 # Install local requirements last as it generally includes our lib source
 .PHONY: .defaults.python-lib-src-venv
@@ -308,13 +318,17 @@ __check_defined = \
 # Install all source from the repo for a python runtime transform into an existing venv 
 .PHONY: .defaults.install-python-lib-src-venv
 .defaults.install-python-lib-src-venv::
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	@# Help: Install Python data processing library source into existing venv
 	@echo Installing Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
-	pip install pytest;						\
 	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;	\
 	echo Installed source from Python processing library for `which $(PYTHON)`
+else
+	@# Help: DO NOT install Python data processing library source into existing venv
+	@echo USE_REPO_LIB_SRC!=1 so do NOT installing Python data processing library source into existing venv
+endif
 
 # Install local requirements last as it generally includes our lib source
 .PHONY: .defaults.ray-lib-src-venv
@@ -325,17 +339,21 @@ __check_defined = \
 # And if there is an adjacent python dir (as for transforms), then also install that source
 .PHONY: .defaults.install-ray-lib-src-venv
 .defaults.install-ray-lib-src-venv::
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	@# Help: Install Ray and Python data processing library source into existing venv
 	@echo Installing Ray and Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
-	pip install pytest;						\
-	pip install pytest pytest-cov;					\
 	$(MAKE) PIP_TARGET=data-prep-toolkit-ray .defaults.pip-uninstall;	\
 	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;		\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_RAY_LIB_DIR) .defaults.install-src-venv;		\
-	echo Installed source from Ray data processing library for `which $(PYTHON)`;		\
-	if [ -d ../python ]; then                                               		\
+	echo Installed source from Python and Ray data processing libraries for `which $(PYTHON)`
+else
+	@# Help: DO NOT install Python or Ray data processing library source into existing venv
+	@echo USE_REPO_LIB_SRC!=1 so do NOT installing Python or Ray data processing library source into existing venv
+endif
+	@if [ -d ../python ]; then                                               		\
+		source venv/bin/activate;				      			\
 		$(MAKE) PYTHON_PROJECT_DIR=../python .defaults.install-src-venv;    		\
 	fi 
 
@@ -347,18 +365,23 @@ __check_defined = \
 # Install the python-based lib BEFORE spark assuming spark depends on the same version as python source.
 .PHONY: .defaults.install-spark-lib-src-venv
 .defaults.install-spark-lib-src-venv::
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	@# Help: Install Spark and Python data processing library source into existing venv
 	@echo Installing Spark and Python data processing library source to existing venv
 	@source venv/bin/activate;				      	\
-	pip install pytest;						\
 	$(MAKE) PIP_TARGET=data-prep-toolkit-spark .defaults.pip-uninstall;	\
 	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_SPARK_LIB_DIR) .defaults.install-src-venv;	\
-	if [ -d ../python ]; then                                               \
-		$(MAKE) PYTHON_PROJECT_DIR=../python .defaults.install-src-venv;    \
+	echo Installed source from Python and Spark processing libraries for `which $(PYTHON)`
+else
+	@# Help: DO NOT install Python or Spark data processing library source into existing venv
+	@echo USE_REPO_LIB_SRC!=1 so do NOT installing Python or Spark data processing library source into existing venv
+endif
+	if [ -d ../python ]; then                                               	\
+		source venv/bin/activate;				      		\
+		$(MAKE) PYTHON_PROJECT_DIR=../python .defaults.install-src-venv;    	\
 	fi
-	echo Installed source from Spark processing library for `which $(PYTHON)`
 
 # Run tests in test directory from that dir after adding ../src to PYTHONPATH
 # Assumes a Makefile target of venv to create the venv 
@@ -434,7 +457,7 @@ __check_defined = \
 	$(PYTHON) -m venv venv
 	@source venv/bin/activate;		\
 	pip install --upgrade pip;		\
-	pip install wheel;			\
+	pip install wheel pytest pytest-cov;
 
 # Install requirements defined in the current directory into an existing venv
 .PHONY: .defaults.install-local-requirements-venv
@@ -444,10 +467,10 @@ __check_defined = \
 		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
 	fi;								\
 	if [ -e requirements.txt ]; then				\
-		echo Install requirements from requirements.txt;	\
+		echo Installing requirements from requirements.txt;	\
 		pip install $$extra_url -r requirements.txt;		\
 	elif [ -e pyproject.toml ]; then				\
-		echo Install requirements using pyproject.toml;		\
+		echo Installing from pyproject.toml;			\
 		pip install $$extra_url -e .;				\
 	fi