From 0e4b06c1672d176aa58d0c713d82ca5be7b4933b Mon Sep 17 00:00:00 2001 From: Yunhui Zhang Date: Fri, 7 Jun 2024 18:01:18 -0700 Subject: [PATCH] User can override pod tempalte --- paasta_tools/cli/cmds/spark_run.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py index ad6bbd1335..f56ecaa9ba 100644 --- a/paasta_tools/cli/cmds/spark_run.py +++ b/paasta_tools/cli/cmds/spark_run.py @@ -1216,7 +1216,9 @@ def paasta_spark_run(args: argparse.Namespace) -> int: paasta_cluster = system_paasta_config.get_eks_cluster_aliases().get( args.cluster, args.cluster ) - + user_pod_template_path=user_spark_opts.get( + "spark.kubernetes.executor.podTemplateFile", "" + ) spark_conf_builder = spark_config.SparkConfBuilder() spark_conf = spark_conf_builder.get_spark_conf( cluster_manager=args.cluster_manager, @@ -1234,7 +1236,13 @@ def paasta_spark_run(args: argparse.Namespace) -> int: use_eks=True, k8s_server_address=k8s_server_address, ) - + if user_pod_template_path: + spark_conf["spark.kubernetes.executor.podTemplateFile"] = user_pod_template_path + pod_template_path="" + else: + pod_template_path = spark_conf.get( + "spark.kubernetes.executor.podTemplateFile", "" + ) return configure_and_run_docker_container( args, docker_img=docker_image_digest, @@ -1243,8 +1251,6 @@ def paasta_spark_run(args: argparse.Namespace) -> int: spark_conf=spark_conf, aws_creds=aws_creds, cluster_manager=args.cluster_manager, - pod_template_path=spark_conf.get( - "spark.kubernetes.executor.podTemplateFile", "" - ), + pod_template_path=pod_template_path, extra_driver_envs=driver_envs_from_tronfig, )