From b6e151cb3b47a6dfed9a2e138fe5eeddd3e9e7c6 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Thu, 14 Nov 2024 10:00:00 -0700 Subject: [PATCH] wip --- configs/config_dataflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/config_dataflow.py b/configs/config_dataflow.py index e758832..131ef05 100644 --- a/configs/config_dataflow.py +++ b/configs/config_dataflow.py @@ -9,12 +9,12 @@ c.Bake.container_image = "quay.io/leap-stc/rclone-beam:2024.09.24" c.DataflowBakery.use_dataflow_prime = False c.DataflowBakery.machine_type = "n2d-highmem-16" -c.DataflowBakery.max_workers = 10 +c.DataflowBakery.max_num_workers = 1 c.DataflowBakery.use_public_ips = True c.DataflowBakery.service_account_email = ( "leap-community-bakery@leap-pangeo.iam.gserviceaccount.com" ) -# c.DataflowBakery.autoscaling_algorithm = "NONE" +c.DataflowBakery.autoscaling_algorithm = "NONE" c.DataflowBakery.project_id = "leap-pangeo" c.DataflowBakery.temp_gcs_location = f"gs://leap-scratch/data-library/feedstocks/temp/{FEEDSTOCK_NAME}" c.TargetStorage.fsspec_class = "gcsfs.GCSFileSystem"