Skip to content

Commit

Permalink
fixing fdedup bug
Browse files Browse the repository at this point in the history
  • Loading branch information
blublinsky committed Jun 16, 2024
1 parent b87abe9 commit ce4b552
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions transforms/universal/fdedup/ray/src/fdedup_transform_ray.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ def __init__(self, params: dict[str, Any]):
num_doc_actors - number of document actors
num_bucket_actors - number of bucket actors
num_minhash_actors - number of minhash actors
n_preprocessors - number of preprocessors
num_preprocessors - number of preprocessors
snapshot_delay - delay (sec) in sending snapshot requests to actors
use_bucket_snapshot - use bucket snapshot
use_doc_snapshot - use doc snapshot
Expand Down Expand Up @@ -541,7 +541,7 @@ def _process_buckets(
"statistics": statistics,
},
actor_options=self.params.get("worker_options", None),
n_actors=self.params.get("n_preprocessors", 1),
n_actors=self.params.get("num_preprocessors", 1),
)
self.logger.info(f"created {len(bucket_processors_list)} bucket processor actors")
# create bucket processors invoker
Expand Down Expand Up @@ -630,7 +630,7 @@ def _preprocess_tables(
from ray.util.metrics import Gauge
worker_options = self.params.get("worker_options", None)
# Here we are limiting the number of readers not to overwhelm COS
n_readers = self.params.get("n_preprocessors", 1)
n_readers = self.params.get("num_preprocessors", 1)
if n_readers > 1000:
n_readers = 1000
self.logger.info(f"Table preprocessing uses {n_readers} readers")
Expand Down

0 comments on commit ce4b552

Please sign in to comment.