storage: Fix partitioner. #TASK-6722

opencb · Oct 29, 2024 · 8268266 · 8268266
1 parent 4ff0655
commit 8268266
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/...-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/StreamVariantDriver.java b/...-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/StreamVariantDriver.java
@@ -155,11 +155,12 @@ protected void setupJob(Job job) throws IOException {
 
     @Override
     protected void setupReducer(Job job, String variantTableName) throws IOException {
-        String numReducersStr = getParam(JobContext.NUM_REDUCES);
+        String numReducersKey = getClass().getSimpleName() + "." + JobContext.NUM_REDUCES;
+        String numReducersStr = getParam(numReducersKey);
         int reduceTasks;
         if (StringUtils.isNotEmpty(numReducersStr)) {
             reduceTasks = Integer.parseInt(numReducersStr);
-            logger.info("Set reduce tasks to " + reduceTasks + " (derived from input parameter '" + JobContext.NUM_REDUCES + "')");
+            logger.info("Set reduce tasks to " + reduceTasks + " (derived from input parameter '" + numReducersKey + "')");
         } else {
             int serversSize = getHBaseManager().act(variantTableName, (table, admin) -> admin.getClusterStatus().getServersSize());
             // Set the number of reduce tasks to 2x the number of hosts

diff --git a/.../src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/StreamVariantPartitioner.java b/.../src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/StreamVariantPartitioner.java
@@ -4,11 +4,11 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Partitioner;
 import org.opencb.opencga.storage.hadoop.variant.GenomeHelper;
 
-import javax.xml.soap.Text;
 import java.io.IOException;
 import java.util.List;
 import java.util.TreeMap;