Merge branch 'oap-project:master' into master

oap-project · Oct 16, 2024 · c29e617 · c29e617
2 parents 7ff407c + e44b052
commit c29e617
Show file tree

Hide file tree

Showing 16 changed files with 57 additions and 47 deletions.
diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml
@@ -186,7 +186,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <version>3.21.7</version>
+      <version>3.25.5</version>
       <scope>compile</scope>
     </dependency>
     <dependency>

diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/ColumnAccessor.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/ColumnAccessor.java
@@ -4,6 +4,11 @@ public class ColumnAccessor {
     private long cObject;
     private Common.ComputeDevice cDevice;
 
+    public ColumnAccessor(long cObject) {
+        this.cObject = cObject;
+        this.cDevice = Common.ComputeDevice.HOST;
+    }
+
     public ColumnAccessor(long cObject, Common.ComputeDevice device) {
         this.cObject = cObject;
         this.cDevice = device;

diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/RowAccessor.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/RowAccessor.java
@@ -3,6 +3,11 @@
 public class RowAccessor {
     private long cObject;
     private Common.ComputeDevice cDevice;
+    public RowAccessor(long cObject) {
+        this.cObject = cObject;
+        this.cDevice = Common.ComputeDevice.HOST;
+    }
+
     public RowAccessor(long cObject, Common.ComputeDevice device) {
         this.cObject = cObject;
         this.cDevice = device;

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala
@@ -69,7 +69,8 @@ object OneDAL {
     matrix
   }
 
-  def homogenTableToMatrix(table: HomogenTable, device: Common.ComputeDevice): Matrix = {
+  def homogenTableToMatrix(table: HomogenTable,
+                           device: Common.ComputeDevice = Common.ComputeDevice.HOST): Matrix = {
     val numRows = table.getRowCount.toInt
     val numCols = table.getColumnCount.toInt
 
@@ -82,7 +83,9 @@ object OneDAL {
     matrix
   }
 
-  def homogenTableToOldMatrix(table: HomogenTable, device: Common.ComputeDevice): OldMatrix = {
+  def homogenTableToOldMatrix(table: HomogenTable,
+                              device: Common.ComputeDevice = Common.ComputeDevice.HOST)
+  : OldMatrix = {
     val numRows = table.getRowCount.toInt
     val numCols = table.getColumnCount.toInt
 
@@ -115,7 +118,8 @@ object OneDAL {
     Vectors.dense(arrayDouble)
   }
 
-  def homogenTableNx1ToVector(cTable: Long, device: Common.ComputeDevice ): Vector = {
+  def homogenTableNx1ToVector(cTable: Long,
+                              device: Common.ComputeDevice = Common.ComputeDevice.HOST): Vector = {
     val columnAcc = new ColumnAccessor(cTable, device)
     val arrayDouble = columnAcc.pullDouble(0)
     Vectors.dense(arrayDouble)
@@ -135,7 +139,8 @@ object OneDAL {
     Vectors.dense(arrayDouble)
   }
 
-  def homogenTable1xNToVector(table: HomogenTable, device: Common.ComputeDevice): Vector = {
+  def homogenTable1xNToVector(table: HomogenTable,
+                              device: Common.ComputeDevice = Common.ComputeDevice.HOST): Vector = {
     val rowAcc = new RowAccessor(table.getcObejct, device)
     val arrayDouble = rowAcc.pullDouble(0, 1)
     Vectors.dense(arrayDouble)
@@ -159,7 +164,9 @@ object OneDAL {
     resArray
   }
 
-  def homogenTableToVectors(table: HomogenTable, device: Common.ComputeDevice): Array[Vector] = {
+  def homogenTableToVectors(table: HomogenTable,
+                            device: Common.ComputeDevice = Common.ComputeDevice.HOST)
+  : Array[Vector] = {
     val numRows = table.getRowCount.toInt
 
     val rowAcc = new RowAccessor(table.getcObejct(), device)

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansDALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansDALImpl.scala
@@ -109,8 +109,7 @@ class KMeansDALImpl(var nClusters: Int,
       val ret = if (rank == 0) {
           assert(cCentroids != 0)
           val centerVectors = if (useDevice == "GPU") {
-            OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(cCentroids),
-              computeDevice)
+            OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(cCentroids))
           } else {
             OneDAL.numericTableToVectors(OneDAL.makeNumericTable(cCentroids))
           }

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCADALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCADALImpl.scala
@@ -104,7 +104,7 @@ class PCADALImpl(val k: Int,
       val ret = if (rank == 0) {
         val principleComponents = if (useDevice == "GPU") {
           val pcNumericTable = OneDAL.makeHomogenTable(result.getPcNumericTable)
-          getPrincipleComponentsFromOneAPI(pcNumericTable, k, computeDevice)
+          getPrincipleComponentsFromOneAPI(pcNumericTable, k)
         } else {
           val pcNumericTable = OneDAL.makeNumericTable(result.getPcNumericTable)
           getPrincipleComponentsFromDAL(pcNumericTable, k)
@@ -114,7 +114,7 @@ class PCADALImpl(val k: Int,
           val explainedVarianceNumericTable = OneDAL.makeHomogenTable(
             result.getExplainedVarianceNumericTable)
           getExplainedVarianceFromOneAPI(
-            explainedVarianceNumericTable, k, computeDevice)
+            explainedVarianceNumericTable, k)
         } else {
           val explainedVarianceNumericTable = OneDAL.makeNumericTable(
             result.getExplainedVarianceNumericTable)
@@ -153,21 +153,20 @@ class PCADALImpl(val k: Int,
   }
 
   private[mllib] def getPrincipleComponentsFromOneAPI(table: HomogenTable,
-                                                      k: Int,
-                                                      device: Common.ComputeDevice): DenseMatrix = {
+                                                      k: Int): DenseMatrix = {
     val numRows = table.getRowCount.toInt
     val numCols = table.getColumnCount.toInt
     require(k <= numRows, "k should be less or equal to row number")
 
-    val arrayDouble = getDoubleBufferDataFromOneAPI(table, numRows, device)
+    val arrayDouble = getDoubleBufferDataFromOneAPI(table, numRows)
 
     // Column-major, transpose of top K rows of NumericTable
     new DenseMatrix(numCols, k, arrayDouble.slice(0, numCols * k), false)
   }
 
-  private[mllib] def getExplainedVarianceFromOneAPI(table_1xn: HomogenTable, k: Int,
-                                          device: Common.ComputeDevice): DenseVector = {
-    val arrayDouble = getDoubleBufferDataFromOneAPI(table_1xn, 1, device)
+  private[mllib] def getExplainedVarianceFromOneAPI(table_1xn: HomogenTable,
+                                                    k: Int): DenseVector = {
+    val arrayDouble = getDoubleBufferDataFromOneAPI(table_1xn, 1)
     val sum = arrayDouble.sum
     val topK = Arrays.copyOfRange(arrayDouble, 0, k)
     for (i <- 0 until k)
@@ -178,11 +177,10 @@ class PCADALImpl(val k: Int,
   // table.asInstanceOf[HomogenNumericTable].getDoubleArray() would error on GPU,
   // so use table.getBlockOfRows instead of it.
   private[mllib] def getDoubleBufferDataFromOneAPI(table: HomogenTable,
-                                         numRows: Int,
-                                         device: Common.ComputeDevice): Array[Double] = {
+                                         numRows: Int): Array[Double] = {
 
     // returned DoubleBuffer is ByteByffer, need to copy as double array
-    val accessor = new RowAccessor(table.getcObejct(), device)
+    val accessor = new RowAccessor(table.getcObejct())
     val arrayDouble: Array[Double] = accessor.pullDouble(0, numRows)
 
     arrayDouble

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionDALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionDALImpl.scala
@@ -167,8 +167,7 @@ class LinearRegressionDALImpl( val fitIntercept: Boolean,
 
         val ret = if (rank == 0) {
           val coefficientArray = if (useDevice == "GPU") {
-              OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(cbeta),
-                computeDevice)
+              OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(cbeta))
             } else {
               OneDAL.numericTableToVectors(OneDAL.makeNumericTable(cbeta))
             }

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorDALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorDALImpl.scala
@@ -129,8 +129,7 @@ class RandomForestRegressorDALImpl(val uid: String,
       val ret = if (rank == 0) {
         val convResultStartTime = System.nanoTime()
         val predictionNumericTable = OneDAL.homogenTableToMatrix(
-          OneDAL.makeHomogenTable(result.getPredictionNumericTable),
-          computeDevice)
+          OneDAL.makeHomogenTable(result.getPredictionNumericTable))
         val convResultEndTime = System.nanoTime()
 
         val durationCovResult = (convResultEndTime - convResultStartTime).toDouble / 1E9

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationDALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationDALImpl.scala
@@ -100,8 +100,7 @@ class CorrelationDALImpl(
       val ret = if (rank == 0) {
         val convResultStartTime = System.nanoTime()
         val correlationNumericTable = if (useDevice == "GPU") {
-          OneDAL.homogenTableToMatrix(OneDAL.makeHomogenTable(result.getCorrelationNumericTable),
-            computeDevice)
+          OneDAL.homogenTableToMatrix(OneDAL.makeHomogenTable(result.getCorrelationNumericTable))
         } else {
           OneDAL.numericTableToMatrix(OneDAL.makeNumericTable(result.getCorrelationNumericTable))
         }

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerDALImpl.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerDALImpl.scala
@@ -103,28 +103,28 @@ class SummarizerDALImpl(val executorNum: Int,
         val convResultStartTime = System.nanoTime()
         val meanVector = if (useDevice == "GPU") {
           OneDAL.homogenTable1xNToVector(
-            OneDAL.makeHomogenTable(result.getMeanNumericTable), computeDevice)
+            OneDAL.makeHomogenTable(result.getMeanNumericTable))
         } else {
           OneDAL.numericTable1xNToVector(
             OneDAL.makeNumericTable(result.getMeanNumericTable))
         }
         val varianceVector = if (useDevice == "GPU") {
           OneDAL.homogenTable1xNToVector(
-            OneDAL.makeHomogenTable(result.getVarianceNumericTable), computeDevice)
+            OneDAL.makeHomogenTable(result.getVarianceNumericTable))
         } else {
           OneDAL.numericTable1xNToVector(
             OneDAL.makeNumericTable(result.getVarianceNumericTable))
         }
         val maxVector = if (useDevice == "GPU") {
           OneDAL.homogenTable1xNToVector(
-            OneDAL.makeHomogenTable(result.getMaximumNumericTable), computeDevice)
+            OneDAL.makeHomogenTable(result.getMaximumNumericTable))
         } else {
           OneDAL.numericTable1xNToVector(
             OneDAL.makeNumericTable(result.getMaximumNumericTable))
         }
         val minVector = if (useDevice == "GPU") {
           OneDAL.homogenTable1xNToVector(
-            OneDAL.makeHomogenTable(result.getMinimumNumericTable), computeDevice)
+            OneDAL.makeHomogenTable(result.getMinimumNumericTable))
         } else {
           OneDAL.numericTable1xNToVector(
             OneDAL.makeNumericTable(result.getMinimumNumericTable))

diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala
@@ -116,7 +116,7 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging {
 
     val expectData = Array(5.308206,9.869278)
     val table = new HomogenTable(5, 2, data, TestCommon.getComputeDevice)
-    val vector = OneDAL.homogenTable1xNToVector(table, TestCommon.getComputeDevice)
+    val vector = OneDAL.homogenTable1xNToVector(table)
 
     assertArrayEquals(expectData, vector.toArray)
   }
@@ -148,7 +148,7 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging {
                      85.208661d, 15.966239d)
     val expectData = Array(5.236359d, 40.724176d, 90.119887d, 53.620204d, 85.208661d)
     val table = new HomogenTable(5, 2, data, TestCommon.getComputeDevice)
-    val vector = OneDAL.homogenTableNx1ToVector(table.getcObejct(), TestCommon.getComputeDevice)
+    val vector = OneDAL.homogenTableNx1ToVector(table.getcObejct())
 
     assertArrayEquals(expectData, vector.toArray)
   }
@@ -162,7 +162,7 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging {
     val expectMatrix = new DenseMatrix(5, 2, data, isTransposed = true)
     val table = new HomogenTable(5, 2, data, TestCommon.getComputeDevice)
 
-    val matrix = OneDAL.homogenTableToMatrix(table, TestCommon.getComputeDevice)
+    val matrix = OneDAL.homogenTableToMatrix(table)
 
     assertArrayEquals(expectMatrix.toArray, matrix.toArray)
   }
@@ -176,7 +176,7 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging {
     val expectMatrix = new OldDenseMatrix(5, 2, data, isTransposed = true)
     val table = new HomogenTable(5, 2, data, TestCommon.getComputeDevice)
 
-    val matrix = OneDAL.homogenTableToOldMatrix(table, TestCommon.getComputeDevice )
+    val matrix = OneDAL.homogenTableToOldMatrix(table)
 
     assertArrayEquals(expectMatrix.toArray, matrix.toArray)
   }
@@ -197,7 +197,7 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging {
 
     val arrayData = TestCommon.convertArray(data)
     val table = new HomogenTable(10, 10, arrayData, TestCommon.getComputeDevice)
-    val array = OneDAL.homogenTableToVectors(table, TestCommon.getComputeDevice)
+    val array = OneDAL.homogenTableToVectors(table)
 
     assertArrayEquals(TestCommon.convertArray(data), TestCommon.convertArray(array))
   }

diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/CorrelationHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/CorrelationHomogenTableSuite.scala
@@ -47,7 +47,7 @@ class CorrelationHomogenTableSuite extends FunctionsSuite with Logging {
         val result = new CorrelationResult()
         correlationDAL.cCorrelationTrainDAL(0, dataTable.getcObejct(), sourceData.length, sourceData(0).length, 1, 1, Common.ComputeDevice.HOST.ordinal(), gpuIndices, result);
         val correlationMatrix = TestCommon.getMatrixFromTable(OneDAL.makeHomogenTable(
-            result.getCorrelationNumericTable), TestCommon.getComputeDevice)
+            result.getCorrelationNumericTable))
 
         assertArrayEquals(TestCommon.convertArray(expectCorrelation), correlationMatrix.toArray, 0.000001)
     }

diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/KmeansHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/KmeansHomogenTableSuite.scala
@@ -51,7 +51,7 @@ class KmeansHomogenTableSuite extends FunctionsSuite with Logging {
         val result = new KMeansResult();
         val centroids = kmeansDAL.cKMeansOneapiComputeWithInitCenters(0, dataTable.getcObejct(), sourceData.length, sourceData(0).length, centroidsTable.getcObejct(),10, 0.001,
             5, 1, 1, TestCommon.getComputeDevice.ordinal(), gpuIndices, result);
-        val resultVectors = OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(centroids), TestCommon.getComputeDevice);
+        val resultVectors = OneDAL.homogenTableToVectors(OneDAL.makeHomogenTable(centroids));
         assertArrayEquals(TestCommon.convertArray(expectCentroids), TestCommon.convertArray(resultVectors), 0.000001)
     }
 }
diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/PCAHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/PCAHomogenTableSuite.scala
@@ -51,8 +51,8 @@ class PCAHomogenTableSuite extends FunctionsSuite with Logging {
         val pcNumericTable = OneDAL.makeHomogenTable(result.getPcNumericTable)
         val explainedVarianceNumericTable = OneDAL.makeHomogenTable(
             result.getExplainedVarianceNumericTable)
-        val principleComponents = OneDAL.homogenTableToMatrix(pcNumericTable, TestCommon.getComputeDevice)
-        val explainedVariance = OneDAL.homogenTable1xNToVector(explainedVarianceNumericTable, TestCommon.getComputeDevice)
+        val principleComponents = OneDAL.homogenTableToMatrix(pcNumericTable)
+        val explainedVariance = OneDAL.homogenTable1xNToVector(explainedVarianceNumericTable)
 
         assertArrayEquals(expectExplainedVariance , explainedVariance.toArray, 0.000001)
         assertArrayEquals(TestCommon.convertArray(expectPC), principleComponents.toDense.values, 0.000001)
@@ -83,8 +83,8 @@ class PCAHomogenTableSuite extends FunctionsSuite with Logging {
         val pcNumericTable = OneDAL.makeHomogenTable(result.getPcNumericTable)
         val explainedVarianceNumericTable = OneDAL.makeHomogenTable(
             result.getExplainedVarianceNumericTable)
-        val principleComponents = OneDAL.homogenTableToMatrix(pcNumericTable, TestCommon.getComputeDevice)
-        val explainedVariance = OneDAL.homogenTable1xNToVector(explainedVarianceNumericTable, TestCommon.getComputeDevice)
+        val principleComponents = OneDAL.homogenTableToMatrix(pcNumericTable)
+        val explainedVariance = OneDAL.homogenTable1xNToVector(explainedVarianceNumericTable)
 
         assertArrayEquals(expectExplainedVariance , explainedVariance.toArray, 0.000001)
         assertArrayEquals(TestCommon.convertArray(expectPC), principleComponents.toDense.values, 0.000001)

diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/SummarizerHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/SummarizerHomogenTableSuite.scala
@@ -36,10 +36,10 @@ class SummarizerHomogenTableSuite extends FunctionsSuite with Logging{
         val gpuIndices = Array(0)
         val result = new SummarizerResult()
         summarizerDAL.cSummarizerTrainDAL(0, dataTable.getcObejct(), sourceData.length, sourceData(0).length, 1, 1, Common.ComputeDevice.HOST.ordinal(), gpuIndices, result)
-        val meanTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMeanNumericTable), Common.ComputeDevice.HOST)
-        val varianceTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getVarianceNumericTable), Common.ComputeDevice.HOST)
-        val minimumTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMinimumNumericTable), Common.ComputeDevice.HOST)
-        val maximumTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMaximumNumericTable), Common.ComputeDevice.HOST)
+        val meanTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMeanNumericTable))
+        val varianceTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getVarianceNumericTable))
+        val minimumTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMinimumNumericTable))
+        val maximumTable = OneDAL.homogenTable1xNToVector(OneDAL.makeHomogenTable(result.getMaximumNumericTable))
 
         assertArrayEquals(expectMean , meanTable.toArray, 0.000001)
         assertArrayEquals(expectVariance, varianceTable.toDense.values, 0.000001)

diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/TestCommon.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/TestCommon.scala
@@ -83,12 +83,11 @@ object TestCommon {
     }
     arrayDouble
   }
-  def getMatrixFromTable(table: HomogenTable,
-                                  device: Common.ComputeDevice): DenseMatrix = {
+  def getMatrixFromTable(table: HomogenTable): DenseMatrix = {
     val numRows = table.getRowCount.toInt
     val numCols = table.getColumnCount.toInt
     // returned DoubleBuffer is ByteByffer, need to copy as double array
-    val accessor = new RowAccessor(table.getcObejct(), device)
+    val accessor = new RowAccessor(table.getcObejct())
     val arrayDouble: Array[Double] = accessor.pullDouble(0, numRows)
 
     // Transpose as DAL numeric table is row-major and DenseMatrix is column major