diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkArrayLoaderFactory.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkArrayLoaderFactory.java
index e003ee7..bff80bc 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkArrayLoaderFactory.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkArrayLoaderFactory.java
@@ -18,7 +18,7 @@
import jp.co.yahoo.yosegi.inmemory.ILoader;
import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkArrayLoader;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyArrayLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkRunLengthEncodingArrayLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionArrayLoader;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -37,7 +37,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
throws IOException {
if (columnBinary == null) {
// FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyArrayLoader(vector, loadSize);
}
switch (getLoadType(columnBinary, loadSize)) {
case ARRAY:
@@ -47,8 +47,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
case UNION:
return new SparkUnionArrayLoader(vector, loadSize);
default:
- // FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyArrayLoader(vector, loadSize);
}
}
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkMapLoaderFactory.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkMapLoaderFactory.java
index b6a38ba..fc06b4c 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkMapLoaderFactory.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkMapLoaderFactory.java
@@ -17,8 +17,8 @@
import jp.co.yahoo.yosegi.binary.ColumnBinary;
import jp.co.yahoo.yosegi.inmemory.ILoader;
import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyMapLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkMapLoader;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionMapLoader;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -36,7 +36,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
throws IOException {
if (columnBinary == null) {
// FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyMapLoader(vector, loadSize);
}
switch (getLoadType(columnBinary, loadSize)) {
case SPREAD:
@@ -45,7 +45,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
return new SparkUnionMapLoader(vector, loadSize);
default:
// FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyMapLoader(vector, loadSize);
}
}
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkStructLoaderFactory.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkStructLoaderFactory.java
index 3c4fb3d..0f0b9f8 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkStructLoaderFactory.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/factory/SparkStructLoaderFactory.java
@@ -17,7 +17,7 @@
import jp.co.yahoo.yosegi.binary.ColumnBinary;
import jp.co.yahoo.yosegi.inmemory.ILoader;
import jp.co.yahoo.yosegi.inmemory.ILoaderFactory;
-import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkNullLoader;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyStructLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkStructLoader;
import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkUnionStructLoader;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -36,7 +36,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
throws IOException {
if (columnBinary == null) {
// FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyStructLoader(vector, loadSize);
}
switch (getLoadType(columnBinary, loadSize)) {
case SPREAD:
@@ -45,7 +45,7 @@ public ILoader createLoader(final ColumnBinary columnBinary, final int loadSize)
return new SparkUnionStructLoader(vector, loadSize);
default:
// FIXME:
- return new SparkNullLoader(vector, loadSize);
+ return new SparkEmptyStructLoader(vector, loadSize);
}
}
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkArrayLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkArrayLoader.java
index f99512a..a272314 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkArrayLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkArrayLoader.java
@@ -38,7 +38,7 @@ public int getLoadSize() {
@Override
public void setNull(final int index) throws IOException {
- vector.putNull(index);
+ vector.putArray(index, 0, 0);
}
@Override
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyArrayLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyArrayLoader.java
new file mode 100644
index 0000000..29ad378
--- /dev/null
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyArrayLoader.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ *
http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+
+import java.io.IOException;
+
+public class SparkEmptyArrayLoader implements ILoader {
+
+ private final WritableColumnVector vector;
+ private final int loadSize;
+
+ public SparkEmptyArrayLoader(final WritableColumnVector vector, final int loadSize) {
+ this.vector = vector;
+ this.loadSize = loadSize;
+ this.vector.getChild(0).reset();
+ this.vector.getChild(0).reserve(0);
+ if (this.vector.getChild(0).hasDictionary()) {
+ this.vector.getChild(0).reserveDictionaryIds(0);
+ this.vector.getChild(0).setDictionary(null);
+ }
+ }
+
+ @Override
+ public LoadType getLoaderType() {
+ return LoadType.NULL;
+ }
+
+ @Override
+ public int getLoadSize() {
+ return loadSize;
+ }
+
+ @Override
+ public void setNull(final int index) throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public void finish() throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public WritableColumnVector build() throws IOException {
+ for (int i = 0; i < loadSize; i++) {
+ vector.putArray(i, 0, 0);
+ }
+ return vector;
+ }
+
+ @Override
+ public boolean isLoadingSkipped() {
+ return true;
+ }
+}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyLoader.java
new file mode 100644
index 0000000..7669968
--- /dev/null
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyLoader.java
@@ -0,0 +1,23 @@
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.MapType;
+import org.apache.spark.sql.types.StructType;
+
+import java.io.IOException;
+
+public class SparkEmptyLoader {
+ public static void load(final WritableColumnVector vector, final int loadSize) throws IOException {
+ final Class klass = vector.dataType().getClass();
+ if (klass == ArrayType.class) {
+ new SparkEmptyArrayLoader(vector, loadSize).build();
+ } else if (klass == StructType.class) {
+ new SparkEmptyStructLoader(vector, loadSize).build();
+ } else if (klass == MapType.class) {
+ new SparkEmptyMapLoader(vector, loadSize).build();
+ } else {
+ new SparkNullLoader(vector, loadSize).build();
+ }
+ }
+}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyMapLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyMapLoader.java
new file mode 100644
index 0000000..7e2f598
--- /dev/null
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyMapLoader.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+
+import java.io.IOException;
+
+public class SparkEmptyMapLoader implements ILoader {
+
+ private final WritableColumnVector vector;
+ private final int loadSize;
+
+ public SparkEmptyMapLoader(final WritableColumnVector vector, final int loadSize) {
+ this.vector = vector;
+ this.loadSize = loadSize;
+ }
+
+ @Override
+ public LoadType getLoaderType() {
+ return LoadType.NULL;
+ }
+
+ @Override
+ public int getLoadSize() {
+ return loadSize;
+ }
+
+ @Override
+ public void setNull(final int index) throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public void finish() throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public WritableColumnVector build() throws IOException {
+ vector.getChild(0).reset();
+ vector.getChild(0).reserve(0);
+ vector.getChild(1).reset();
+ vector.getChild(1).reserve(0);
+ return vector;
+ }
+
+ @Override
+ public boolean isLoadingSkipped() {
+ return true;
+ }
+}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyStructLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyStructLoader.java
new file mode 100644
index 0000000..cabc740
--- /dev/null
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkEmptyStructLoader.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package jp.co.yahoo.yosegi.spark.inmemory.loader;
+
+import jp.co.yahoo.yosegi.inmemory.ILoader;
+import jp.co.yahoo.yosegi.inmemory.LoadType;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.MapType;
+import org.apache.spark.sql.types.StructType;
+
+import java.io.IOException;
+
+public class SparkEmptyStructLoader implements ILoader {
+
+ private final WritableColumnVector vector;
+ private final int loadSize;
+ private final String[] names;
+
+ public SparkEmptyStructLoader(final WritableColumnVector vector, final int loadSize) {
+ this.vector = vector;
+ this.loadSize = loadSize;
+ final StructType structType = (StructType) vector.dataType();
+ this.names = structType.fieldNames();
+ for (int i = 0; i < names.length; i++) {
+ vector.getChild(i).reset();
+ vector.getChild(i).reserve(loadSize);
+ if (vector.getChild(i).hasDictionary()) {
+ vector.getChild(i).reserveDictionaryIds(0);
+ vector.getChild(i).setDictionary(null);
+ }
+ }
+ }
+
+ @Override
+ public LoadType getLoaderType() {
+ return LoadType.NULL;
+ }
+
+ @Override
+ public int getLoadSize() {
+ return loadSize;
+ }
+
+ @Override
+ public void setNull(final int index) throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public void finish() throws IOException {
+ // FIXME:
+ }
+
+ @Override
+ public WritableColumnVector build() throws IOException {
+ for (int i = 0; i < names.length; i++) {
+ SparkEmptyLoader.load(vector.getChild(i), loadSize);
+ }
+ return vector;
+ }
+
+ @Override
+ public boolean isLoadingSkipped() {
+ return true;
+ }
+}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkNullLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkNullLoader.java
index d8d04d1..9f8112f 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkNullLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkNullLoader.java
@@ -14,13 +14,13 @@
*/
package jp.co.yahoo.yosegi.spark.inmemory.loader;
-import jp.co.yahoo.yosegi.inmemory.ISequentialLoader;
+import jp.co.yahoo.yosegi.inmemory.ILoader;
import jp.co.yahoo.yosegi.inmemory.LoadType;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
import java.io.IOException;
-public class SparkNullLoader implements ISequentialLoader {
+public class SparkNullLoader implements ILoader {
private final WritableColumnVector vector;
private final int loadSize;
@@ -42,17 +42,22 @@ public int getLoadSize() {
@Override
public void setNull(final int index) throws IOException {
- // TODO:
+ // FIXME:
}
@Override
public void finish() throws IOException {
// FIXME:
- vector.putNulls(0, loadSize);
}
@Override
public WritableColumnVector build() throws IOException {
+ vector.putNulls(0, loadSize);
return vector;
}
+
+ @Override
+ public boolean isLoadingSkipped() {
+ return true;
+ }
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkRunLengthEncodingArrayLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkRunLengthEncodingArrayLoader.java
index 9c5dda1..3ebb236 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkRunLengthEncodingArrayLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkRunLengthEncodingArrayLoader.java
@@ -49,7 +49,7 @@ public WritableColumnVector build() throws IOException {
@Override
public void setNull(final int index) throws IOException {
- vector.putNull(index);
+ vector.putArray(index, 0, 0);
}
@Override
@@ -59,7 +59,7 @@ public void setRowGroupCount(final int count) throws IOException {}
public void setNullAndRepetitions(
final int startIndex, final int repetitions, final int rowGroupIndex) throws IOException {
for (int i = 0; i < repetitions; i++) {
- vector.putNull(rowId);
+ vector.putArray(rowId, 0, 0);
rowId++;
}
}
@@ -83,6 +83,10 @@ public void setRowGourpIndexAndRepetitions(
public void loadChild(final ColumnBinary columnBinary, final int childLength) throws IOException {
vector.getChild(0).reset();
vector.getChild(0).reserve(childLength);
+ if (vector.getChild(0).hasDictionary()) {
+ vector.getChild(0).reserveDictionaryIds(0);
+ vector.getChild(0).setDictionary(null);
+ }
SparkLoaderFactoryUtil.createLoaderFactory(vector.getChild(0))
.create(columnBinary, childLength);
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkStructLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkStructLoader.java
index fb2e52e..828a1c7 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkStructLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkStructLoader.java
@@ -70,7 +70,7 @@ public WritableColumnVector build() throws IOException {
// NOTE: Fill unloaded columns with nulls.
for (int i = 0; i < names.length; i++) {
if (loaderFactoryMap.containsKey(names[i])) {
- vector.getChild(i).putNulls(0, loadSize);
+ SparkEmptyLoader.load(vector.getChild(i), loadSize);
}
}
return vector;
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionArrayLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionArrayLoader.java
index 346fe26..0950840 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionArrayLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionArrayLoader.java
@@ -29,6 +29,12 @@ public class SparkUnionArrayLoader implements IUnionLoader
public SparkUnionArrayLoader(final WritableColumnVector vector, final int loadSize) {
this.vector = vector;
this.loadSize = loadSize;
+ this.vector.getChild(0).reset();
+ this.vector.getChild(0).reserve(0);
+ if (this.vector.getChild(0).hasDictionary()) {
+ this.vector.getChild(0).reserveDictionaryIds(0);
+ this.vector.getChild(0).setDictionary(null);
+ }
}
@Override
@@ -38,12 +44,12 @@ public int getLoadSize() {
@Override
public void setNull(final int index) throws IOException {
- vector.putNull(index);
+ vector.putArray(index, 0, 0);
}
@Override
public void finish() throws IOException {
- //
+ // FIXME:
}
@Override
@@ -55,13 +61,14 @@ public WritableColumnVector build() throws IOException {
public void setIndexAndColumnType(final int index, final ColumnType columnType) throws IOException {
// FIXME:
if (columnType != ColumnType.ARRAY) {
- vector.putNull(index);
+ vector.putArray(index, 0, 0);
}
}
@Override
public void loadChild(final ColumnBinary columnBinary, final int childLoadSize) throws IOException {
if (columnBinary.columnType == ColumnType.ARRAY) {
+ vector.getChild(0).reserve(childLoadSize);
SparkLoaderFactoryUtil.createLoaderFactory(vector).create(columnBinary, childLoadSize);
}
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionMapLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionMapLoader.java
index 107ccb0..68274e2 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionMapLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionMapLoader.java
@@ -29,6 +29,10 @@ public class SparkUnionMapLoader implements IUnionLoader {
public SparkUnionMapLoader(WritableColumnVector vector, int loadSize) {
this.vector = vector;
this.loadSize = loadSize;
+ this.vector.getChild(0).reset();
+ this.vector.getChild(0).reserve(0);
+ this.vector.getChild(1).reset();
+ this.vector.getChild(1).reserve(0);
}
@Override
@@ -38,12 +42,12 @@ public int getLoadSize() {
@Override
public void setNull(int index) throws IOException {
- vector.putNull(index);
+ // FIXME:
}
@Override
public void finish() throws IOException {
- //
+ // FIXME:
}
@Override
@@ -54,9 +58,6 @@ public WritableColumnVector build() throws IOException {
@Override
public void setIndexAndColumnType(int index, ColumnType columnType) throws IOException {
// FIXME:
- if (columnType != ColumnType.SPREAD) {
- vector.putNull(index);
- }
}
@Override
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionStructLoader.java b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionStructLoader.java
index c4e86de..dfc2341 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionStructLoader.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/inmemory/loader/SparkUnionStructLoader.java
@@ -25,6 +25,7 @@
public class SparkUnionStructLoader implements IUnionLoader {
private final WritableColumnVector vector;
private final int loadSize;
+ private boolean childLoaded;
public SparkUnionStructLoader(WritableColumnVector vector, int loadSize) {
this.vector = vector;
@@ -38,30 +39,31 @@ public int getLoadSize() {
@Override
public void setNull(int index) throws IOException {
- vector.putNull(index);
+ // FIXME:
}
@Override
public void finish() throws IOException {
- //
+ // FIXME:
}
@Override
public WritableColumnVector build() throws IOException {
+ if (!childLoaded) {
+ new SparkEmptyStructLoader(vector, loadSize).build();
+ }
return vector;
}
@Override
public void setIndexAndColumnType(int index, ColumnType columnType) throws IOException {
// FIXME:
- if (columnType != ColumnType.SPREAD) {
- vector.putNull(index);
- }
}
@Override
public void loadChild(ColumnBinary columnBinary, int childLoadSize) throws IOException {
if (columnBinary.columnType == ColumnType.SPREAD) {
+ childLoaded = true;
SparkLoaderFactoryUtil.createLoaderFactory(vector).create(columnBinary, childLoadSize);
}
}
diff --git a/src/main/java/jp/co/yahoo/yosegi/spark/reader/SparkColumnarBatchConverter.java b/src/main/java/jp/co/yahoo/yosegi/spark/reader/SparkColumnarBatchConverter.java
index fb38e8f..19b9a58 100644
--- a/src/main/java/jp/co/yahoo/yosegi/spark/reader/SparkColumnarBatchConverter.java
+++ b/src/main/java/jp/co/yahoo/yosegi/spark/reader/SparkColumnarBatchConverter.java
@@ -17,9 +17,11 @@
import jp.co.yahoo.yosegi.binary.ColumnBinary;
import jp.co.yahoo.yosegi.inmemory.IRawConverter;
import jp.co.yahoo.yosegi.spark.inmemory.SparkLoaderFactoryUtil;
+import jp.co.yahoo.yosegi.spark.inmemory.loader.SparkEmptyLoader;
import jp.co.yahoo.yosegi.spark.utils.PartitionColumnUtil;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnVector;
import org.apache.spark.sql.vectorized.ColumnarBatch;
@@ -72,10 +74,10 @@ public ColumnarBatch convert(final List raw, final int loadSize) t
isSet[index] = true;
SparkLoaderFactoryUtil.createLoaderFactory(((WritableColumnVector) childColumns[index])).create(columnBinary, loadSize);
}
- // NOTE: null columns
+ // NOTE: Empty columns
for (int i = 0; i < schema.length(); i++) {
if (!isSet[i]) {
- ((WritableColumnVector) childColumns[i]).putNulls(0, loadSize);
+ SparkEmptyLoader.load((WritableColumnVector) childColumns[i], loadSize);
}
}
// NOTE: partitionColumns
diff --git a/src/test/java/jp/co/yahoo/yosegi/spark/blackbox/LoadTest.java b/src/test/java/jp/co/yahoo/yosegi/spark/blackbox/LoadTest.java
index 0b8b6c6..65ddaf7 100644
--- a/src/test/java/jp/co/yahoo/yosegi/spark/blackbox/LoadTest.java
+++ b/src/test/java/jp/co/yahoo/yosegi/spark/blackbox/LoadTest.java
@@ -322,33 +322,24 @@ void T_load_Array_Array_Integer_1() throws IOException {
final int jIndex = ldfj.get(i).fieldIndex("aa");
final int yIndex = ldfy.get(i).fieldIndex("aa");
if (ldfj.get(i).isNullAt(jIndex)) {
- if (ldfy.get(i).isNullAt(yIndex)) {
- // NOTE: json:null, yosegi:null
- assertTrue(ldfy.get(i).isNullAt(yIndex));
- } else {
- // FIXME: json:null, yosegi:[]
- assertTrue(false);
- }
+ // NOTE: json:null, yosegi:[]
+ assertEquals(0, ldfy.get(i).getList(yIndex).size());
} else {
if (ldfy.get(i).isNullAt(yIndex)) {
// NOTE: json:[], yosegi:null
- assertEquals(0, ldfj.get(i).getList(jIndex).size());
+ assertTrue(false);
} else {
final List> ldfj2 = ldfj.get(i).getList(jIndex);
final List> ldfy2 = ldfy.get(i).getList(yIndex);
for (int j = 0; j < ldfj2.size(); j++) {
final WrappedArray waj = ldfj2.get(j);
final WrappedArray way = ldfy2.get(j);
- if (way == null) {
- if (waj == null) {
- // NOTE: json:[null], yosegi:[null]
- assertNull(waj);
- } else {
- // NOTE: json:[[]], yosegi:[null]
- assertEquals(0, waj.size());
- }
+ if (waj == null) {
+ // NOTE: json:[null], yosegi:[[]]
+ assertEquals(0, way.size());
} else {
// NOTE: json:[[]], yosegi:[[]]
+ assertEquals(waj.size(), way.size());
for (int k = 0; k < waj.size(); k++) {
assertEquals(waj.apply(k), way.apply(k));
}
@@ -397,59 +388,28 @@ void T_load_Array_Struct_Primitive_1() throws IOException {
final int jIndex = ldfj.get(i).fieldIndex("as");
final int yIndex = ldfy.get(i).fieldIndex("as");
if (ldfj.get(i).isNullAt(jIndex)) {
- if (ldfy.get(i).isNullAt(yIndex)) {
- // NOTE: json:null, yosegi:null
- assertTrue(ldfy.get(i).isNullAt(yIndex));
- } else {
- // FIXME: json:null, yosegi:[]
- assertTrue(false);
- }
+ // NOTE: json:null, yosegi:[]
+ assertEquals(0, ldfy.get(i).getList(yIndex).size());
} else {
if (ldfy.get(i).isNullAt(yIndex)) {
- final List lrj = ldfj.get(i).getList(jIndex);
- for (int j = 0; j < lrj.size(); j++) {
- final Row rj = lrj.get(j);
- if (rj == null) {
- // NOTE: json:[null], yosegi:null
- assertNull(rj);
- } else {
- // NOTE: json[as.field:null], yosegi:null
- for (final StructField field : fields) {
- final String name = field.name();
- assertNull(rj.getAs(name));
- }
- }
- }
+ assertTrue(false);
} else {
final List lrj = ldfj.get(i).getList(jIndex);
final List lry = ldfy.get(i).getList(yIndex);
for (int j = 0; j < lrj.size(); j++) {
final Row rj = lrj.get(j);
final Row ry = lry.get(j);
- if (ry == null) {
- if (rj == null) {
- // NOTE: json:[null], yosegi:[null]
- assertNull(rj);
- } else {
- // NOTE: json:[{}], yosegi:[null]
- for (final StructField field : fields) {
- final String name = field.name();
- assertNull(rj.getAs(name));
- }
+ if (rj == null) {
+ // NOTE: json:[null], yosegi:[{}]
+ for (final StructField field : fields) {
+ final String name = field.name();
+ assertNull(ry.getAs(name));
}
} else {
- if (rj == null) {
- // NOTE: json:[null], yosegi:[{}]
- for (final StructField field : fields) {
- final String name = field.name();
- assertNull(ry.getAs(name));
- }
- } else {
- // NOTE: json:[{}], yosegi:[{}]
- for (final StructField field : fields) {
- final String name = field.name();
- assertEquals((Object) rj.getAs(name), (Object) ry.getAs(name));
- }
+ // NOTE: json:[{}], yosegi:[{}]
+ for (final StructField field : fields) {
+ final String name = field.name();
+ assertEquals((Object) rj.getAs(name), (Object) ry.getAs(name));
}
}
}
@@ -484,31 +444,12 @@ void T_load_Array_Map_Integer_1() throws IOException {
final int jIndex = ldfj.get(i).fieldIndex("am");
final int yIndex = ldfy.get(i).fieldIndex("am");
if (ldfj.get(i).isNullAt(jIndex)) {
- if (ldfy.get(i).isNullAt(yIndex)) {
- // NOTE: json:null, yosegi:null
- assertTrue(ldfy.get(i).isNullAt(yIndex));
- } else {
- // FIXME: json:null, yosegi:[]
- assertTrue(false);
- }
+ // NOTE: json:null, yosegi:[]
+ assertEquals(0, ldfy.get(i).getList(yIndex).size());
} else {
if (ldfy.get(i).isNullAt(yIndex)) {
// NOTE: json:[], yosegi:null
- final List