Skip to content

Commit

Permalink
addressing comments
Browse files Browse the repository at this point in the history
  • Loading branch information
burhan94 committed Jul 29, 2021
1 parent 7eed5c4 commit 1f7b303
Show file tree
Hide file tree
Showing 6 changed files with 41 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ else if (lhs instanceof org.joda.time.LocalDateTime) {
else {
return ((java.time.LocalDateTime) lhs).compareTo((java.time.LocalDateTime) rhs);
}
case MAP:
case LIST:
case STRUCT: // struct maps to java.util.map
//This could lead to thrashing if used to sort a collection
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1144,10 +1144,10 @@ protected static void writeMap(BufferAllocator allocator,
* @param allocator The BlockAllocator that can be used for allocating Arrow Buffers for fields which require conversion
* to Arrow Buff before being written.
* @param value The value to write.
* @note This method and its List complement violate the DRY mantra because ListWriter and StructWriter don't share
* a meaningful ancestor despite having identical methods. This requires us to either further wrap and abstract the writer
* or duplicate come code. In a future release we hope to have contributed a better option to Apache Arrow which allows
* us to simplify this method.
* @note This method and its List and struct complement violate the DRY mantra because
* ListWriter, StructWriter, and UnionMapWriter don't share a meaningful ancestor despite having identical methods.
* This requires us to either further wrap and abstract the writer or duplicate come code. In a future
* release we hope to have contributed a better option to Apache Arrow which allows us to simplify this method.
*/
@VisibleForTesting
protected static void writeMapValue(UnionMapWriter writer, Field field, BufferAllocator allocator, Object value)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -482,6 +483,11 @@ protected static class ExampleTable
FieldBuilder.newBuilder("outerlist", new ArrowType.List())
.addListField("innerList", Types.MinorType.VARCHAR.getType())
.build())
.addField(FieldBuilder.newBuilder("simplemap", new ArrowType.Map(false))
.addField("entries", Types.MinorType.STRUCT.getType(), false, Arrays.asList(
FieldBuilder.newBuilder("key", Types.MinorType.VARCHAR.getType(), false).build(),
FieldBuilder.newBuilder("value", Types.MinorType.INT.getType()).build()))
.build())
.addMetadata("partitionCols", "day,month,year")
.addMetadata("randomProp1", "randomPropVal1")
.addMetadata("randomProp2", "randomPropVal2").build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@
import org.apache.arrow.util.VisibleForTesting;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.impl.UnionMapWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter;
import org.apache.arrow.vector.holders.NullableBigIntHolder;
import org.apache.arrow.vector.holders.NullableBitHolder;
Expand Down Expand Up @@ -402,6 +404,25 @@ private FieldWriterFactory makeFactory(Field field, RowContext rowContext)
default:
throw new IllegalArgumentException("Unsupported type " + childType);
}
case MAP:
return (FieldVector vector, Extractor extractor, ConstraintProjector constraint) ->
(FieldWriter) (Object context, int rowNum) -> {
UnionMapWriter writer = ((MapVector) vector).getWriter();
writer.setPosition(rowNum);
writer.startMap();
writer.startEntry();
byte[] bytes = "chars".getBytes(Charsets.UTF_8);
try (ArrowBuf buf = vector.getAllocator().buffer(bytes.length)) {
buf.writeBytes(bytes);
writer.key().varChar("key").writeVarChar(0, (int) (buf.readableBytes()), buf);
}

writer.value().integer("value").writeInt(1001);
writer.endEntry();
writer.endMap();
((MapVector) vector).setNotNull(rowNum);
return true;
};
default:
throw new IllegalArgumentException("Unsupported type " + fieldType);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -689,8 +689,8 @@ protected ArrowType doTypedDeserialize(JsonParser jparser, DeserializationContex

private static final class MapSerDe
{
private static final String KEY = "key";
private static final String VALUE = "value";
//Arrow map types are represented as Map(false)<entries: Struct<key:keyType,value:valueType>>
private static final Boolean keysSortedFalse = false;

private static final class Serializer extends TypedSerializer<ArrowType>
{
Expand Down Expand Up @@ -718,7 +718,7 @@ private Deserializer()
protected ArrowType doTypedDeserialize(JsonParser jparser, DeserializationContext ctxt)
throws IOException
{
return new ArrowType.Map(false);
return new ArrowType.Map(keysSortedFalse);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -132,6 +133,11 @@ public void setUp()
FieldBuilder.newBuilder("outerlist", new ArrowType.List())
.addListField("innerList", Types.MinorType.VARCHAR.getType())
.build())
.addField(FieldBuilder.newBuilder("simplemap", new ArrowType.Map(false))
.addField("entries", Types.MinorType.STRUCT.getType(), false, Arrays.asList(
FieldBuilder.newBuilder("key", Types.MinorType.VARCHAR.getType(), false).build(),
FieldBuilder.newBuilder("value", Types.MinorType.INT.getType()).build()))
.build())
.addMetadata("partitionCols", "year,month,day")
.build();

Expand Down

0 comments on commit 1f7b303

Please sign in to comment.