Skip to content

Commit

Permalink
Athena neptune bugfix - Null value handling and Type casting (#414)
Browse files Browse the repository at this point in the history 
* bug fix to allow missing values from Amazon Neptune Property Graph

* added new test cases to check for for null values and typecasting

Co-authored-by: abhishekpradeepmishra <[email protected]>
Co-authored-by: abhishek.pradeep.mishra <[email protected]>
Co-authored-by: Sandeep Veldi <[email protected]>
Co-authored-by: Anthony Virtuoso <[email protected]>
  • Loading branch information
@sandeepveldi @abhishekpradeepmishra @avirtuos
5 people authored May 26, 2021
1 parent 3c917c1 commit 90b7fb5
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 23 deletions.
59 changes: 40 additions & 19 deletions athena-neptune/src/main/java/com/amazonaws/athena/connectors/neptune/TypeRowWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
*/
public final class TypeRowWriter
{
private TypeRowWriter()
private TypeRowWriter()
{
//Empty private constructor
}
Expand All @@ -58,64 +58,85 @@ public static void writeRowTemplate(RowWriterBuilder rowWriterBuilder, Field fie
case BIT:
rowWriterBuilder.withExtractor(field.getName(),
(BitExtractor) (Object context, NullableBitHolder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());

Boolean booleanValue = Boolean.parseBoolean(objValues.get(0).toString());
value.value = booleanValue ? 1 : 0;
value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
Boolean booleanValue = Boolean.parseBoolean(objValues.get(0).toString());
value.value = booleanValue ? 1 : 0;
value.isSet = 1;
}
});
break;

case VARCHAR:
rowWriterBuilder.withExtractor(field.getName(),
(VarCharExtractor) (Object context, NullableVarCharHolder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());

value.value = objValues.get(0).toString();
value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
value.value = objValues.get(0).toString();
value.isSet = 1;
}
});
break;

case INT:
rowWriterBuilder.withExtractor(field.getName(),
(IntExtractor) (Object context, NullableIntHolder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());
value.value = Integer.parseInt(objValues.get(0).toString());

value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
value.value = Integer.parseInt(objValues.get(0).toString());
value.isSet = 1;
}
});
break;

case BIGINT:
rowWriterBuilder.withExtractor(field.getName(),
(BigIntExtractor) (Object context, NullableBigIntHolder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());
(BigIntExtractor) (Object context, NullableBigIntHolder value) -> {
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());

value.value = Long.parseLong(objValues.get(0).toString());
});
break;
value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
value.value = Long.parseLong(objValues.get(0).toString());
value.isSet = 1;
}
});
break;

case FLOAT4:
rowWriterBuilder.withExtractor(field.getName(),
(Float4Extractor) (Object context, NullableFloat4Holder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());
value.value = (Float) (objValues.get(0));

value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
value.value = Float.parseFloat(objValues.get(0).toString());
value.isSet = 1;
}
});
break;

case FLOAT8:
rowWriterBuilder.withExtractor(field.getName(),
(Float8Extractor) (Object context, NullableFloat8Holder value) -> {
value.isSet = 1;
Map<Object, Object> obj = (Map<Object, Object>) context;
ArrayList<Object> objValues = (ArrayList) obj.get(field.getName());
value.value = (Double) (objValues.get(0));

value.isSet = 0;
if (objValues != null && objValues.get(0) != null) {
value.value = Double.parseDouble(objValues.get(0).toString());
value.isSet = 1;
}
});

break;
Expand Down
27 changes: 23 additions & 4 deletions ...ptune/src/test/java/com/amazonaws/athena/connectors/neptune/NeptuneRecordHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public void setUp() {
logger.info("{}: enter", testName.getMethodName());

schemaForRead = SchemaBuilder.newBuilder().addIntField("property1").addStringField("property2")
.addFloat8Field("property3").addBitField("property4").addBigIntField("property5")
.addFloat8Field("property3").addBitField("property4").addBigIntField("property5").addFloat4Field("property6")
.build();

allocator = new BlockAllocatorImpl();
Expand Down Expand Up @@ -170,20 +170,31 @@ private void buildGraphTraversal() {
vertex1.property("property3", 12.4);
vertex1.property("property4", true);
vertex1.property("property5", 12379878123l);
vertex1.property("property6", 15.45);

Vertex vertex2 = tinkerGraph.addVertex(T.label, "default");
vertex2.property("property1", 5);
vertex2.property("property2", "string2");
vertex2.property("property3", 20.4);
vertex2.property("property4", true);
vertex2.property("property5", 12379878123l);
vertex2.property("property6", 13.4523);

Vertex vertex3 = tinkerGraph.addVertex(T.label, "default");
vertex3.property("property1", 9);
vertex3.property("property2", "string3");
vertex3.property("property3", 15.4);
vertex3.property("property4", true);
vertex3.property("property5", 12379878123l);
vertex3.property("property6", 13.4523);

//add vertex with missing property values to check for nulls
tinkerGraph.addVertex(T.label, "default");

//add vertex to check for conversion from int to float,double.
Vertex vertex4 = tinkerGraph.addVertex(T.label, "default");
vertex4.property("property3", 15);
vertex4.property("property6", 13);

GraphTraversal<Vertex, Vertex> traversal = (GraphTraversal<Vertex, Vertex>) tinkerGraph.traversal().V();
when(graphTraversalSource.V()).thenReturn(traversal);
Expand Down Expand Up @@ -271,15 +282,24 @@ public void doReadRecordsNoSpill() throws Exception {
.newBuilder(allocator, Types.MinorType.INT.getType(), false, true).add(10).build());

invokeAndAssert(constraintsMap9, 2);

// Check for null values, expect all vertices to return as part of resultset
invokeAndAssert(new HashMap<>(), 5);

// Check for integer to float,double conversion
HashMap<String, ValueSet> constraintsMap10 = new HashMap<>();
constraintsMap10.put("property3", SortedRangeSet
.of(Range.greaterThan(allocator, Types.MinorType.FLOAT8.getType(), 13.2)));
constraintsMap10.put("property6", SortedRangeSet
.of(Range.greaterThan(allocator, Types.MinorType.FLOAT4.getType(), 11.11f)));
invokeAndAssert(constraintsMap10, 3);
}

/**
* Used to invoke each test condition and assert
*
* @param constraintMap Constraint Map for Gremlin Query
* @param expectedRecordCount Expected Row Count as per Gremlin Query Response
*
* @return A Gremlin Query Part equivalent to Contraint.
*/
private void invokeAndAssert(HashMap<String, ValueSet> constraintMap, Integer expectedRecordCount)
throws Exception {
Expand All @@ -303,7 +323,6 @@ private void invokeAndAssert(HashMap<String, ValueSet> constraintMap, Integer ex
assertTrue(response.getRecords().getRowCount() == expectedRecordCount);

logger.info("doReadRecordsNoSpill: {}", BlockUtils.rowToString(response.getRecords(), 0));

}

@Test
Expand Down

0 comments on commit 90b7fb5

Please sign in to comment.