From 1e3d8794ee62bbb261ca73c5987c164c390d8c4e Mon Sep 17 00:00:00 2001 From: forestmvey Date: Wed, 5 Jul 2023 15:21:27 -0700 Subject: [PATCH 1/5] Initial implementation for array support. Signed-off-by: forestmvey --- .../sql/analysis/ExpressionAnalyzer.java | 49 +++++++++++- .../analysis/SelectExpressionAnalyzer.java | 15 +++- .../sql/ast/AbstractNodeVisitor.java | 4 + .../ast/expression/ArrayQualifiedName.java | 29 +++++++ .../org/opensearch/sql/expression/DSL.java | 4 + .../sql/expression/ExpressionNodeVisitor.java | 4 + .../sql/expression/HighlightExpression.java | 2 +- .../IndexedReferenceExpression.java | 77 +++++++++++++++++++ .../sql/expression/ReferenceExpression.java | 11 +++ .../function/OpenSearchFunctions.java | 4 +- .../storage/bindingtuple/BindingTuple.java | 5 +- .../java/org/opensearch/sql/sql/NestedIT.java | 4 +- .../value/OpenSearchExprValueFactory.java | 9 +-- .../scan/OpenSearchIndexScanQueryBuilder.java | 5 ++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 5 ++ .../sql/sql/parser/AstExpressionBuilder.java | 10 ++- 16 files changed, 218 insertions(+), 19 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 601e3e00cc..f5c25b12cd 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -25,6 +25,7 @@ import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; @@ -57,6 +58,7 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.HighlightExpression; +import org.opensearch.sql.expression.IndexedReferenceExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; @@ -368,8 +370,28 @@ public Expression visitAllFields(AllFields node, AnalysisContext context) { public Expression visitQualifiedName(QualifiedName node, AnalysisContext context) { QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context); - // check for reserved words in the identifier - for (String part : node.getParts()) { + Expression reserved = checkForReservedIdentifier(node.getParts(), context, qualifierAnalyzer, node); + if (reserved != null) { + return reserved; + } + + return visitIdentifier(qualifierAnalyzer.unqualified(node), context); + } + + @Override + public Expression visitArrayQualifiedName(ArrayQualifiedName node, AnalysisContext context) { + QualifierAnalyzer qualifierAnalyzer = new QualifierAnalyzer(context); + + Expression reserved = checkForReservedIdentifier(node.getParts(), context, qualifierAnalyzer, node); + if (reserved != null) { + return reserved; + } + + return visitIndexedIdentifier(qualifierAnalyzer.unqualified(node), context, node.getIndex()); + } + + private Expression checkForReservedIdentifier(List parts, AnalysisContext context, QualifierAnalyzer qualifierAnalyzer, QualifiedName node) { + for (String part : parts) { for (TypeEnvironment typeEnv = context.peek(); typeEnv != null; typeEnv = typeEnv.getParent()) { @@ -384,7 +406,7 @@ public Expression visitQualifiedName(QualifiedName node, AnalysisContext context } } } - return visitIdentifier(qualifierAnalyzer.unqualified(node), context); + return null; } @Override @@ -422,8 +444,27 @@ private Expression visitIdentifier(String ident, AnalysisContext context) { } TypeEnvironment typeEnv = context.peek(); + var type = typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)); ReferenceExpression ref = DSL.ref(ident, - typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + + if (type.equals(ExprCoreType.ARRAY)) { + return new IndexedReferenceExpression(ref); + } + return ref; + } + + private Expression visitIndexedIdentifier(String ident, AnalysisContext context, int index) { + // ParseExpression will always override ReferenceExpression when ident conflicts + for (NamedExpression expr : context.getNamedParseExpressions()) { + if (expr.getNameOrAlias().equals(ident) && expr.getDelegated() instanceof ParseExpression) { + return expr.getDelegated(); + } + } + + TypeEnvironment typeEnv = context.peek(); + IndexedReferenceExpression ref = DSL.indexedRef(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)), index); return ref; } diff --git a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java index 734f37378b..e91f474813 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java @@ -23,9 +23,11 @@ import org.opensearch.sql.ast.expression.NestedAllTupleFields; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.IndexedReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; @@ -105,8 +107,17 @@ public List visitAllFields(AllFields node, AnalysisContext context) { TypeEnvironment environment = context.peek(); Map lookupAllFields = environment.lookupAllFields(Namespace.FIELD_NAME); - return lookupAllFields.entrySet().stream().map(entry -> DSL.named(entry.getKey(), - new ReferenceExpression(entry.getKey(), entry.getValue()))).collect(Collectors.toList()); + return lookupAllFields.entrySet().stream().map(entry -> + { + ReferenceExpression ref = new ReferenceExpression(entry.getKey(), entry.getValue()); + if (entry.getValue().equals(ExprCoreType.ARRAY)) { + return DSL.named(entry.getKey(), + new IndexedReferenceExpression(ref)); + } else { + return DSL.named(entry.getKey(), ref); + } + } + ).collect(Collectors.toList()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index f02bc07ccc..adb666669e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -11,6 +11,7 @@ import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.AttributeList; import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; @@ -194,6 +195,9 @@ public T visitField(Field node, C context) { public T visitQualifiedName(QualifiedName node, C context) { return visitChildren(node, context); } + public T visitArrayQualifiedName(ArrayQualifiedName node, C context) { + return visitChildren(node, context); + } public T visitRename(Rename node, C context) { return visitChildren(node, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java new file mode 100644 index 0000000000..1c791bce96 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.expression; + + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +@Getter +@EqualsAndHashCode(callSuper = false) +public class ArrayQualifiedName extends QualifiedName { + + private int index; + + public ArrayQualifiedName(String name, int index) { + super(name); + this.index = index; + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitArrayQualifiedName(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 3f1897e483..f9900435d6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -87,6 +87,10 @@ public static ReferenceExpression ref(String ref, ExprType type) { return new ReferenceExpression(ref, type); } + public static IndexedReferenceExpression indexedRef(String ref, ExprType type, int index) { + return new IndexedReferenceExpression(ref, type, index); + } + /** * Wrap a named expression if not yet. The intent is that different languages may use * Alias or not when building AST. This caused either named or unnamed expression diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index e3d4e38674..a4b0970f02 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -64,6 +64,10 @@ public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } + public T visitIndexedReference(IndexedReferenceExpression node, C context) { + return visitNode(node, context); + } + public T visitParse(ParseExpression node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java index 804c38a6f7..a89623f523 100644 --- a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java @@ -51,7 +51,7 @@ public ExprValue valueOf(Environment valueEnv) { if (this.type == ExprCoreType.ARRAY) { refName += "." + StringUtils.unquoteText(getHighlightField().toString()); } - ExprValue value = valueEnv.resolve(DSL.ref(refName, ExprCoreType.STRING)); + ExprValue value = valueEnv.resolve(new IndexedReferenceExpression(DSL.ref(refName, ExprCoreType.STRING))); // In the event of multiple returned highlights and wildcard being // used in conjunction with other highlight calls, we need to ensure diff --git a/core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java new file mode 100644 index 0000000000..dd9a20f0cc --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression; + +import static org.opensearch.sql.utils.ExpressionUtils.PATH_SEP; + +import java.util.Arrays; +import java.util.List; +import java.util.OptionalInt; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.env.Environment; + +@EqualsAndHashCode +public class IndexedReferenceExpression extends ReferenceExpression { + @Getter + private final OptionalInt index; + @Getter + private final ExprType type; + @Getter + private final List paths; + public IndexedReferenceExpression(String ref, ExprType type, int index) { + super(ref, type); + this.index = OptionalInt.of(index); + this.type = type; + this.paths = Arrays.asList(ref.split("\\.")); + } + + public IndexedReferenceExpression(ReferenceExpression ref) { + super(ref.toString(), ref.type()); + this.index = OptionalInt.empty(); + this.type = ref.type(); + this.paths = Arrays.asList(ref.toString().split("\\.")); + } + + @Override + public ExprValue valueOf(Environment env) { + return env.resolve(this); + } + + @Override + public ExprType type() { + return type; + } + + @Override + public T accept(ExpressionNodeVisitor visitor, C context) { + return visitor.visitIndexedReference(this, context); + } + + public ExprValue resolve(ExprTupleValue value) { + return resolve(value, paths); + } + + private ExprValue resolve(ExprValue value, List paths) { + ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, paths)); + // For array types only first index currently supported. + if (!index.isEmpty() && wholePathValue.type().equals(ExprCoreType.ARRAY)) { + wholePathValue = wholePathValue.collectionValue().get(index.getAsInt()); + } + + if (!wholePathValue.isMissing() || paths.size() == 1) { + return wholePathValue; + } else { + return resolve(value.keyValue(paths.get(0)), paths.subList(1, paths.size())); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java index 3c5b2af23c..4bac2fb14d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java @@ -106,6 +106,9 @@ private ExprValue resolve(ExprValue value, List paths) { if (value.type().equals(ExprCoreType.ARRAY)) { wholePathValue = value.collectionValue().get(0).keyValue(paths.get(0)); } + if (wholePathValue.type().equals(ExprCoreType.ARRAY)) { + return getFirstValueOfCollection(wholePathValue); + } if (!wholePathValue.isMissing() || paths.size() == 1) { return wholePathValue; @@ -113,4 +116,12 @@ private ExprValue resolve(ExprValue value, List paths) { return resolve(value.keyValue(paths.get(0)), paths.subList(1, paths.size())); } } + + private ExprValue getFirstValueOfCollection(ExprValue value) { + ExprValue collectionVal = value; + while(collectionVal.type().equals(ExprCoreType.ARRAY)) { + collectionVal = collectionVal.collectionValue().get(0); + } + return collectionVal; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index c5fcb010f5..9d48499498 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -17,7 +17,9 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.IndexedReferenceExpression; import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @UtilityClass @@ -106,7 +108,7 @@ public Pair resolve( new FunctionExpression(BuiltinFunctionName.NESTED.getName(), arguments) { @Override public ExprValue valueOf(Environment valueEnv) { - return valueEnv.resolve(getArguments().get(0)); + return valueEnv.resolve(new IndexedReferenceExpression((ReferenceExpression) getArguments().get(0))); } @Override diff --git a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java index 51a0348116..ff09345ae4 100644 --- a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java +++ b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java @@ -10,6 +10,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.IndexedReferenceExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @@ -30,7 +31,9 @@ public ExprValue resolve(ReferenceExpression ref) { */ @Override public ExprValue resolve(Expression var) { - if (var instanceof ReferenceExpression) { + if (var instanceof IndexedReferenceExpression) { + return resolve(((IndexedReferenceExpression) var)); + } else if (var instanceof ReferenceExpression) { return resolve(((ReferenceExpression) var)); } else { throw new ExpressionEvaluationException(String.format("can resolve expression: %s", var)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index d3230188b7..e09e9a4f53 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -254,7 +254,7 @@ public void nested_function_and_field_with_order_by_clause() { rows("a", 4), rows("b", 2), rows("c", 3), - rows("zz", new JSONArray(List.of(3, 4)))); + rows("zz", 3)); } // Nested function in GROUP BY clause is not yet implemented for JDBC format. This test ensures @@ -535,7 +535,7 @@ public void nested_function_all_subfields_and_non_nested_field() { rows("g", 1, "c", 3), rows("h", 4, "c", 4), rows("i", 5, "a", 4), - rows("zz", 6, "zz", new JSONArray(List.of(3, 4)))); + rows("zz", 6, "zz", 3)); } @Test diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 22a43d3444..55c69eebe9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -190,7 +190,8 @@ private ExprValue parse( ExprType type = fieldType.get(); if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) - || content.isArray()) { + || content.objectValue() instanceof ArrayNode + ) { return parseArray(content, field, type, supportArrays); } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) || type == STRUCT) { @@ -330,12 +331,6 @@ private ExprValue parseArray( // ARRAY is mapped to nested but can take the json structure of an Object. if (content.objectValue() instanceof ObjectNode) { result.add(parseStruct(content, prefix, supportArrays)); - // non-object type arrays are only supported when parsing inner_hits of OS response. - } else if ( - !(type instanceof OpenSearchDataType - && ((OpenSearchDataType) type).getExprType().equals(ARRAY)) - && !supportArrays) { - return parseInnerArrayValue(content.array().next(), prefix, type, supportArrays); } else { content.array().forEachRemaining(v -> { result.add(parseInnerArrayValue(v, prefix, type, supportArrays)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java index 590272a9f1..abc6042232 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java @@ -19,6 +19,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.IndexedReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.OpenSearchFunctions; @@ -156,6 +157,10 @@ public static List findReferenceExpression(NamedExpression public Object visitReference(ReferenceExpression node, Object context) { return results.add(node); } + @Override + public Object visitIndexedReference(IndexedReferenceExpression node, Object context) { + return results.add(node); + } }, null); return results; } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index e68edbbc58..fe90c97d9a 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -303,6 +303,7 @@ expressions expressionAtom : constant #constantExpressionAtom | columnName #fullColumnNameExpressionAtom + | arrayColumnName #arrayColumnNameExpressionAtom | functionCall #functionCallExpressionAtom | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom | left=expressionAtom @@ -814,6 +815,10 @@ columnName : qualifiedName ; +arrayColumnName + : qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS + ; + allTupleFields : path=qualifiedName DOT STAR ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 7279553106..6a1ce01f62 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -82,6 +82,7 @@ import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.DataType; @@ -103,9 +104,9 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AndExpressionContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ArrayColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IdentContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IntervalLiteralContext; @@ -129,6 +130,13 @@ public UnresolvedExpression visitColumnName(ColumnNameContext ctx) { return visit(ctx.qualifiedName()); } + @Override + public UnresolvedExpression visitArrayColumnName(ArrayColumnNameContext ctx) { + UnresolvedExpression qualifiedName = visit(ctx.qualifiedName()); + return new ArrayQualifiedName( + qualifiedName.toString(), Integer.parseInt(ctx.decimalLiteral().getText())); + } + @Override public UnresolvedExpression visitIdent(IdentContext ctx) { return visitIdentifiers(Collections.singletonList(ctx)); From 05048ccee5bc7ac8956dff16e655fb5484af7da3 Mon Sep 17 00:00:00 2001 From: forestmvey Date: Thu, 6 Jul 2023 08:27:36 -0700 Subject: [PATCH 2/5] Adding support for returning arrays as a whole. Signed-off-by: forestmvey --- .../sql/analysis/ExpressionAnalyzer.java | 18 ++++++---- .../analysis/SelectExpressionAnalyzer.java | 4 +-- .../ast/expression/ArrayQualifiedName.java | 11 ++++-- ...ion.java => ArrayReferenceExpression.java} | 36 +++++++++++++++---- .../org/opensearch/sql/expression/DSL.java | 8 +++-- .../sql/expression/ExpressionNodeVisitor.java | 2 +- .../sql/expression/HighlightExpression.java | 2 +- .../function/OpenSearchFunctions.java | 4 +-- .../storage/bindingtuple/BindingTuple.java | 6 ++-- .../scan/OpenSearchIndexScanQueryBuilder.java | 4 +-- sql/src/main/antlr/OpenSearchSQLParser.g4 | 3 +- .../sql/sql/parser/AstExpressionBuilder.java | 8 +++-- 12 files changed, 74 insertions(+), 32 deletions(-) rename core/src/main/java/org/opensearch/sql/expression/{IndexedReferenceExpression.java => ArrayReferenceExpression.java} (60%) diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index f5c25b12cd..0faaac0702 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.stream.Collectors; import lombok.Getter; import org.opensearch.sql.analysis.symbol.Namespace; @@ -50,7 +51,6 @@ import org.opensearch.sql.ast.expression.When; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; -import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -58,7 +58,7 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.HighlightExpression; -import org.opensearch.sql.expression.IndexedReferenceExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; @@ -387,7 +387,7 @@ public Expression visitArrayQualifiedName(ArrayQualifiedName node, AnalysisConte return reserved; } - return visitIndexedIdentifier(qualifierAnalyzer.unqualified(node), context, node.getIndex()); + return visitArrayIdentifier(qualifierAnalyzer.unqualified(node), context, node.getIndex()); } private Expression checkForReservedIdentifier(List parts, AnalysisContext context, QualifierAnalyzer qualifierAnalyzer, QualifiedName node) { @@ -449,12 +449,12 @@ private Expression visitIdentifier(String ident, AnalysisContext context) { typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); if (type.equals(ExprCoreType.ARRAY)) { - return new IndexedReferenceExpression(ref); + return new ArrayReferenceExpression(ref); } return ref; } - private Expression visitIndexedIdentifier(String ident, AnalysisContext context, int index) { + private Expression visitArrayIdentifier(String ident, AnalysisContext context, OptionalInt index) { // ParseExpression will always override ReferenceExpression when ident conflicts for (NamedExpression expr : context.getNamedParseExpressions()) { if (expr.getNameOrAlias().equals(ident) && expr.getDelegated() instanceof ParseExpression) { @@ -463,8 +463,12 @@ private Expression visitIndexedIdentifier(String ident, AnalysisContext context, } TypeEnvironment typeEnv = context.peek(); - IndexedReferenceExpression ref = DSL.indexedRef(ident, - typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)), index); + ArrayReferenceExpression ref = index.isEmpty() ? + DSL.indexedRef(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))) + : + DSL.indexedRef(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)), index.getAsInt()); return ref; } diff --git a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java index e91f474813..7798aaf198 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/SelectExpressionAnalyzer.java @@ -27,7 +27,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.IndexedReferenceExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; @@ -112,7 +112,7 @@ public List visitAllFields(AllFields node, ReferenceExpression ref = new ReferenceExpression(entry.getKey(), entry.getValue()); if (entry.getValue().equals(ExprCoreType.ARRAY)) { return DSL.named(entry.getKey(), - new IndexedReferenceExpression(ref)); + new ArrayReferenceExpression(ref)); } else { return DSL.named(entry.getKey(), ref); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java index 1c791bce96..8683fccf7f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java @@ -11,15 +11,22 @@ import lombok.Getter; import org.opensearch.sql.ast.AbstractNodeVisitor; +import java.util.OptionalInt; + @Getter @EqualsAndHashCode(callSuper = false) public class ArrayQualifiedName extends QualifiedName { - private int index; + private final OptionalInt index; public ArrayQualifiedName(String name, int index) { super(name); - this.index = index; + this.index = OptionalInt.of(index); + } + + public ArrayQualifiedName(String name) { + super(name); + this.index = OptionalInt.empty(); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java similarity index 60% rename from core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java rename to core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java index dd9a20f0cc..0a18ce6cdd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/IndexedReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java @@ -16,26 +16,34 @@ import lombok.Getter; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.env.Environment; @EqualsAndHashCode -public class IndexedReferenceExpression extends ReferenceExpression { +public class ArrayReferenceExpression extends ReferenceExpression { @Getter private final OptionalInt index; @Getter private final ExprType type; @Getter private final List paths; - public IndexedReferenceExpression(String ref, ExprType type, int index) { + public ArrayReferenceExpression(String ref, ExprType type, int index) { super(ref, type); this.index = OptionalInt.of(index); this.type = type; this.paths = Arrays.asList(ref.split("\\.")); } - public IndexedReferenceExpression(ReferenceExpression ref) { + public ArrayReferenceExpression(String ref, ExprType type) { + super(ref, type); + this.index = OptionalInt.empty(); + this.type = type; + this.paths = Arrays.asList(ref.split("\\.")); + } + + public ArrayReferenceExpression(ReferenceExpression ref) { super(ref.toString(), ref.type()); this.index = OptionalInt.empty(); this.type = ref.type(); @@ -54,7 +62,7 @@ public ExprType type() { @Override public T accept(ExpressionNodeVisitor visitor, C context) { - return visitor.visitIndexedReference(this, context); + return visitor.visitArrayReference(this, context); } public ExprValue resolve(ExprTupleValue value) { @@ -63,9 +71,11 @@ public ExprValue resolve(ExprTupleValue value) { private ExprValue resolve(ExprValue value, List paths) { ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, paths)); - // For array types only first index currently supported. - if (!index.isEmpty() && wholePathValue.type().equals(ExprCoreType.ARRAY)) { - wholePathValue = wholePathValue.collectionValue().get(index.getAsInt()); + + if (!index.isEmpty()) { + wholePathValue = getIndexValueOfCollection(wholePathValue); + } else if (paths.size() == 1 && !wholePathValue.type().equals(ExprCoreType.ARRAY)) { + wholePathValue = ExprValueUtils.missingValue(); } if (!wholePathValue.isMissing() || paths.size() == 1) { @@ -74,4 +84,16 @@ private ExprValue resolve(ExprValue value, List paths) { return resolve(value.keyValue(paths.get(0)), paths.subList(1, paths.size())); } } + + private ExprValue getIndexValueOfCollection(ExprValue value) { + ExprValue collectionVal = value; + for (OptionalInt blah : List.of(index)) { + if (collectionVal.type().equals(ExprCoreType.ARRAY)) { + collectionVal = collectionVal.collectionValue().get(blah.getAsInt()); + } else { + return ExprValueUtils.missingValue(); + } + } + return collectionVal; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index f9900435d6..5aa786fd76 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -87,8 +87,12 @@ public static ReferenceExpression ref(String ref, ExprType type) { return new ReferenceExpression(ref, type); } - public static IndexedReferenceExpression indexedRef(String ref, ExprType type, int index) { - return new IndexedReferenceExpression(ref, type, index); + public static ArrayReferenceExpression indexedRef(String ref, ExprType type, int index) { + return new ArrayReferenceExpression(ref, type, index); + } + + public static ArrayReferenceExpression indexedRef(String ref, ExprType type) { + return new ArrayReferenceExpression(ref, type); } /** diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index a4b0970f02..1999cef44f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -64,7 +64,7 @@ public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } - public T visitIndexedReference(IndexedReferenceExpression node, C context) { + public T visitArrayReference(ArrayReferenceExpression node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java index a89623f523..0557ac727c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java @@ -51,7 +51,7 @@ public ExprValue valueOf(Environment valueEnv) { if (this.type == ExprCoreType.ARRAY) { refName += "." + StringUtils.unquoteText(getHighlightField().toString()); } - ExprValue value = valueEnv.resolve(new IndexedReferenceExpression(DSL.ref(refName, ExprCoreType.STRING))); + ExprValue value = valueEnv.resolve(new ArrayReferenceExpression(DSL.ref(refName, ExprCoreType.STRING))); // In the event of multiple returned highlights and wildcard being // used in conjunction with other highlight calls, we need to ensure diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index 9d48499498..7aa7f7df4b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -17,7 +17,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.IndexedReferenceExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @@ -108,7 +108,7 @@ public Pair resolve( new FunctionExpression(BuiltinFunctionName.NESTED.getName(), arguments) { @Override public ExprValue valueOf(Environment valueEnv) { - return valueEnv.resolve(new IndexedReferenceExpression((ReferenceExpression) getArguments().get(0))); + return valueEnv.resolve(new ArrayReferenceExpression((ReferenceExpression) getArguments().get(0))); } @Override diff --git a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java index ff09345ae4..00890fe2d8 100644 --- a/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java +++ b/core/src/main/java/org/opensearch/sql/storage/bindingtuple/BindingTuple.java @@ -10,7 +10,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.IndexedReferenceExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; @@ -31,8 +31,8 @@ public ExprValue resolve(ReferenceExpression ref) { */ @Override public ExprValue resolve(Expression var) { - if (var instanceof IndexedReferenceExpression) { - return resolve(((IndexedReferenceExpression) var)); + if (var instanceof ArrayReferenceExpression) { + return resolve(((ArrayReferenceExpression) var)); } else if (var instanceof ReferenceExpression) { return resolve(((ReferenceExpression) var)); } else { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java index abc6042232..bd3023b340 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanQueryBuilder.java @@ -19,7 +19,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.IndexedReferenceExpression; +import org.opensearch.sql.expression.ArrayReferenceExpression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.OpenSearchFunctions; @@ -158,7 +158,7 @@ public Object visitReference(ReferenceExpression node, Object context) { return results.add(node); } @Override - public Object visitIndexedReference(IndexedReferenceExpression node, Object context) { + public Object visitArrayReference(ArrayReferenceExpression node, Object context) { return results.add(node); } }, null); diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index fe90c97d9a..627fbe819e 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -816,7 +816,8 @@ columnName ; arrayColumnName - : qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS + : qualifiedName LT_SQR_PRTHS COLON_SYMB RT_SQR_PRTHS + | qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS ; allTupleFields diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 6a1ce01f62..df6c32f47f 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -133,8 +133,12 @@ public UnresolvedExpression visitColumnName(ColumnNameContext ctx) { @Override public UnresolvedExpression visitArrayColumnName(ArrayColumnNameContext ctx) { UnresolvedExpression qualifiedName = visit(ctx.qualifiedName()); - return new ArrayQualifiedName( - qualifiedName.toString(), Integer.parseInt(ctx.decimalLiteral().getText())); + if (ctx.decimalLiteral() == null) { + return new ArrayQualifiedName(qualifiedName.toString()); + } else { + return new ArrayQualifiedName( + qualifiedName.toString(), Integer.parseInt(ctx.decimalLiteral().getText())); + } } @Override From 95e2c82ad94c4219168d9d5d36ad5e39c6017f3a Mon Sep 17 00:00:00 2001 From: forestmvey Date: Thu, 6 Jul 2023 09:39:33 -0700 Subject: [PATCH 3/5] Adding comment. Signed-off-by: forestmvey --- .../opensearch/sql/expression/ArrayReferenceExpression.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java index 0a18ce6cdd..afb7c26c50 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java @@ -24,7 +24,7 @@ @EqualsAndHashCode public class ArrayReferenceExpression extends ReferenceExpression { @Getter - private final OptionalInt index; + private final OptionalInt index; // Should be a list of indexes to support multiple nesting levels @Getter private final ExprType type; @Getter @@ -87,9 +87,9 @@ private ExprValue resolve(ExprValue value, List paths) { private ExprValue getIndexValueOfCollection(ExprValue value) { ExprValue collectionVal = value; - for (OptionalInt blah : List.of(index)) { + for (OptionalInt currentIndex : List.of(index)) { if (collectionVal.type().equals(ExprCoreType.ARRAY)) { - collectionVal = collectionVal.collectionValue().get(blah.getAsInt()); + collectionVal = collectionVal.collectionValue().get(currentIndex.getAsInt()); } else { return ExprValueUtils.missingValue(); } From 579d44a30824b4d14f91e54e52378d0e30bd9528 Mon Sep 17 00:00:00 2001 From: forestmvey Date: Fri, 7 Jul 2023 15:47:42 -0700 Subject: [PATCH 4/5] Added indexing for each part of array column name. Signed-off-by: forestmvey --- .../sql/common/utils/StringUtils.java | 4 ++ .../sql/analysis/ExpressionAnalyzer.java | 17 +++--- .../sql/analysis/QualifierAnalyzer.java | 20 ++++++- .../ast/expression/ArrayQualifiedName.java | 16 +++-- .../expression/ArrayReferenceExpression.java | 42 ++++++------- .../org/opensearch/sql/expression/DSL.java | 8 --- sql/src/main/antlr/OpenSearchSQLParser.g4 | 22 +++++-- .../sql/sql/parser/AstExpressionBuilder.java | 60 +++++++++++++++---- 8 files changed, 121 insertions(+), 68 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index bd3a5a9779..53e75178a7 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -105,4 +105,8 @@ public static String format(final String format, Object... args) { private static boolean isQuoted(String text, String mark) { return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); } + + public static String removeParenthesis(String qualifier) { + return qualifier.replaceAll("\\[.+\\]", ""); + } } diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 0faaac0702..488f7f4f33 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -20,6 +20,7 @@ import java.util.OptionalInt; import java.util.stream.Collectors; import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -51,6 +52,7 @@ import org.opensearch.sql.ast.expression.When; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -387,7 +389,7 @@ public Expression visitArrayQualifiedName(ArrayQualifiedName node, AnalysisConte return reserved; } - return visitArrayIdentifier(qualifierAnalyzer.unqualified(node), context, node.getIndex()); + return visitArrayIdentifier(qualifierAnalyzer.unqualified(node), context, node.getPartsAndIndexes()); } private Expression checkForReservedIdentifier(List parts, AnalysisContext context, QualifierAnalyzer qualifierAnalyzer, QualifiedName node) { @@ -454,7 +456,8 @@ private Expression visitIdentifier(String ident, AnalysisContext context) { return ref; } - private Expression visitArrayIdentifier(String ident, AnalysisContext context, OptionalInt index) { + private Expression visitArrayIdentifier(String ident, AnalysisContext context, + List> partsAndIndexes) { // ParseExpression will always override ReferenceExpression when ident conflicts for (NamedExpression expr : context.getNamedParseExpressions()) { if (expr.getNameOrAlias().equals(ident) && expr.getDelegated() instanceof ParseExpression) { @@ -463,13 +466,7 @@ private Expression visitArrayIdentifier(String ident, AnalysisContext context, O } TypeEnvironment typeEnv = context.peek(); - ArrayReferenceExpression ref = index.isEmpty() ? - DSL.indexedRef(ident, - typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))) - : - DSL.indexedRef(ident, - typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident)), index.getAsInt()); - - return ref; + return new ArrayReferenceExpression(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, StringUtils.removeParenthesis(ident))), partsAndIndexes); } } diff --git a/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java index d1e31d0079..d75603be13 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/QualifierAnalyzer.java @@ -11,8 +11,10 @@ import lombok.RequiredArgsConstructor; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; +import org.opensearch.sql.ast.expression.ArrayQualifiedName; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; /** @@ -38,6 +40,10 @@ public String unqualified(QualifiedName fullName) { return isQualifierIndexOrAlias(fullName) ? fullName.rest().toString() : fullName.toString(); } + public String unqualified(ArrayQualifiedName fullName) { + return isQualifierIndexOrAlias(fullName) ? fullName.rest().toString() : fullName.toString(); + } + private boolean isQualifierIndexOrAlias(QualifiedName fullName) { Optional qualifier = fullName.first(); if (qualifier.isPresent()) { @@ -50,10 +56,22 @@ private boolean isQualifierIndexOrAlias(QualifiedName fullName) { return false; } + private boolean isQualifierIndexOrAlias(ArrayQualifiedName fullName) { + Optional qualifier = fullName.first(); + if (qualifier.isPresent()) { + if (isFieldName(qualifier.get())) { + return false; + } + resolveQualifierSymbol(fullName, qualifier.get()); + return true; + } + return false; + } + private boolean isFieldName(String qualifier) { try { // Resolve the qualifier in Namespace.FIELD_NAME - context.peek().resolve(new Symbol(Namespace.FIELD_NAME, qualifier)); + context.peek().resolve(new Symbol(Namespace.FIELD_NAME, StringUtils.removeParenthesis(qualifier))); return true; } catch (SemanticCheckException e2) { return false; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java index 8683fccf7f..910cfdc5be 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ArrayQualifiedName.java @@ -9,24 +9,22 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; +import java.util.List; import java.util.OptionalInt; +import java.util.stream.Collectors; @Getter @EqualsAndHashCode(callSuper = false) public class ArrayQualifiedName extends QualifiedName { - private final OptionalInt index; + private final List> partsAndIndexes; - public ArrayQualifiedName(String name, int index) { - super(name); - this.index = OptionalInt.of(index); - } - - public ArrayQualifiedName(String name) { - super(name); - this.index = OptionalInt.empty(); + public ArrayQualifiedName(List> parts) { + super(parts.stream().map(p -> p.getLeft()).collect(Collectors.toList())); + this.partsAndIndexes = parts; } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java index afb7c26c50..2da8e6a5dd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java @@ -11,9 +11,12 @@ import java.util.Arrays; import java.util.List; import java.util.OptionalInt; +import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; @@ -24,30 +27,20 @@ @EqualsAndHashCode public class ArrayReferenceExpression extends ReferenceExpression { @Getter - private final OptionalInt index; // Should be a list of indexes to support multiple nesting levels + private final List> partsAndIndexes; @Getter private final ExprType type; - @Getter - private final List paths; - public ArrayReferenceExpression(String ref, ExprType type, int index) { - super(ref, type); - this.index = OptionalInt.of(index); - this.type = type; - this.paths = Arrays.asList(ref.split("\\.")); - } - - public ArrayReferenceExpression(String ref, ExprType type) { + public ArrayReferenceExpression(String ref, ExprType type, List> partsAndIndexes) { super(ref, type); - this.index = OptionalInt.empty(); + this.partsAndIndexes = partsAndIndexes; this.type = type; - this.paths = Arrays.asList(ref.split("\\.")); } public ArrayReferenceExpression(ReferenceExpression ref) { super(ref.toString(), ref.type()); - this.index = OptionalInt.empty(); + this.partsAndIndexes = Arrays.stream(ref.toString().split("\\.")).map(e -> Pair.of(e, OptionalInt.empty())).collect( + Collectors.toList()); this.type = ref.type(); - this.paths = Arrays.asList(ref.toString().split("\\.")); } @Override @@ -66,13 +59,15 @@ public T accept(ExpressionNodeVisitor visitor, C context) { } public ExprValue resolve(ExprTupleValue value) { - return resolve(value, paths); + return resolve(value, partsAndIndexes); } - private ExprValue resolve(ExprValue value, List paths) { - ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, paths)); + private ExprValue resolve(ExprValue value, List> paths) { + List pathsWithoutParenthesis = + paths.stream().map(p -> StringUtils.removeParenthesis(p.getLeft())).collect(Collectors.toList()); + ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, pathsWithoutParenthesis)); - if (!index.isEmpty()) { + if (!paths.get(0).getRight().isEmpty()) { wholePathValue = getIndexValueOfCollection(wholePathValue); } else if (paths.size() == 1 && !wholePathValue.type().equals(ExprCoreType.ARRAY)) { wholePathValue = ExprValueUtils.missingValue(); @@ -81,19 +76,20 @@ private ExprValue resolve(ExprValue value, List paths) { if (!wholePathValue.isMissing() || paths.size() == 1) { return wholePathValue; } else { - return resolve(value.keyValue(paths.get(0)), paths.subList(1, paths.size())); + return resolve(value.keyValue(pathsWithoutParenthesis.get(0) + ), paths.subList(1, paths.size())); } } private ExprValue getIndexValueOfCollection(ExprValue value) { ExprValue collectionVal = value; - for (OptionalInt currentIndex : List.of(index)) { +// for (OptionalInt currentIndex : List.of(index)) { if (collectionVal.type().equals(ExprCoreType.ARRAY)) { - collectionVal = collectionVal.collectionValue().get(currentIndex.getAsInt()); +// collectionVal = collectionVal.collectionValue().get(currentIndex.getAsInt()); } else { return ExprValueUtils.missingValue(); } - } +// } return collectionVal; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 5aa786fd76..3f1897e483 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -87,14 +87,6 @@ public static ReferenceExpression ref(String ref, ExprType type) { return new ReferenceExpression(ref, type); } - public static ArrayReferenceExpression indexedRef(String ref, ExprType type, int index) { - return new ArrayReferenceExpression(ref, type, index); - } - - public static ArrayReferenceExpression indexedRef(String ref, ExprType type) { - return new ArrayReferenceExpression(ref, type); - } - /** * Wrap a named expression if not yet. The intent is that different languages may use * Alias or not when building AST. This caused either named or unnamed expression diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 627fbe819e..9fc72b19f5 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -303,7 +303,7 @@ expressions expressionAtom : constant #constantExpressionAtom | columnName #fullColumnNameExpressionAtom - | arrayColumnName #arrayColumnNameExpressionAtom +// | arrayColumnName #arrayColumnNameExpressionAtom | functionCall #functionCallExpressionAtom | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom | left=expressionAtom @@ -815,10 +815,11 @@ columnName : qualifiedName ; -arrayColumnName - : qualifiedName LT_SQR_PRTHS COLON_SYMB RT_SQR_PRTHS - | qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS - ; +//arrayColumnName +// : arrayQualifiedName +// | qualifiedName LT_SQR_PRTHS COLON_SYMB RT_SQR_PRTHS (arrayColumnName | columnName) * +// | qualifiedName LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS (arrayColumnName | columnName) * +// ; allTupleFields : path=qualifiedName DOT STAR @@ -828,12 +829,21 @@ alias : ident ; +//arrayQualifiedName +// : (ident | indexedIdentifier) (DOT (ident | indexedIdentifier))* +//// (( LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS ) | (DOT ident ( LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS )* ))+ +// ; + +//indexedIdentifier +// : ident LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS +// ; + qualifiedName : ident (DOT ident)* ; ident - : DOT? ID + : DOT? ID (LT_SQR_PRTHS decimalLiteral RT_SQR_PRTHS)? | BACKTICK_QUOTE_ID | keywordsCanBeId | scalarFunctionName diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index df6c32f47f..d1f0c9ee1c 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -69,11 +69,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import java.util.stream.Collectors; import org.antlr.v4.runtime.RuleContext; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -106,7 +109,6 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AndExpressionContext; -import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ArrayColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IdentContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IntervalLiteralContext; @@ -130,16 +132,26 @@ public UnresolvedExpression visitColumnName(ColumnNameContext ctx) { return visit(ctx.qualifiedName()); } - @Override - public UnresolvedExpression visitArrayColumnName(ArrayColumnNameContext ctx) { - UnresolvedExpression qualifiedName = visit(ctx.qualifiedName()); - if (ctx.decimalLiteral() == null) { - return new ArrayQualifiedName(qualifiedName.toString()); - } else { - return new ArrayQualifiedName( - qualifiedName.toString(), Integer.parseInt(ctx.decimalLiteral().getText())); - } - } +// @Override +// public UnresolvedExpression visitArrayColumnName(ArrayColumnNameContext ctx) { +//// return new QualifiedName( +//// identifiers.stream() +//// .map(RuleContext::getText) +//// .map(StringUtils::unquoteIdentifier) +//// .collect(Collectors.toList())); +// +// var blah = ctx.arrayQualifiedName().indexedIdentifier(); +// var hmm = ctx.arrayQualifiedName().ident(); +// +// +// UnresolvedExpression qualifiedName = visit(ctx.arrayQualifiedName()); +//// if (ctx.arrayQualifiedName().decimalLiteral() == null) { +// return new ArrayQualifiedName(qualifiedName.toString()); +//// } else { +// return new ArrayQualifiedName( +// qualifiedName.toString(), Integer.parseInt(ctx.arrayQualifiedName().decimalLiteral().toString())); +//// } +// } @Override public UnresolvedExpression visitIdent(IdentContext ctx) { @@ -544,6 +556,32 @@ public UnresolvedExpression visitExtractFunctionCall(ExtractFunctionCallContext private QualifiedName visitIdentifiers(List identifiers) { + + List> parts = new ArrayList<>(); + boolean supportsArrays = false; + for(var blah : identifiers) { + if (blah.decimalLiteral() != null) { + parts.add( + Pair.of( + StringUtils.unquoteIdentifier(blah.getText()), + OptionalInt.of(Integer.parseInt(blah.decimalLiteral().getText())) + ) + ); + supportsArrays = true; + } else { + parts.add( + Pair.of( + StringUtils.unquoteIdentifier(blah.getText()), + OptionalInt.empty() + ) + ); + } + } + + if (supportsArrays) { + return new ArrayQualifiedName(parts); + } + return new QualifiedName( identifiers.stream() .map(RuleContext::getText) From 41d7bf5e3ba1d4a3d907cab887a46eca06fe9551 Mon Sep 17 00:00:00 2001 From: forestmvey Date: Wed, 12 Jul 2023 08:20:09 -0700 Subject: [PATCH 5/5] Base array functionality working with IT tests. Signed-off-by: forestmvey --- .../sql/common/utils/StringUtils.java | 2 +- .../expression/ArrayReferenceExpression.java | 37 ++++++----- .../sql/legacy/SQLIntegTestCase.java | 7 ++- .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../java/org/opensearch/sql/sql/ArraysIT.java | 62 +++++++++++++++++++ integ-test/src/test/resources/arrays.json | 2 + .../indexDefinitions/arrays_mapping.json | 23 +++++++ 8 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java create mode 100644 integ-test/src/test/resources/arrays.json create mode 100644 integ-test/src/test/resources/indexDefinitions/arrays_mapping.json diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index 53e75178a7..1d53504566 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -107,6 +107,6 @@ private static boolean isQuoted(String text, String mark) { } public static String removeParenthesis(String qualifier) { - return qualifier.replaceAll("\\[.+\\]", ""); + return qualifier.replaceAll("\\[\\d+\\]", ""); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java index 2da8e6a5dd..6674a266ae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ArrayReferenceExpression.java @@ -17,6 +17,8 @@ import lombok.Getter; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; @@ -31,13 +33,13 @@ public class ArrayReferenceExpression extends ReferenceExpression { @Getter private final ExprType type; public ArrayReferenceExpression(String ref, ExprType type, List> partsAndIndexes) { - super(ref, type); + super(StringUtils.removeParenthesis(ref), type); this.partsAndIndexes = partsAndIndexes; this.type = type; } public ArrayReferenceExpression(ReferenceExpression ref) { - super(ref.toString(), ref.type()); + super(StringUtils.removeParenthesis(ref.toString()), ref.type()); this.partsAndIndexes = Arrays.stream(ref.toString().split("\\.")).map(e -> Pair.of(e, OptionalInt.empty())).collect( Collectors.toList()); this.type = ref.type(); @@ -68,28 +70,25 @@ private ExprValue resolve(ExprValue value, List> paths ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, pathsWithoutParenthesis)); if (!paths.get(0).getRight().isEmpty()) { - wholePathValue = getIndexValueOfCollection(wholePathValue); - } else if (paths.size() == 1 && !wholePathValue.type().equals(ExprCoreType.ARRAY)) { - wholePathValue = ExprValueUtils.missingValue(); + if (value.keyValue(pathsWithoutParenthesis.get(0)) instanceof ExprCollectionValue) { // TODO check array size + wholePathValue = value + .keyValue(pathsWithoutParenthesis.get(0)) + .collectionValue() + .get(paths.get(0).getRight().getAsInt()); + if (paths.size() != 1) { + return resolve(wholePathValue, paths.subList(1, paths.size())); + } + } else { + return ExprValueUtils.missingValue(); + } + } else if (wholePathValue.isMissing()) { + return resolve(value.keyValue(pathsWithoutParenthesis.get(0)), paths.subList(1, paths.size())); } if (!wholePathValue.isMissing() || paths.size() == 1) { return wholePathValue; } else { - return resolve(value.keyValue(pathsWithoutParenthesis.get(0) - ), paths.subList(1, paths.size())); + return resolve(value.keyValue(pathsWithoutParenthesis.get(0)), paths.subList(1, paths.size())); } } - - private ExprValue getIndexValueOfCollection(ExprValue value) { - ExprValue collectionVal = value; -// for (OptionalInt currentIndex : List.of(index)) { - if (collectionVal.type().equals(ExprCoreType.ARRAY)) { -// collectionVal = collectionVal.collectionValue().get(currentIndex.getAsInt()); - } else { - return ExprValueUtils.missingValue(); - } -// } - return collectionVal; - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 7216c03d08..5c2c599c5f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -38,6 +38,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient; import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getArraysIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping; @@ -681,7 +682,11 @@ public enum Index { NESTED_WITH_NULLS(TestsConstants.TEST_INDEX_NESTED_WITH_NULLS, "multi_nested", getNestedTypeIndexMapping(), - "src/test/resources/nested_with_nulls.json"); + "src/test/resources/nested_with_nulls.json"), + ARRAYS(TestsConstants.TEST_INDEX_ARRAYS, + "arrays", + getArraysIndexMapping(), + "src/test/resources/arrays.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 30cee86e15..49641c0c29 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -223,6 +223,11 @@ public static String getDateIndexMapping() { return getMappingFile(mappingFile); } + public static String getArraysIndexMapping() { + String mappingFile = "arrays_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getDateTimeIndexMapping() { String mappingFile = "date_time_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 338be25a0c..ea9101b3f0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -48,6 +48,7 @@ public class TestsConstants { public final static String TEST_INDEX_ORDER = TEST_INDEX + "_order"; public final static String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; public final static String TEST_INDEX_DATE = TEST_INDEX + "_date"; + public final static String TEST_INDEX_ARRAYS = TEST_INDEX + "_arrays"; public final static String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; public final static String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java new file mode 100644 index 0000000000..65e5af0cc8 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ArraysIT.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ARRAYS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import java.util.List; + +import com.google.common.collect.ImmutableMap; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class ArraysIT extends SQLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.ARRAYS); + } + + @Test + public void object_array_index_1_test() { + String query = "SELECT objectArray[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONObject(ImmutableMap.of("innerObject", List.of(1, 2))))); + } + + @Test + public void object_array_index_1_inner_object_test() { + String query = "SELECT objectArray[0].innerObject FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONArray(List.of(1, 2)))); + } + + @Test + public void object_array_index_1_inner_object_index_1_test() { + String query = "SELECT objectArray[0].innerObject[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(1)); + } + + @Test + public void multi_object_array_index_1_test() { + String query = "SELECT multiObjectArray[0] FROM " + TEST_INDEX_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows(new JSONObject(ImmutableMap.of("id", 1, "name", "blah")))); + } +} diff --git a/integ-test/src/test/resources/arrays.json b/integ-test/src/test/resources/arrays.json new file mode 100644 index 0000000000..ce70bf08e9 --- /dev/null +++ b/integ-test/src/test/resources/arrays.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"objectArray": [{"innerObject": [1, 2]}, {"innerObject": 3}], "multiObjectArray":[{"id": 1, "name": "blah"}, {"id": 2,"name": "hoo"}]} diff --git a/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json b/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json new file mode 100644 index 0000000000..2e905dd8b7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/arrays_mapping.json @@ -0,0 +1,23 @@ +{ + "mappings": { + "properties": { + "objectArray": { + "properties": { + "innerObject": { + "type": "keyword" + } + } + }, + "multiObjectArray": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "type": "keyword" + } + } + } + } + } +} \ No newline at end of file