Skip to content

Commit

Permalink
fix: opensearch script score with filters (#3321)
Browse files Browse the repository at this point in the history 
* fix opensearch script score filters

* add comment

* add integration test

* update schema
  • Loading branch information
@tstadel
tstadel authored Oct 6, 2022
1 parent 6cb4e93 commit b84a6b1
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 59 deletions.
9 changes: 6 additions & 3 deletions haystack/document_stores/opensearch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,9 +429,12 @@ def query_by_embedding(
# +1 in similarity to avoid negative numbers (for cosine sim)
body: Dict[str, Any] = {"size": top_k, "query": self._get_vector_similarity_query(query_emb, top_k)}
if filters:
if not "bool" in body["query"]:
body["query"]["bool"] = {}
body["query"]["bool"]["filter"] = LogicalFilterClause.parse(filters).convert_to_elasticsearch()
filter_ = LogicalFilterClause.parse(filters).convert_to_elasticsearch()
if "script_score" in body["query"]:
# set filter for pre-filtering (see https://opensearch.org/docs/latest/search-plugins/knn/knn-score-script/)
body["query"]["script_score"]["query"] = {"bool": {"filter": filter_}}
else:
body["query"]["bool"]["filter"] = filter_

excluded_meta_data: Optional[list] = None

Expand Down
56 changes: 28 additions & 28 deletions haystack/json-schemas/haystack-pipeline-1.10.0rc0.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3044,10 +3044,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -3345,10 +3345,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4423,10 +4423,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4577,10 +4577,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4720,10 +4720,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4853,10 +4853,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5013,10 +5013,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5105,10 +5105,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5289,10 +5289,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5872,10 +5872,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6002,10 +6002,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6133,10 +6133,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6264,10 +6264,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6376,10 +6376,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down
56 changes: 28 additions & 28 deletions haystack/json-schemas/haystack-pipeline-main.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3044,10 +3044,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -3345,10 +3345,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4423,10 +4423,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4577,10 +4577,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4720,10 +4720,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -4853,10 +4853,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5013,10 +5013,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5105,10 +5105,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5289,10 +5289,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -5872,10 +5872,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6002,10 +6002,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6133,10 +6133,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6264,10 +6264,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down Expand Up @@ -6376,10 +6376,10 @@
"title": "Use Auth Token",
"anyOf": [
{
"type": "boolean"
"type": "string"
},
{
"type": "string"
"type": "boolean"
},
{
"type": "null"
Expand Down
Loading

0 comments on commit b84a6b1

Please sign in to comment.