hybrid search improvements (#2796)

Author: abeglova

learning_resources_search/api.py

@@ -21,6 +21,7 @@
     DEPARTMENT_QUERY_FIELDS,
     HYBRID_COMBINED_INDEX,
     HYBRID_SEARCH_MODE,
+    HYBRID_SEARCH_PIPELINE_NAME,
     LEARNING_RESOURCE,
     LEARNING_RESOURCE_QUERY_FIELDS,
     LEARNING_RESOURCE_SEARCH_SORTBY_OPTIONS,
@@ -55,21 +56,6 @@
 ]
 
 HYBRID_SEARCH_KNN_K_VALUE = 5
-HYBRID_SEARCH_PAGINATION_DEPTH = 10
-HYBRID_SEARCH_POST_PROCESSOR = {
-    "description": "Post processor for hybrid search",
-    "phase_results_processors": [
-        {
-            "normalization-processor": {
-                "normalization": {"technique": "min_max"},
-                "combination": {
-                    "technique": "arithmetic_mean",
-                    "parameters": {"weights": [0.8, 0.2]},
-                },
-            }
-        }
-    ],
-}
 
 
 def gen_content_file_id(content_file_id):
@@ -679,10 +665,11 @@ def add_text_query_to_search(
             }
         }
 
+        pagination_depth = search_params.get("limit") * 3
         search = search.extra(
             query={
                 "hybrid": {
-                    "pagination_depth": HYBRID_SEARCH_PAGINATION_DEPTH,
+                    "pagination_depth": pagination_depth,
                     "queries": [text_query, vector_query],
                 }
             }
@@ -800,7 +787,7 @@ def execute_learn_search(search_params):
     search = construct_search(search_params)
 
     if search_params.get("search_mode") == HYBRID_SEARCH_MODE:
-        search = search.extra(search_pipeline=HYBRID_SEARCH_POST_PROCESSOR)
+        search = search.extra(search_pipeline=HYBRID_SEARCH_PIPELINE_NAME)
 
     results = search.execute().to_dict()
     if results.get("_shards", {}).get("failures"):

learning_resources_search/api_test.py

@@ -2470,7 +2470,7 @@ def test_execute_learn_search_with_hybrid_search(mocker, settings, opensearch):
         "size": 1,
         "query": {
             "hybrid": {
-                "pagination_depth": 10,
+                "pagination_depth": 3,
                 "queries": [
                     {
                         "bool": {
@@ -2776,20 +2776,7 @@ def test_execute_learn_search_with_hybrid_search(mocker, settings, opensearch):
                 },
             }
         },
-        "search_pipeline": {
-            "description": "Post processor for hybrid search",
-            "phase_results_processors": [
-                {
-                    "normalization-processor": {
-                        "normalization": {"technique": "min_max"},
-                        "combination": {
-                            "technique": "arithmetic_mean",
-                            "parameters": {"weights": [0.8, 0.2]},
-                        },
-                    }
-                }
-            ],
-        },
+        "search_pipeline": "hybrid_search_pipeline",
         "_source": {
             "excludes": [
                 "created_on",

learning_resources_search/constants.py

@@ -25,6 +25,22 @@
 HYBRID_COMBINED_INDEX = "combined_hybrid"
 LEARNING_RESOURCE = "learning_resource"
 HYBRID_SEARCH_MODE = "hybrid"
+HYBRID_SEARCH_PIPELINE_NAME = "hybrid_search_pipeline"
+
+HYBRID_SEARCH_PIPELINE_BODY = {
+    "description": "Post processor for hybrid search",
+    "phase_results_processors": [
+        {
+            "normalization-processor": {
+                "normalization": {"technique": "min_max"},
+                "combination": {
+                    "technique": "arithmetic_mean",
+                    "parameters": {"weights": [0.8, 0.2]},
+                },
+            }
+        }
+    ],
+}
 
 
 class IndexestoUpdate(Enum):
@@ -322,7 +338,13 @@ class FilterConfig:
     "max_weekly_hours": {"type": "integer"},
 }
 
-EMBEDDING_FIELDS = {"vector_embedding": {"type": "knn_vector"}}
+EMBEDDING_FIELDS = {
+    "vector_embedding": {
+        "type": "knn_vector",
+        "space_type": "cosinesimil",
+        "method": {"name": "hnsw", "space_type": "cosinesimil", "engine": "faiss"},
+    }
+}
 
 
 CONTENT_FILE_MAP = {

learning_resources_search/indexing_api.py

@@ -26,6 +26,8 @@
     COURSE_TYPE,
     EMBEDDING_FIELDS,
     HYBRID_COMBINED_INDEX,
+    HYBRID_SEARCH_PIPELINE_BODY,
+    HYBRID_SEARCH_PIPELINE_NAME,
     LEARNING_RESOURCE_MAP,
     MAPPING,
     PERCOLATE_INDEX_TYPE,
@@ -196,6 +198,17 @@ def clear_and_create_index(*, index_name=None, skip_mapping=False, object_type=N
             "properties": (LEARNING_RESOURCE_MAP | vector_map)
         }
         index_create_data["settings"]["index.knn"] = True
+
+        try:
+            conn.transport.perform_request(
+                "GET", f"/_search/pipeline/{HYBRID_SEARCH_PIPELINE_NAME}"
+            )
+        except NotFoundError:
+            conn.transport.perform_request(
+                "PUT",
+                f"/_search/pipeline/{HYBRID_SEARCH_PIPELINE_NAME}",
+                body=HYBRID_SEARCH_PIPELINE_BODY,
+            )
     elif not skip_mapping:
         index_create_data["mappings"] = {"properties": MAPPING[object_type]}