✨ search add OR "||" - search refactoring

2 files changed, 198 insertions, 127 deletions

diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 10bc52967..bfd2a162b 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -3070,6 +3070,23 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
         result = json.loads(response.content.decode())
         self.assertEqual(result["recordsTotal"], 1)
 
+    def test_or_search_vector(self):
+        operation_1 = models.Operation.objects.get(pk=self.operations[0].pk)
+        operation_2 = models.Operation.objects.get(pk=self.operations[1].pk)
+        operation_1.common_name = "Château de Fougères"
+        operation_1.save()
+        operation_2.code_patriarche = "OA4242"
+        operation_2.save()
+
+        c = Client()
+        c.login(username=self.username, password=self.password)
+
+        patriarche_key = pgettext_lazy("key for text search", "patriarche")
+        name_key = pgettext_lazy("key for text search", "name")
+        request = f'{patriarche_key}="OA4242" || {name_key}="{operation_1.common_name}"'
+        response = c.get(reverse("get-operation"), {"search_vector": request})
+        self.assertEqual(json.loads(response.content.decode())["recordsTotal"], 2)
+
     def test_complex_search_vector(self):
         c = Client()
         c.login(username=self.username, password=self.password)
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index b5c63cc65..48b83e654 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -1560,6 +1560,8 @@ def _manage_direct_id(dct):
     Manage "id:1234" syntax
     """
     for k, v in list(dct.items()):
+        if not isinstance(v, str):
+            continue
         m = RE_ID.match(v)
         if not m:
             continue
@@ -1951,6 +1953,147 @@ def _get_table_cols(data_type, own_table_cols, full, model):
     return table_cols
 
 
+def split_dict(dct):
+    if not dct.get("search_vector", None):
+        return [dct]
+    new_dcts = []
+    for vector in dct["search_vector"].split(" || "):
+        new_dct = deepcopy(dct)
+        new_dct["search_vector"] = vector
+        new_dcts.append(new_dct)
+    return new_dcts
+
+
+def main_manager(
+    request, model, query_own, full, dct, distinct_queries, query_parameters,
+    my_relation_types_prefix, my_bool_fields, my_reversed_bool_fields,
+    related_name_fields, many_counted_fields, reversed_many_counted_fields,
+    my_dated_fields, my_number_fields, and_reqs
+        ):
+    excluded_dct = {}
+    or_reqs = []
+    exc_and_reqs, exc_or_reqs = [], []
+    dct, excluded_dct, distinct_queries = _search_manage_search_vector(
+        model,
+        dct,
+        excluded_dct,
+        distinct_queries,
+        query_parameters,
+    )
+
+    if "search_vector" in dct:
+        dct.pop("search_vector")
+
+    # manage relations types
+    if "relation_types" not in my_relation_types_prefix:
+        my_relation_types_prefix["relation_types"] = ""
+
+    relation_types = _get_relation_type_dict(my_relation_types_prefix, dct)
+    exc_relation_types = _get_relation_type_dict(my_relation_types_prefix,
+                                                 excluded_dct)
+
+    _manage_bool_fields(
+        model, my_bool_fields, my_reversed_bool_fields, dct, or_reqs
+    )
+    _manage_bool_fields(
+        model, my_bool_fields, my_reversed_bool_fields, excluded_dct, exc_or_reqs
+    )
+
+    tmp_excluded = {}
+    _manage_many_counted_fields(
+        many_counted_fields, reversed_many_counted_fields, dct, tmp_excluded
+    )
+    _manage_many_counted_fields(
+        many_counted_fields, reversed_many_counted_fields, excluded_dct, dct
+    )
+    if tmp_excluded:
+        excluded_dct.update(tmp_excluded)
+
+    # dated_fields, number_fields
+    # ['signature_date', ...], ['signature_date__year', ...]
+    # -> remove 'signature_date'
+    filtered_dated_fields = []
+    for field_name in my_dated_fields:
+        exc = False
+        for number_field in my_number_fields:
+            if number_field.startswith(field_name):
+                exc = True
+                break
+        if not exc:
+            filtered_dated_fields.append(field_name)
+    my_dated_fields = filtered_dated_fields
+
+    _manage_dated_fields(my_dated_fields, dct)
+    _manage_dated_fields(my_dated_fields, excluded_dct)
+
+    _manage_number_fields(my_number_fields, dct)
+    _manage_number_fields(my_number_fields, excluded_dct)
+
+    _manage_hierarchic_fields(model, dct, and_reqs)
+    _manage_hierarchic_fields(model, excluded_dct, exc_and_reqs)
+
+    _manage_facet_search(model, dct, and_reqs)
+    _manage_facet_search(model, excluded_dct, exc_and_reqs)
+
+    extras = []
+    if "extras" in dct:
+        extras = dct.pop("extras")
+    if "and_reqs" in dct:
+        and_reqs += dct.pop("and_reqs")
+    if "exc_and_reqs" in dct:
+        exc_and_reqs += dct.pop("exc_and_reqs")
+
+    updated_excluded = {}
+    _manage_clean_search_field(
+        dct, updated_excluded, related_name_fields=related_name_fields
+    )
+    _manage_clean_search_field(
+        excluded_dct, dct, reverse=True, related_name_fields=related_name_fields
+    )
+    if updated_excluded:
+        excluded_dct.update(updated_excluded)
+
+    # manage direct ID 'id:1234' syntax - used by "{USER}" search
+    _manage_direct_id(dct)
+    _manage_direct_id(excluded_dct)
+
+    query = _construct_query(relation_types, dct, or_reqs, and_reqs)
+    exc_query = None
+    if excluded_dct or exc_and_reqs or exc_or_reqs or exc_relation_types:
+        exc_query = _construct_query(
+            exc_relation_types, excluded_dct, exc_or_reqs, exc_and_reqs,
+            excluded_relation=True
+        )
+
+    if query_own:
+        query = query & query_own
+
+    # manage hierarchic in shortcut menu
+    if full == "shortcut":
+        query = manage_hierarchy_shorcut(model, request, query)
+    return query, exc_query, extras
+
+
+def manage_hierarchy_shorcut(model, request, query):
+    File = apps.get_model("archaeological_files", "File")
+    Operation = apps.get_model("archaeological_operations", "Operation")
+    ContextRecord = apps.get_model("archaeological_context_records", "ContextRecord")
+    Find = apps.get_model("archaeological_finds", "Find")
+    ASSOCIATED_ITEMS = {
+        Operation: (File, "associated_file__pk"),
+        ContextRecord: (Operation, "operation__pk"),
+        Find: (ContextRecord, "base_finds__context_record__pk"),
+    }
+    if model in ASSOCIATED_ITEMS:
+        upper_model, upper_key = ASSOCIATED_ITEMS[model]
+        model_name = upper_model.SLUG
+        current = model_name in request.session and request.session[model_name]
+        if current:
+            dct = {upper_key: current}
+            query &= Q(**dct)
+    return query
+
+
 DEFAULT_ROW_NUMBER = 10
 # length is used by ajax DataTables requests
 EXCLUDED_FIELDS = ["length"]
@@ -2253,9 +2396,7 @@ def get_item(
             dct = {}
         else:
             dct = copy(my_base_request)
-        excluded_dct = {}
-        and_reqs, or_reqs = [], []
-        exc_and_reqs, exc_or_reqs = [], []
+        and_reqs = []
         distinct_queries = []
         dct["extras"], dct["and_reqs"], dct["exc_and_reqs"] = [], [], []
 
@@ -2352,128 +2493,44 @@ def get_item(
         related_name_fields = [query_parameters[k].related_name for k in query_parameters
                                if query_parameters[k].related_name]
 
-        dct, excluded_dct, distinct_queries = _search_manage_search_vector(
-            model,
-            dct,
-            excluded_dct,
-            distinct_queries,
-            query_parameters,
-        )
-        search_vector = ""
-        if "search_vector" in dct:
-            search_vector = dct.pop("search_vector")
-
-        # manage relations types
-        if "relation_types" not in my_relation_types_prefix:
-            my_relation_types_prefix["relation_types"] = ""
-
-        relation_types = _get_relation_type_dict(my_relation_types_prefix, dct)
-        exc_relation_types = _get_relation_type_dict(my_relation_types_prefix,
-                                                     excluded_dct)
-
-        _manage_bool_fields(
-            model, my_bool_fields, my_reversed_bool_fields, dct, or_reqs
-        )
-        _manage_bool_fields(
-            model, my_bool_fields, my_reversed_bool_fields, excluded_dct, exc_or_reqs
-        )
-
-        tmp_excluded = {}
-        _manage_many_counted_fields(
-            many_counted_fields, reversed_many_counted_fields, dct, tmp_excluded
-        )
-        _manage_many_counted_fields(
-            many_counted_fields, reversed_many_counted_fields, excluded_dct, dct
-        )
-        if tmp_excluded:
-            excluded_dct.update(tmp_excluded)
-
-        # dated_fields, number_fields
-        # ['signature_date', ...], ['signature_date__year', ...]
-        # -> remove 'signature_date'
-        filtered_dated_fields = []
-        for field_name in my_dated_fields:
-            exc = False
-            for number_field in my_number_fields:
-                if number_field.startswith(field_name):
-                    exc = True
-                    break
-            if not exc:
-                filtered_dated_fields.append(field_name)
-        my_dated_fields = filtered_dated_fields
-
-        _manage_dated_fields(my_dated_fields, dct)
-        _manage_dated_fields(my_dated_fields, excluded_dct)
-
-        _manage_number_fields(my_number_fields, dct)
-        _manage_number_fields(my_number_fields, excluded_dct)
-
-        _manage_hierarchic_fields(model, dct, and_reqs)
-        _manage_hierarchic_fields(model, excluded_dct, exc_and_reqs)
-
-        _manage_facet_search(model, dct, and_reqs)
-        _manage_facet_search(model, excluded_dct, exc_and_reqs)
-
-        extras = []
-        if "extras" in dct:
-            extras = dct.pop("extras")
-        if "and_reqs" in dct:
-            and_reqs += dct.pop("and_reqs")
-        if "exc_and_reqs" in dct:
-            exc_and_reqs += dct.pop("exc_and_reqs")
-
-        updated_excluded = {}
-        _manage_clean_search_field(dct, updated_excluded, related_name_fields=related_name_fields)
-        _manage_clean_search_field(excluded_dct, dct, reverse=True, related_name_fields=related_name_fields)
-        if updated_excluded:
-            excluded_dct.update(updated_excluded)
-
-        # manage direct ID 'id:1234' syntax - used by "{USER}" search
-        _manage_direct_id(dct)
-        _manage_direct_id(excluded_dct)
-
-        query = _construct_query(relation_types, dct, or_reqs, and_reqs)
-        exc_query = None
-        if excluded_dct or exc_and_reqs or exc_or_reqs or exc_relation_types:
-            exc_query = _construct_query(
-                exc_relation_types, excluded_dct, exc_or_reqs, exc_and_reqs,
-                excluded_relation=True
+        items = None
+        for sub_dct in split_dict(dct):
+            query, exc_query, extras = main_manager(
+                request,
+                model,
+                query_own,
+                full,
+                sub_dct,
+                distinct_queries,
+                query_parameters,
+                my_relation_types_prefix,
+                my_bool_fields,
+                my_reversed_bool_fields,
+                related_name_fields,
+                many_counted_fields,
+                reversed_many_counted_fields,
+                my_dated_fields,
+                my_number_fields,
+                and_reqs
             )
 
-        if query_own:
-            query = query & query_own
-
-        # manage hierarchic in shortcut menu
-        if full == "shortcut":
-            File = apps.get_model("archaeological_files", "File")
-            Operation = apps.get_model("archaeological_operations", "Operation")
-            ContextRecord = apps.get_model("archaeological_context_records", "ContextRecord")
-            Find = apps.get_model("archaeological_finds", "Find")
-            ASSOCIATED_ITEMS = {
-                Operation: (File, "associated_file__pk"),
-                ContextRecord: (Operation, "operation__pk"),
-                Find: (ContextRecord, "base_finds__context_record__pk"),
-            }
-            if model in ASSOCIATED_ITEMS:
-                upper_model, upper_key = ASSOCIATED_ITEMS[model]
-                model_name = upper_model.SLUG
-                current = model_name in request.session and request.session[model_name]
-                if current:
-                    dct = {upper_key: current}
-                    query &= Q(**dct)
-        # print("ishtar_common/views_item.py - 2455")
-        # print(query, distinct_queries, base_query, exc_query, extras)
-        items = model.objects.filter(query)
-        for d_q in distinct_queries:
-            items = items.filter(d_q)
-
-        if base_query:
-            items = items.filter(base_query)
-        if exc_query:
-            items = items.exclude(exc_query)
-
-        for extra in extras:
-            items = items.extra(**extra)
+            # print("ishtar_common/views_item.py - 2515")
+            # print(query, distinct_queries, base_query, exc_query, extras)
+            sub_items = model.objects.filter(query)
+            for d_q in distinct_queries:
+                sub_items = sub_items.filter(d_q)
+
+            if base_query:
+                sub_items = sub_items.filter(base_query)
+            if exc_query:
+                sub_items = sub_items.exclude(exc_query)
+
+            for extra in extras:
+                sub_items = sub_items.extra(**extra)
+            if not items:
+                items = sub_items
+            else:
+                items |= sub_items
 
         if return_query:
             return items
@@ -2502,9 +2559,6 @@ def get_item(
             return items_nb
         # print(str(items.values("id").query))
 
-        if search_vector:  # for serialization
-            dct["search_vector"] = search_vector
-
         if data_type == "json-stats":
             stats_modality_1 = request_items.get("stats_modality_1", None)
             stats_modality_2 = request_items.get("stats_modality_2", None)
@@ -2734,7 +2788,7 @@ def get_item(
         lock = '&nbsp;<i class="fa fa-lock text-danger" aria-hidden="true"></i>'
         own_lock = '&nbsp;<i class="fa fa-lock text-success" ' 'aria-hidden="true"></i>'
         has_locks = hasattr(model, "locked")
-        current_user_id = request.user and request.user.id
+        current_user_id = request and request.user and request.user.id
         if data_type.startswith("json"):
             rows = []
             if data_type == "json-map":