Search: Fix many excluded search (refs #4793, refs #5209)

4 files changed, 123 insertions, 22 deletions

diff --git a/CHANGES.md b/CHANGES.md
index 36813b525..f89848dd4 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -7,20 +7,23 @@ Ishtar changelog
 ================
 
 ### Features/improvements ###
-- search: add created before/after field
-- add custom cached_label configuration for each main item
-- sheet document: better UI for files
-- add "created" field on main items in order to facilitate queries
-- model: add history for document and containers
-- remove "Administration" entry - put Account management in "Directory" entry
-- Admin - Global variable: edit in table, add import/export in CSV/JSON
-- Admin: overload index to add sub-section headers
+- General:
+  - add custom cached_label configuration for each main item
+  - add "created" field on main items in order to facilitate queries
+- Search: add created before/after field
+- Sheet document: better UI for files
+- Model: add history for document and containers
+- Menu: remove "Administration" entry - put Account management in "Directory" entry
+- Admin:
+  - Global variable: edit in table, add import/export in CSV/JSON
+  - overload index to add subsection headers
 - Geo: create/edit form - new openlayers version - add default IGN tiles
 - Import: improve bad encoding detection
 
 ### Bug fixes ###
-- find form: remove TAQ/TPQ check
-- fix treatment and file treatment sheet display (bad QR code link)
+- Search: Fix many excluded search
+- Find form: remove TAQ/TPQ check
+- Sheet: fix treatment and file treatment sheet display (bad QR code link)
 - File: filter plan action when preventive_operator is activated in profile 
 
 v4.0.42 - 2023-01-25
@@ -164,7 +167,7 @@ v4.0.29 - 2022-11-29
 
 ### Bug fixes ###
 - Fix redirect URL after logout
-- Fix choice display - non relevant translation is deactivated
+- Fix choice display - non-relevant translation is deactivated
 - Minor translation fixes
 
 
@@ -234,7 +237,7 @@ v4.0.22 - 2022-10-14
 --------------------
 
 ### Features ###
-- Context records: Excavation technic become many2many
+- Context records: Excavation technic become many-to-many
 
 v4.0.21 - 2022-10-14
 --------------------
@@ -277,7 +280,7 @@ v4.0.17 - 2022-09-10
 
 ### Bug fix ###
 
-- Geo: fix zoom to extent - fix item counts
+- Geo: fix zoom to the extent - fix item counts
 
 v4.0.16 - 2022-09-09
 --------------------
@@ -326,7 +329,7 @@ v4.0.12 - 2022-08-05
 
 ### Bug fix ###
 
-- fix bug for vanilla installation (db query before db is initialize)
+- fix bug for vanilla installation (db query before db is initialized)
 
 v4.0.11 - 2022-08-02
 --------------------
@@ -365,7 +368,7 @@ v4.0.8 - 2022-07-18
 
 ### Features ###
 
-- Update french translations
+- Update French translations
 
 v4.0.7 - 2022-07-18
 -------------------
diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 0f7645ec8..342f3d127 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -1340,6 +1340,24 @@ class FindSearchTest(FindInit, TestCase, SearchText):
         ]
         self._test_search(c, result, context="Text period search")
 
+    def test_operation_search(self):
+        values = [
+            str(f.operation.code_patriarche)
+            for f in self.finds
+        ]
+
+        c = Client()
+        c.login(username=self.username, password=self.password)
+
+        key = str(pgettext_lazy("key for text search", "code-patriarche"))
+        result = [
+            (f'{key}="{values[0]}"', 1),
+            (f'-{key}="{values[0]}"', 1),
+            (f'{key}="{values[0]}" {key}="{values[1]}"', 2),
+            (f'-{key}="{values[0]}" -{key}="{values[1]}"', 0),
+        ]
+        self._test_search(c, result, context="Operation")
+
     def test_operator_search(self):
         operation = self.operations[0]
         operator = Organization.objects.create(
@@ -1471,6 +1489,43 @@ class FindSearchTest(FindInit, TestCase, SearchText):
         res = json.loads(response.content.decode())
         self.assertEqual(res["recordsTotal"], 1)
 
+    def test_mixed_search(self):
+        ope_values = [
+            str(f.operation.code_patriarche)
+            for f in self.finds
+        ]
+        neo = Period.objects.get(txt_idx="neolithic")
+        final_neo = Period.objects.get(txt_idx="final-neolithic")
+        metal = models.MaterialType.objects.get(txt_idx="metal")
+        iron_metal = models.MaterialType.objects.get(txt_idx="iron_metal")
+
+        find = self.finds[0]
+        find2 = self.finds[1]
+        dating = Dating.objects.create(period=final_neo)
+        find.datings.add(dating)
+        find.material_types.add(iron_metal)
+        find2.material_types.add(iron_metal)
+        find = models.Find.objects.get(pk=find.pk)
+        find.save()
+        find2 = models.Find.objects.get(pk=find2.pk)
+        find2.save()
+
+        material_key = str(pgettext_lazy("key for text search", "material"))
+        period_key = str(pgettext_lazy("key for text search", "datings-period"))
+        ope_key = str(pgettext_lazy("key for text search", "code-patriarche"))
+        result = [
+            (f'{ope_key}="{ope_values[0]}" {period_key}="{neo}" '
+             f'{material_key}={iron_metal}', 1),
+            (f'-{ope_key}="{ope_values[0]}" {period_key}="{neo}"', 0),
+            (f'-{ope_key}="{ope_values[0]}" -{period_key}="{neo}"', 1),
+            (f'-{ope_key}="{ope_values[0]}" -{period_key}="{neo}" '
+             f'-{material_key}={metal}', 0),
+        ]
+
+        c = Client()
+        c.login(username=self.username, password=self.password)
+        self._test_search(c, result, context="Operation")
+
     def test_search_with_callable(self):
         find = self.finds[0]
         find2 = self.finds[1]
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 26da94f60..8d134e519 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -2981,6 +2981,12 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
         result = json.loads(response.content.decode())
         self.assertEqual(result["recordsTotal"], 1)
 
+        # multiple exclude
+        response = c.get(reverse("get-operation"),
+                         {"search_vector": "-fougere -chateau"})
+        result = json.loads(response.content.decode())
+        self.assertEqual(result["recordsTotal"], 0)
+
     def test_facet_search_vector(self):
         ope1 = self.operations[0]
         ope2 = self.operations[1]
@@ -2993,6 +2999,7 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
         ope1.save()
         ope2.year = 2020
         ope2.save()
+        # ope3.year: 2018
 
         data = {"numero_insee": "05000", "name": "Champoleon (test)"}
         town = self.create_towns(datas=data)[-1]
@@ -3010,6 +3017,9 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
 
         villa = models.RemainType.objects.get(txt_idx="villa")
         ope1.remains.add(villa)
+        cairn = models.RemainType.objects.get(txt_idx="cairn")
+        ope2.remains.add(cairn)
+        ope3.remains.add(cairn)
 
         search_period_q = str(pgettext("key for text search", "period"))
         result = [
@@ -3028,6 +3038,11 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
         ]
         self._test_search(c, result, context="Many integer")
 
+        result = [
+            ('-{}="{}"'.format(search_year_q, '2042";"2020'), 1),
+        ]
+        self._test_search(c, result, context="Exclude many integer")
+
         search_town_q = str(pgettext("key for text search", "town"))
         town = Town.objects.get(pk=town.pk)
         result = [
@@ -3044,6 +3059,12 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
         ]
         self._test_search(c, result, context="Period exclude")
 
+        search_period_q = str(pgettext("key for text search", "period"))
+        result = [
+            (f'-{search_period_q}="{neo}" -{search_period_q}="{gallo}"', 0),
+        ]
+        self._test_search(c, result, context="Many period exclude")
+
         result = [
             (
                 '{}="{}"'.format(
@@ -3076,6 +3097,11 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText):
             ('{}="{}"'.format(search_remain_q, str(villa)), 1),
         ]
         self._test_search(c, result, context="Non hierarchic remain search")
+        result = [
+            (f'-{search_remain_q}="{cairn}"', 1),
+            (f'-{search_remain_q}="{cairn}" -{search_remain_q}="{villa}"', 0),
+        ]
+        self._test_search(c, result, context="Non hierarchic remain search exclude")
 
         # boolean search
         search_open_q = str(pgettext("key for text search", "is-open"))
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 839480ba7..eaf05f027 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -1241,11 +1241,22 @@ def _manage_relation_types(relation_types, dct, query, or_reqs):
     return query
 
 
-def _construct_query(relation_types, dct, or_reqs, and_reqs):
+def _construct_query(relation_types, dct, or_reqs, and_reqs, excluded_relation=False):
+    # excluded -> reverse logic
+    if excluded_relation:
+        and_reqs, or_reqs = or_reqs, and_reqs
+        for key in dct:
+            if isinstance(dct[key], str):
+                values = [v for v in dct[key].split(";") if v]
+            else:
+                values = [dct[key]]
+            for value in values:
+                or_reqs.append((key, {key: value}))
+        dct = {}
 
     # manage multi value not already managed
     for key in list(dct.keys()):
-        if type(dct[key]) == str and ";" in dct[key]:
+        if isinstance(dct[key], str) and ";" in dct[key]:
             values = [v for v in dct[key].split(";") if v]
             if not values:
                 dct.pop(key)
@@ -1267,11 +1278,16 @@ def _construct_query(relation_types, dct, or_reqs, and_reqs):
             or_reqs.append((first_key, {other_key: value}))
 
     query = Q(**dct)
-    for k, or_req in or_reqs:
+    for or_req in or_reqs:
         alt_dct = dct.copy()
-        alt_dct.pop(k)
-        alt_dct.update(or_req)
-        query |= Q(**alt_dct)
+        if isinstance(or_req, (tuple, list)):
+            k, or_req = or_req
+            if k in alt_dct:
+                alt_dct.pop(k)
+            alt_dct.update(or_req)
+            query |= Q(**alt_dct)
+        else:
+            query |= (Q(**alt_dct) & Q(or_req))
     query = _manage_relation_types(relation_types, dct, query, or_reqs)
 
     done = []
@@ -2073,7 +2089,8 @@ def get_item(
         exc_query = None
         if excluded_dct or exc_and_reqs or exc_or_reqs or exc_relation_types:
             exc_query = _construct_query(
-                exc_relation_types, excluded_dct, exc_or_reqs, exc_and_reqs
+                exc_relation_types, excluded_dct, exc_or_reqs, exc_and_reqs,
+                excluded_relation=True
             )
 
         if query_own: