Full text search: manage facet search (simple, hierarchic, OR) (refs #4180)

6 files changed, 220 insertions, 38 deletions

diff --git a/archaeological_operations/forms.py b/archaeological_operations/forms.py
index cbaa37310..4e01a492a 100644
--- a/archaeological_operations/forms.py
+++ b/archaeological_operations/forms.py
@@ -472,6 +472,8 @@ RecordRelationsFormSet.form_slug = "operation-080-relations"
 
 
 class OperationSelect(TableSelect):
+    _model = models.Operation
+
     search_vector = forms.CharField(label=_(u"Full text search"),
                                     widget=widgets.SearchWidget)
     year = forms.IntegerField(label=_("Year"))
@@ -486,12 +488,9 @@ class OperationSelect(TableSelect):
     if settings.ISHTAR_DPTS:
         towns__numero_insee__startswith = forms.ChoiceField(
             label=_(u"Department"), choices=[])
-    common_name = forms.CharField(label=_(u"Name"),
-                                  max_length=30)
-    address = forms.CharField(label=_(u"Address / Locality"),
-                              max_length=100)
-    operation_type = forms.ChoiceField(label=_(u"Operation type"),
-                                       choices=[])
+    common_name = forms.CharField(label=_(u"Name"), max_length=30)
+    address = forms.CharField(label=_(u"Address / Locality"), max_length=100)
+    operation_type = forms.ChoiceField(label=_(u"Operation type"), choices=[])
     end_date = forms.NullBooleanField(label=_(u"Is open?"))
     in_charge = forms.IntegerField(
         widget=widgets.JQueryAutoComplete(
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index d50c9e43c..688c12bea 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -27,7 +27,7 @@ from django.db import IntegrityError, transaction
 from django.db.models import Q, Count, Sum, Max, Avg
 from django.db.models.signals import post_save, m2m_changed, post_delete
 from django.forms import ValidationError
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _, pgettext_lazy
 
 from ishtar_common.models import BaseHistorizedItem, Dashboard, \
     DashboardFormItem, Department, Document, DocumentTemplate, \
@@ -362,6 +362,17 @@ class Operation(ClosedItem, BaseHistorizedItem, OwnPerms, ValueGetter,
         },
     }
 
+    # alternative names of fields for searches
+    ALT_NAMES = {
+        'periods__pk': pgettext_lazy(
+            "key for text search (no accent, no spaces)", u"period"),
+        'operation_type__pk': pgettext_lazy(
+            "key for text search (no accent, no spaces)", u"operation-type"),
+        'remains__pk': pgettext_lazy(
+            "key for text search (no accent, no spaces)", u"remain"),
+    }
+    EXTRA_REQUEST_KEYS.update(dict([(v, k) for k, v in ALT_NAMES.items()]))
+
     # fields definition
     creation_date = models.DateField(_(u"Creation date"),
                                      default=datetime.date.today)
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 381efd070..1480eb502 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -33,7 +33,7 @@ from django.db.models import Q
 from django.test.client import Client
 
 from django.contrib.auth.models import User, Permission
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _, pgettext
 import models
 
 from archaeological_operations import views
@@ -1538,11 +1538,64 @@ class OperationSearchTest(TestCase, OperationInitTest):
         self.assertEqual(result['recordsTotal'], 2)
 
         # open search
-        response = c.get(reverse('get-operation'),
-                         {'search_vector': 'cha*'})
+        response = c.get(reverse('get-operation'), {'search_vector': 'cha*'})
         result = json.loads(response.content)
         self.assertEqual(result['recordsTotal'], 3)
 
+    def test_facet_search_vector(self):
+        ope1 = self.operations[0]
+        ope2 = self.operations[1]
+        ope3 = self.operations[2]
+        c = Client()
+        c.login(username=self.username, password=self.password)
+
+        neo = models.Period.objects.get(txt_idx='neolithic')
+        final_neo = models.Period.objects.get(txt_idx='final-neolithic')
+        gallo = models.Period.objects.get(txt_idx="gallo-roman")
+        ope1.periods.add(final_neo)
+        ope1.periods.add(gallo)
+        ope2.periods.add(neo)
+        ope3.periods.add(gallo)
+
+        villa = models.RemainType.objects.get(txt_idx='villa')
+        ope1.remains.add(villa)
+
+        # simple
+        search_q = unicode(
+            pgettext("key for text search (no accent, no spaces)", u"period")
+        )
+        search = {'search_vector': u'{}="{}"'.format(search_q, final_neo.label)}
+        response = c.get(reverse('get-operation'), search)
+        self.assertEqual(response.status_code, 200)
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 1)
+
+        # hierarchic
+        search = {'search_vector': u'{}="{}"'.format(search_q, neo.label)}
+        response = c.get(reverse('get-operation'), search)
+        self.assertEqual(response.status_code, 200)
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 2)
+
+        # OR
+        search = {'search_vector': u'{}="{}";"{}"'.format(search_q, neo.label,
+                                                          gallo.label)}
+        response = c.get(reverse('get-operation'), search)
+        self.assertEqual(response.status_code, 200)
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 3)
+
+        # non hierarchic search
+        search_q = unicode(
+            pgettext("key for text search (no accent, no spaces)", u"remain")
+        )
+        search = {'search_vector': u'{}="{}"'.format(search_q, villa.label)}
+        response = c.get(reverse('get-operation'), search)
+        self.assertEqual(response.status_code, 200)
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 1)
+
+
     def create_relations(self):
         rel1 = models.RelationType.objects.create(
             symmetrical=True, label='Include', txt_idx='include')
diff --git a/ishtar_common/forms.py b/ishtar_common/forms.py
index e01e74a14..bb468eff1 100644
--- a/ishtar_common/forms.py
+++ b/ishtar_common/forms.py
@@ -461,13 +461,19 @@ class IshtarForm(forms.Form):
 class TableSelect(IshtarForm):
     def __init__(self, *args, **kwargs):
         super(TableSelect, self).__init__(*args, **kwargs)
-        # no field is required for search
+        ALT_NAMES = {}
+        if hasattr(self, '_model') and hasattr(self._model, "ALT_NAMES"):
+            ALT_NAMES = self._model.ALT_NAMES
         for k in self.fields:
-            self.fields[k].required = False
+            self.fields[k].required = False  # no field is required for search
             cls = 'form-control'
             if k == 'search_vector':
                 cls += " search-vector"
             self.fields[k].widget.attrs['class'] = cls
+            if k in ALT_NAMES:
+                self.fields[k].alt_name = ALT_NAMES[k]
+            else:
+                self.fields[k].alt_name = k
         key = self.fields.keys()[0]
         self.fields[key].widget.attrs['autofocus'] = 'autofocus'
 
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 86b74693f..d65c08f93 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -1094,6 +1094,16 @@ class FullSearch(models.Model):
     class Meta:
         abstract = True
 
+    @classmethod
+    def general_types(cls):
+        for k in get_all_field_names(cls):
+            field = cls._meta.get_field(k)
+            if not hasattr(field, 'rel') or not field.rel:
+                continue
+            rel_model = field.rel.to
+            if issubclass(rel_model, (GeneralType, HierarchicalType)):
+                yield k
+
     def update_search_vector(self, save=True):
         """
         Update the search vector
@@ -1275,7 +1285,7 @@ class BaseHistorizedItem(DocumentItem, FullSearch, Imported, JsonData,
                          FixAssociated):
     """
     Historized item with external ID management.
-    All historized items are searcheable and have a data json field
+    All historized items are searchable and have a data json field.
     """
     IS_BASKET = False
     EXTERNAL_ID_KEY = ''
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index f5e47a832..a2cc0762c 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -312,8 +312,16 @@ FORBIDDEN_CHAR = [u":"]
 RESERVED_CHAR = [u"|", u"&"]
 
 
-def _parse_query_string(string):
+def _parse_query_string(string, request_keys, current_dct):
     string = string.strip().lower()
+
+    if u"=" in string:
+        splited = string.split(u"=")
+        if len(splited) == 2 and splited[0] in request_keys:
+            term, query = splited
+            term = request_keys[term]
+            current_dct[term] = query
+            return ""
     for reserved_char in FORBIDDEN_CHAR:
         string = string.replace(reserved_char, u"")
     if len(string) != 1:
@@ -326,22 +334,53 @@ def _parse_query_string(string):
     return string
 
 
-def _parse_parentheses_groups(groups):
+def _parse_parentheses_groups(groups, request_keys, current_dct=None):
     """
     Transform parentheses groups to query
+
+    :param groups: groups to transform (list)
+    :param request_keys: request keys for facet search
+    :param current_dct:
+    :return: query string, query dict
     """
+    if not current_dct:
+        current_dct = {}
     if type(groups) is not list:
         string = groups.strip()
         # split into many groups if spaces
-        if ' ' not in string:
-            return _parse_query_string(groups)
-        return _parse_parentheses_groups(string.split(u" "))
+
+        # do not split inside quotes
+        current_index = 0
+        found = string.find('"', current_index)
+        SEP = u"?ç;?"  # replace spaces inside quote with this characters
+        previous_quote = None
+        while found != -1:
+            if previous_quote:
+                string = string[0:previous_quote] + \
+                         string[previous_quote:found].replace(u' ', SEP) + \
+                         string[found:]
+                previous_quote = None
+                # SEP is larger than a space
+                found = string.find('"', current_index)
+            else:
+                previous_quote = found
+            current_index = found + 1
+            found = string.find('"', current_index)
+
+        string_groups = [gp.replace(SEP, u" ") for gp in string.split(u" ")]
+        if len(string_groups) == 1:
+            return _parse_query_string(string_groups[0], request_keys,
+                                       current_dct), current_dct
+        return _parse_parentheses_groups(string_groups,
+                                         request_keys, current_dct)
     if not groups:  # empty list
-        return ""
+        return "", current_dct
     query = u"("
     previous_sep, has_item = None, False
     for item in groups:
-        q = _parse_parentheses_groups(item).strip()
+        q, current_dct = _parse_parentheses_groups(item, request_keys,
+                                                   current_dct)
+        q = q.strip()
         if not q:
             continue
         if q in (u"|", u"&"):
@@ -358,17 +397,24 @@ def _parse_parentheses_groups(groups):
         has_item = True
         previous_sep = None
     query += u")"
-    return unidecode(query)
+    if query == u"()":
+        query = u""
+    return unidecode(query), current_dct
 
 
-def _search_manage_search_vector(dct):
-    if 'search_vector' in dct:
-        parentheses_groups = _parse_parentheses(dct['search_vector'].strip())
-        query = _parse_parentheses_groups(parentheses_groups)
+def _search_manage_search_vector(dct, request_keys):
+    if 'search_vector' not in dct:
+        return dct
+
+    parentheses_groups = _parse_parentheses(dct['search_vector'].strip())
+    search_query, extra_dct = _parse_parentheses_groups(parentheses_groups,
+                                                        request_keys)
+    dct.update(extra_dct)
+    if search_query:
         dct['extras'].append(
             {'where': ["search_vector @@ (to_tsquery(%s, %s)) = true"],
              'params': [settings.ISHTAR_SEARCH_LANGUAGE,
-                        query]}
+                        search_query]}
         )
     return dct
 
@@ -478,6 +524,8 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
         else:
             my_relation_types_prefix = copy(relation_types_prefix)
 
+        general_types = model.general_types()
+
         fields = [model._meta.get_field(k)
                   for k in get_all_field_names(model)]
 
@@ -599,6 +647,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                 dct = request.session[func_name]
         else:
             request.session[func_name] = dct
+
+        dct['extras'] = []
+        dct = _search_manage_search_vector(dct, request_keys)
+        search_vector = ""
+        if 'search_vector' in dct:
+            search_vector = dct.pop('search_vector')
+
         for k in (list(my_bool_fields) + list(my_reversed_bool_fields)):
             if k in dct:
                 if dct[k] == u"1":
@@ -682,19 +737,67 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                     break
                 elif req.endswith(k_hr + '__pk'):
                     val = dct.pop(req)
-                    reqs = Q(**{req: val})
-                    req = req[:-2] + '__'
-                    for idx in range(HIERARCHIC_LEVELS):
-                        req = req[:-2] + 'parent__pk'
-                        q = Q(**{req: val})
-                        reqs |= q
-                    and_reqs.append(reqs)
+
+                    if u";" in val:
+                        # OR request
+                        values = val.split(u";")
+                    else:
+                        values = [val]
+                    base_req = req[:]
+                    reqs = None
+                    for val in values:
+                        suffix = "pk"
+                        req = base_req[:]
+
+                        if val.startswith(u'"') and val.startswith(u'"'):
+                            # manage search text by label
+                            if u"%" in val:
+                                suffix = "label__icontains"
+                            else:
+                                suffix = "label__iexact"
+                            val = val[1:-1]
+                            req = req[:-2] + suffix
+
+                        if not reqs:
+                            reqs = Q(**{req: val})
+                        else:
+                            reqs |= Q(**{req: val})
+                        for idx in range(HIERARCHIC_LEVELS):
+                            req = req[:-(len(suffix))] + 'parent__' + suffix
+                            q = Q(**{req: val})
+                            reqs |= q
+                    if reqs:
+                        and_reqs.append(reqs)
                     break
-        dct['extras'] = []
-        dct = _search_manage_search_vector(dct)
-        search_vector = ""
-        if 'search_vector' in dct:
-            search_vector = dct.pop('search_vector')
+
+        # manage search text by label
+        for base_k in general_types:
+            if base_k in HIERARCHIC_FIELDS:
+                continue
+            k = base_k + "__pk"
+            if k not in dct or not dct[k].startswith(u'"') \
+                    or not dct[k].startswith(u'"'):
+                continue
+            val = dct.pop(k)
+            if u";" in val:
+                # OR request
+                values = val.split(u";")
+            else:
+                values = [val]
+            reqs = None
+            for val in values:
+                if not val.endswith(u'"') or not val.startswith(u""):
+                    continue
+                query = val[1:-1]
+                suffix = "__label__icontains" if u"%" in val else \
+                    "__label__iexact"
+                if not reqs:
+                    reqs = Q(**{base_k + suffix: query})
+                else:
+                    reqs |= Q(**{base_k + suffix: query})
+            if reqs:
+                and_reqs.append(reqs)
+
         extras = dct.pop('extras')
 
         query = Q(**dct)