Search: manage exclude - refactor search (refs #4180)

1 files changed, 282 insertions, 217 deletions

diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 675c36575..d19cbf7ff 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -312,16 +312,26 @@ FORBIDDEN_CHAR = [u":"]
 RESERVED_CHAR = [u"|", u"&"]
 
 
-def _parse_query_string(string, request_keys, current_dct):
+def _parse_query_string(string, request_keys, current_dct, exc_dct):
     string = string.strip().lower()
 
     if u"=" in string:
         splited = string.split(u"=")
-        if len(splited) == 2 and splited[0] in request_keys:
+        if len(splited) == 2:
             term, query = splited
-            term = request_keys[term]
-            current_dct[term] = query
-            return ""
+            excluded = term.startswith(u"-")
+            if excluded:
+                term = term[1:]
+            if term in request_keys:
+                term = request_keys[term]
+                dct = current_dct
+                if excluded:
+                    dct = exc_dct
+                if term in dct:
+                    dct[term] += u";" + query
+                else:
+                    dct[term] = query
+                return ""
     for reserved_char in FORBIDDEN_CHAR:
         string = string.replace(reserved_char, u"")
     if len(string) != 1:
@@ -330,21 +340,27 @@ def _parse_query_string(string, request_keys, current_dct):
     # like search
     if string.endswith(u'*'):
         string = string[:-1] + u':*'
+    if string.startswith(u'-'):
+        string = u"!" + string[1:]
 
     return string
 
 
-def _parse_parentheses_groups(groups, request_keys, current_dct=None):
+def _parse_parentheses_groups(groups, request_keys, current_dct=None,
+                              exc_dct=None):
     """
     Transform parentheses groups to query
 
     :param groups: groups to transform (list)
     :param request_keys: request keys for facet search
-    :param current_dct:
-    :return: query string, query dict
+    :param current_dct: query dict
+    :param exc_dct: exclude query dict
+    :return: query string, query dict, excluded query dict
     """
     if not current_dct:
         current_dct = {}
+    if not exc_dct:
+        exc_dct = {}
     if type(groups) is not list:
         string = groups.strip()
         # split into many groups if spaces
@@ -369,17 +385,18 @@ def _parse_parentheses_groups(groups, request_keys, current_dct=None):
 
         string_groups = [gp.replace(SEP, u" ") for gp in string.split(u" ")]
         if len(string_groups) == 1:
-            return _parse_query_string(string_groups[0], request_keys,
-                                       current_dct), current_dct
+            return _parse_query_string(
+                string_groups[0], request_keys, current_dct, exc_dct), \
+                current_dct, exc_dct
         return _parse_parentheses_groups(string_groups,
-                                         request_keys, current_dct)
+                                         request_keys, current_dct, exc_dct)
     if not groups:  # empty list
-        return "", current_dct
+        return "", current_dct, exc_dct
     query = u"("
     previous_sep, has_item = None, False
     for item in groups:
-        q, current_dct = _parse_parentheses_groups(item, request_keys,
-                                                   current_dct)
+        q, current_dct, exc_dct = _parse_parentheses_groups(
+            item, request_keys, current_dct, exc_dct)
         q = q.strip()
         if not q:
             continue
@@ -399,24 +416,234 @@ def _parse_parentheses_groups(groups, request_keys, current_dct=None):
     query += u")"
     if query == u"()":
         query = u""
-    return unidecode(query), current_dct
+    return unidecode(query), current_dct, exc_dct
 
 
-def _search_manage_search_vector(dct, request_keys):
+def _search_manage_search_vector(dct, exc_dct, request_keys):
     if 'search_vector' not in dct:
-        return dct
+        return dct, exc_dct
 
     parentheses_groups = _parse_parentheses(dct['search_vector'].strip())
-    search_query, extra_dct = _parse_parentheses_groups(parentheses_groups,
-                                                        request_keys)
+    search_query, extra_dct, extra_exc_dct = _parse_parentheses_groups(
+        parentheses_groups, request_keys)
     dct.update(extra_dct)
+    exc_dct.update(extra_exc_dct)
     if search_query:
         dct['extras'].append(
             {'where': ["search_vector @@ (to_tsquery(%s, %s)) = true"],
              'params': [settings.ISHTAR_SEARCH_LANGUAGE,
                         search_query]}
         )
-    return dct
+    return dct, exc_dct
+
+
+def _manage_bool_fields(model, bool_fields, reversed_bool_fields, dct, or_reqs):
+    bool_fields = list(bool_fields) + list(reversed_bool_fields)
+    for k in bool_fields:
+        if k in dct:
+            if dct[k] == u"1":
+                dct.pop(k)
+            else:
+                dct[k] = dct[k] == u"2" and True or False
+                if k in reversed_bool_fields:
+                    dct[k] = not dct[k]
+                # check also for empty value with image field
+                field_name = k.split('__')[0]
+                # TODO: can be improved in later version of Django
+                try:
+                    c_field = model._meta.get_field(field_name)
+                    if k.endswith('__isnull') and \
+                            isinstance(c_field, ImageField):
+                        if dct[k]:
+                            or_reqs.append(
+                                (k, {k.split('__')[0] + '__exact': ''}))
+                        else:
+                            dct[k.split('__')[0] + '__regex'] = '.{1}.*'
+                except FieldDoesNotExist:
+                    pass
+
+
+def _manage_dated_fields(dated_fields, dct):
+    for k in dated_fields:
+        if k in dct:
+            if not dct[k]:
+                dct.pop(k)
+            try:
+                items = dct[k].split('/')
+                assert len(items) == 3
+                dct[k] = datetime.date(*map(lambda x: int(x),
+                                            reversed(items))) \
+                    .strftime('%Y-%m-%d')
+            except AssertionError:
+                dct.pop(k)
+
+
+def _manage_facet_search(model, dct, and_reqs):
+    general_types = model.general_types()
+    for base_k in general_types:
+        if base_k in HIERARCHIC_FIELDS:  # already managed
+            continue
+        k = base_k + "__pk"
+        if k not in dct or not dct[k].startswith(u'"') \
+                or not dct[k].startswith(u'"'):
+            continue
+        val = dct.pop(k)
+        if u";" in val:
+            # OR request
+            values = val.split(u";")
+        else:
+            values = [val]
+        reqs = None
+        for val in values:
+            if not val.endswith(u'"') or not val.startswith(u""):
+                continue
+            suffix = "__label__icontains" if u"%" in val else \
+                "__label__iexact"
+            query = val[1:-1].replace(u'*', u"")
+            if not reqs:
+                reqs = Q(**{base_k + suffix: query})
+            else:
+                reqs |= Q(**{base_k + suffix: query})
+        if reqs:
+            and_reqs.append(reqs)
+
+
+def _manage_hierarchic_fields(dct, and_reqs):
+    for req in dct.copy():
+        if req.endswith('areas__pk'):
+            val = dct.pop(req)
+            reqs = Q(**{req: val})
+            base_req = req[:-2] + '__'
+            req = base_req[:]
+            for idx in range(HIERARCHIC_LEVELS):
+                req = req[:-2] + 'parent__pk'
+                q = Q(**{req: val})
+                reqs |= q
+            req = base_req[:]
+            for idx in range(HIERARCHIC_LEVELS):
+                req = req[:-2] + 'children__pk'
+                q = Q(**{req: val})
+                reqs |= q
+            and_reqs.append(reqs)
+            continue
+
+        if req.endswith('town__pk') or req.endswith('towns__pk'):
+            val = dct.pop(req)
+            reqs = Q(**{req: val})
+            base_req = req[:-2] + '__'
+            req = base_req[:]
+            for idx in range(HIERARCHIC_LEVELS):
+                req = req[:-2] + 'parents__pk'
+                q = Q(**{req: val})
+                reqs |= q
+            req = base_req[:]
+            for idx in range(HIERARCHIC_LEVELS):
+                req = req[:-2] + 'children__pk'
+                q = Q(**{req: val})
+                reqs |= q
+            and_reqs.append(reqs)
+            continue
+
+        for k_hr in HIERARCHIC_FIELDS:
+            if type(req) in (list, tuple):
+                val = dct.pop(req)
+                q = None
+                for idx, r in enumerate(req):
+                    if not idx:
+                        q = Q(**{r: val})
+                    else:
+                        q |= Q(**{r: val})
+                and_reqs.append(q)
+                break
+            elif req.endswith(k_hr + '__pk'):
+                val = dct.pop(req)
+
+                if u";" in val:
+                    # OR request
+                    values = val.split(u";")
+                else:
+                    values = [val]
+                base_req = req[:]
+                reqs = None
+                for val in values:
+                    suffix = "pk"
+                    req = base_req[:]
+
+                    if val.startswith(u'"') and val.startswith(u'"'):
+                        # manage search text by label
+                        if u"*" in val:
+                            suffix = "label__icontains"
+                            val = val.replace(u'*', u"")
+                        else:
+                            suffix = "label__iexact"
+                        val = val[1:-1]
+                        req = req[:-2] + suffix
+
+                    if not reqs:
+                        reqs = Q(**{req: val})
+                    else:
+                        reqs |= Q(**{req: val})
+                    for idx in range(HIERARCHIC_LEVELS):
+                        req = req[:-(len(suffix))] + 'parent__' + suffix
+                        q = Q(**{req: val})
+                        reqs |= q
+                if reqs:
+                    and_reqs.append(reqs)
+                break
+
+
+def _manage_relation_types(relation_types, dct, query, or_reqs):
+    for rtype_prefix in relation_types:
+        vals = list(relation_types[rtype_prefix])
+        if not vals:
+            continue
+        alt_dct = {
+            rtype_prefix + 'right_relations__relation_type__pk__in': vals}
+        for k in dct:
+            val = dct[k]
+            if rtype_prefix:
+                # only get conditions related to the object
+                if rtype_prefix not in k:
+                    continue
+                # tricky: reconstruct the key to make sense - remove the
+                # prefix from the key
+                k = k[0:k.index(rtype_prefix)] + \
+                    k[k.index(rtype_prefix) + len(rtype_prefix):]
+            if k.endswith('year'):
+                k += '__exact'
+            alt_dct[rtype_prefix + 'right_relations__right_record__' + k] = \
+                val
+        if not dct:
+            # fake condition to trick Django (1.4): without it only the
+            # alt_dct is managed
+            query &= Q(pk__isnull=False)
+        query |= Q(**alt_dct)
+        for k, or_req in or_reqs:
+            altor_dct = alt_dct.copy()
+            altor_dct.pop(k)
+            for j in or_req:
+                val = or_req[j]
+                if j == 'year':
+                    j = 'year__exact'
+                altor_dct[
+                    rtype_prefix + 'right_relations__right_record__' + j] = \
+                    val
+            query |= Q(**altor_dct)
+    return query
+
+
+def _contruct_query(relation_types, dct, or_reqs, and_reqs):
+    query = Q(**dct)
+    for k, or_req in or_reqs:
+        alt_dct = dct.copy()
+        alt_dct.pop(k)
+        alt_dct.update(or_req)
+        query |= Q(**alt_dct)
+    query = _manage_relation_types(relation_types, dct, query, or_reqs)
+
+    for and_req in and_reqs:
+        query = query & and_req
+    return query
 
 
 def _format_val(val):
@@ -486,6 +713,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                 request.session['SHORTCUT_SEARCH'] == 'own':
             own = True
 
+        query_own = None
+        if own:
+            q = models.IshtarUser.objects.filter(user_ptr=request.user)
+            if not q.count():
+                return HttpResponse(EMPTY, content_type='text/plain')
+            query_own = model.get_query_owns(q.all()[0])
+
         # get defaults from model
         if not extra_request_keys and hasattr(model, 'EXTRA_REQUEST_KEYS'):
             my_extra_request_keys = copy(model.EXTRA_REQUEST_KEYS)
@@ -524,8 +758,6 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
         else:
             my_relation_types_prefix = copy(relation_types_prefix)
 
-        general_types = model.general_types()
-
         fields = [model._meta.get_field(k)
                   for k in get_all_field_names(model)]
 
@@ -569,9 +801,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
         request_items = dct_request_items
 
         dct = my_base_request
+        excluded_dct = {}
+        and_reqs, or_reqs = [], []
+        exc_and_reqs, exc_or_reqs = [], []
+
         if full == 'shortcut':
             dct['cached_label__icontains'] = request.GET.get('term', None)
-        and_reqs, or_reqs = [], []
+
         try:
             old = 'old' in request_items and int(request_items['old'])
         except ValueError:
@@ -649,211 +885,36 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
             request.session[func_name] = dct
 
         dct['extras'] = []
-        dct = _search_manage_search_vector(dct, request_keys)
+        dct, excluded_dct = _search_manage_search_vector(dct, excluded_dct,
+                                                         request_keys)
         search_vector = ""
         if 'search_vector' in dct:
             search_vector = dct.pop('search_vector')
 
-        for k in (list(my_bool_fields) + list(my_reversed_bool_fields)):
-            if k in dct:
-                if dct[k] == u"1":
-                    dct.pop(k)
-                else:
-                    dct[k] = dct[k] == u"2" and True or False
-                    if k in my_reversed_bool_fields:
-                        dct[k] = not dct[k]
-                    # check also for empty value with image field
-                    field_name = k.split('__')[0]
-                    # TODO: can be improved in later version of Django
-                    try:
-                        c_field = model._meta.get_field(field_name)
-                        if k.endswith('__isnull') and \
-                                isinstance(c_field, ImageField):
-                            if dct[k]:
-                                or_reqs.append(
-                                    (k, {k.split('__')[0] + '__exact': ''}))
-                            else:
-                                dct[k.split('__')[0] + '__regex'] = '.{1}.*'
-                    except FieldDoesNotExist:
-                        pass
-        for k in my_dated_fields:
-            if k in dct:
-                if not dct[k]:
-                    dct.pop(k)
-                try:
-                    items = dct[k].split('/')
-                    assert len(items) == 3
-                    dct[k] = datetime.date(*map(lambda x: int(x),
-                                                reversed(items))) \
-                        .strftime('%Y-%m-%d')
-                except AssertionError:
-                    dct.pop(k)
-        # manage hierarchic conditions
-        for req in dct.copy():
-            if req.endswith('areas__pk'):
-                val = dct.pop(req)
-                reqs = Q(**{req: val})
-                base_req = req[:-2] + '__'
-                req = base_req[:]
-                for idx in range(HIERARCHIC_LEVELS):
-                    req = req[:-2] + 'parent__pk'
-                    q = Q(**{req: val})
-                    reqs |= q
-                req = base_req[:]
-                for idx in range(HIERARCHIC_LEVELS):
-                    req = req[:-2] + 'children__pk'
-                    q = Q(**{req: val})
-                    reqs |= q
-                and_reqs.append(reqs)
-                continue
-
-            if req.endswith('town__pk') or req.endswith('towns__pk'):
-                val = dct.pop(req)
-                reqs = Q(**{req: val})
-                base_req = req[:-2] + '__'
-                req = base_req[:]
-                for idx in range(HIERARCHIC_LEVELS):
-                    req = req[:-2] + 'parents__pk'
-                    q = Q(**{req: val})
-                    reqs |= q
-                req = base_req[:]
-                for idx in range(HIERARCHIC_LEVELS):
-                    req = req[:-2] + 'children__pk'
-                    q = Q(**{req: val})
-                    reqs |= q
-                and_reqs.append(reqs)
-                continue
-
-            for k_hr in HIERARCHIC_FIELDS:
-                if type(req) in (list, tuple):
-                    val = dct.pop(req)
-                    q = None
-                    for idx, r in enumerate(req):
-                        if not idx:
-                            q = Q(**{r: val})
-                        else:
-                            q |= Q(**{r: val})
-                    and_reqs.append(q)
-                    break
-                elif req.endswith(k_hr + '__pk'):
-                    val = dct.pop(req)
+        _manage_bool_fields(model, my_bool_fields, my_reversed_bool_fields,
+                            dct, or_reqs)
+        _manage_bool_fields(model, my_bool_fields, my_reversed_bool_fields,
+                            excluded_dct, exc_or_reqs)
 
-                    if u";" in val:
-                        # OR request
-                        values = val.split(u";")
-                    else:
-                        values = [val]
-                    base_req = req[:]
-                    reqs = None
-                    for val in values:
-                        suffix = "pk"
-                        req = base_req[:]
-
-                        if val.startswith(u'"') and val.startswith(u'"'):
-                            # manage search text by label
-                            if u"*" in val:
-                                suffix = "label__icontains"
-                                val = val.replace(u'*', u"")
-                            else:
-                                suffix = "label__iexact"
-                            val = val[1:-1]
-                            req = req[:-2] + suffix
+        _manage_dated_fields(my_dated_fields, dct)
+        _manage_dated_fields(my_dated_fields, excluded_dct)
 
-                        if not reqs:
-                            reqs = Q(**{req: val})
-                        else:
-                            reqs |= Q(**{req: val})
-                        for idx in range(HIERARCHIC_LEVELS):
-                            req = req[:-(len(suffix))] + 'parent__' + suffix
-                            q = Q(**{req: val})
-                            reqs |= q
-                    if reqs:
-                        and_reqs.append(reqs)
-                    break
+        _manage_hierarchic_fields(dct, and_reqs)
+        _manage_hierarchic_fields(excluded_dct, exc_and_reqs)
 
-        # manage search text by label
-        for base_k in general_types:
-            if base_k in HIERARCHIC_FIELDS:
-                continue
-            k = base_k + "__pk"
-            if k not in dct or not dct[k].startswith(u'"') \
-                    or not dct[k].startswith(u'"'):
-                continue
-            val = dct.pop(k)
-            if u";" in val:
-                # OR request
-                values = val.split(u";")
-            else:
-                values = [val]
-            reqs = None
-            for val in values:
-                if not val.endswith(u'"') or not val.startswith(u""):
-                    continue
-                suffix = "__label__icontains" if u"%" in val else \
-                    "__label__iexact"
-                query = val[1:-1].replace(u'*', u"")
-                if not reqs:
-                    reqs = Q(**{base_k + suffix: query})
-                else:
-                    reqs |= Q(**{base_k + suffix: query})
-            if reqs:
-                and_reqs.append(reqs)
+        _manage_facet_search(model, dct, and_reqs)
+        _manage_facet_search(model, excluded_dct, exc_and_reqs)
 
         extras = dct.pop('extras')
 
-        query = Q(**dct)
-        for k, or_req in or_reqs:
-            alt_dct = dct.copy()
-            alt_dct.pop(k)
-            alt_dct.update(or_req)
-            query |= Q(**alt_dct)
-
-        for rtype_prefix in relation_types:
-            vals = list(relation_types[rtype_prefix])
-            if not vals:
-                continue
-            alt_dct = {
-                rtype_prefix + 'right_relations__relation_type__pk__in': vals}
-            for k in dct:
-                val = dct[k]
-                if rtype_prefix:
-                    # only get conditions related to the object
-                    if rtype_prefix not in k:
-                        continue
-                    # tricky: reconstruct the key to make sense - remove the
-                    # prefix from the key
-                    k = k[0:k.index(rtype_prefix)] + k[
-                                                     k.index(rtype_prefix) + len(rtype_prefix):]
-                if k.endswith('year'):
-                    k += '__exact'
-                alt_dct[rtype_prefix + 'right_relations__right_record__' + k] = \
-                    val
-            if not dct:
-                # fake condition to trick Django (1.4): without it only the
-                # alt_dct is managed
-                query &= Q(pk__isnull=False)
-            query |= Q(**alt_dct)
-            for k, or_req in or_reqs:
-                altor_dct = alt_dct.copy()
-                altor_dct.pop(k)
-                for j in or_req:
-                    val = or_req[j]
-                    if j == 'year':
-                        j = 'year__exact'
-                    altor_dct[
-                        rtype_prefix + 'right_relations__right_record__' + j] = \
-                        val
-                query |= Q(**altor_dct)
-
-        if own:
-            q = models.IshtarUser.objects.filter(user_ptr=request.user)
-            if q.count():
-                query = query & model.get_query_owns(q.all()[0])
-            else:
-                return HttpResponse(EMPTY, content_type='text/plain')
+        query = _contruct_query(relation_types, dct, or_reqs, and_reqs)
+        exc_query = None
+        if excluded_dct or exc_and_reqs or exc_or_reqs:
+            exc_query = _contruct_query(
+                relation_types, excluded_dct, exc_or_reqs, exc_and_reqs)
 
-        for and_req in and_reqs:
-            query = query & and_req
+        if query_own:
+            query = query & query_own
 
         # manage hierarchic in shortcut menu
         if full == 'shortcut':
@@ -872,8 +933,12 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                     query &= Q(**dct)
 
         items = model.objects.filter(query)
+        if exc_query:
+            items = items.exclude(exc_query)
+
         for extra in extras:
             items = items.extra(**extra)
+
         items = items.distinct()
         # print(items.query)