diff options
| -rw-r--r-- | archaeological_operations/forms.py | 11 | ||||
| -rw-r--r-- | archaeological_operations/models.py | 13 | ||||
| -rw-r--r-- | archaeological_operations/tests.py | 59 | ||||
| -rw-r--r-- | ishtar_common/forms.py | 10 | ||||
| -rw-r--r-- | ishtar_common/models.py | 12 | ||||
| -rw-r--r-- | ishtar_common/views_item.py | 153 |
6 files changed, 220 insertions, 38 deletions
diff --git a/archaeological_operations/forms.py b/archaeological_operations/forms.py index cbaa37310..4e01a492a 100644 --- a/archaeological_operations/forms.py +++ b/archaeological_operations/forms.py @@ -472,6 +472,8 @@ RecordRelationsFormSet.form_slug = "operation-080-relations" class OperationSelect(TableSelect): + _model = models.Operation + search_vector = forms.CharField(label=_(u"Full text search"), widget=widgets.SearchWidget) year = forms.IntegerField(label=_("Year")) @@ -486,12 +488,9 @@ class OperationSelect(TableSelect): if settings.ISHTAR_DPTS: towns__numero_insee__startswith = forms.ChoiceField( label=_(u"Department"), choices=[]) - common_name = forms.CharField(label=_(u"Name"), - max_length=30) - address = forms.CharField(label=_(u"Address / Locality"), - max_length=100) - operation_type = forms.ChoiceField(label=_(u"Operation type"), - choices=[]) + common_name = forms.CharField(label=_(u"Name"), max_length=30) + address = forms.CharField(label=_(u"Address / Locality"), max_length=100) + operation_type = forms.ChoiceField(label=_(u"Operation type"), choices=[]) end_date = forms.NullBooleanField(label=_(u"Is open?")) in_charge = forms.IntegerField( widget=widgets.JQueryAutoComplete( diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py index d50c9e43c..688c12bea 100644 --- a/archaeological_operations/models.py +++ b/archaeological_operations/models.py @@ -27,7 +27,7 @@ from django.db import IntegrityError, transaction from django.db.models import Q, Count, Sum, Max, Avg from django.db.models.signals import post_save, m2m_changed, post_delete from django.forms import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, pgettext_lazy from ishtar_common.models import BaseHistorizedItem, Dashboard, \ DashboardFormItem, Department, Document, DocumentTemplate, \ @@ -362,6 +362,17 @@ class Operation(ClosedItem, BaseHistorizedItem, OwnPerms, ValueGetter, }, } + # alternative names of fields for searches + ALT_NAMES = { + 'periods__pk': pgettext_lazy( + "key for text search (no accent, no spaces)", u"period"), + 'operation_type__pk': pgettext_lazy( + "key for text search (no accent, no spaces)", u"operation-type"), + 'remains__pk': pgettext_lazy( + "key for text search (no accent, no spaces)", u"remain"), + } + EXTRA_REQUEST_KEYS.update(dict([(v, k) for k, v in ALT_NAMES.items()])) + # fields definition creation_date = models.DateField(_(u"Creation date"), default=datetime.date.today) diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 381efd070..1480eb502 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -33,7 +33,7 @@ from django.db.models import Q from django.test.client import Client from django.contrib.auth.models import User, Permission -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, pgettext import models from archaeological_operations import views @@ -1538,11 +1538,64 @@ class OperationSearchTest(TestCase, OperationInitTest): self.assertEqual(result['recordsTotal'], 2) # open search - response = c.get(reverse('get-operation'), - {'search_vector': 'cha*'}) + response = c.get(reverse('get-operation'), {'search_vector': 'cha*'}) result = json.loads(response.content) self.assertEqual(result['recordsTotal'], 3) + def test_facet_search_vector(self): + ope1 = self.operations[0] + ope2 = self.operations[1] + ope3 = self.operations[2] + c = Client() + c.login(username=self.username, password=self.password) + + neo = models.Period.objects.get(txt_idx='neolithic') + final_neo = models.Period.objects.get(txt_idx='final-neolithic') + gallo = models.Period.objects.get(txt_idx="gallo-roman") + ope1.periods.add(final_neo) + ope1.periods.add(gallo) + ope2.periods.add(neo) + ope3.periods.add(gallo) + + villa = models.RemainType.objects.get(txt_idx='villa') + ope1.remains.add(villa) + + # simple + search_q = unicode( + pgettext("key for text search (no accent, no spaces)", u"period") + ) + search = {'search_vector': u'{}="{}"'.format(search_q, final_neo.label)} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 1) + + # hierarchic + search = {'search_vector': u'{}="{}"'.format(search_q, neo.label)} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 2) + + # OR + search = {'search_vector': u'{}="{}";"{}"'.format(search_q, neo.label, + gallo.label)} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 3) + + # non hierarchic search + search_q = unicode( + pgettext("key for text search (no accent, no spaces)", u"remain") + ) + search = {'search_vector': u'{}="{}"'.format(search_q, villa.label)} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 1) + + def create_relations(self): rel1 = models.RelationType.objects.create( symmetrical=True, label='Include', txt_idx='include') diff --git a/ishtar_common/forms.py b/ishtar_common/forms.py index e01e74a14..bb468eff1 100644 --- a/ishtar_common/forms.py +++ b/ishtar_common/forms.py @@ -461,13 +461,19 @@ class IshtarForm(forms.Form): class TableSelect(IshtarForm): def __init__(self, *args, **kwargs): super(TableSelect, self).__init__(*args, **kwargs) - # no field is required for search + ALT_NAMES = {} + if hasattr(self, '_model') and hasattr(self._model, "ALT_NAMES"): + ALT_NAMES = self._model.ALT_NAMES for k in self.fields: - self.fields[k].required = False + self.fields[k].required = False # no field is required for search cls = 'form-control' if k == 'search_vector': cls += " search-vector" self.fields[k].widget.attrs['class'] = cls + if k in ALT_NAMES: + self.fields[k].alt_name = ALT_NAMES[k] + else: + self.fields[k].alt_name = k key = self.fields.keys()[0] self.fields[key].widget.attrs['autofocus'] = 'autofocus' diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 86b74693f..d65c08f93 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1094,6 +1094,16 @@ class FullSearch(models.Model): class Meta: abstract = True + @classmethod + def general_types(cls): + for k in get_all_field_names(cls): + field = cls._meta.get_field(k) + if not hasattr(field, 'rel') or not field.rel: + continue + rel_model = field.rel.to + if issubclass(rel_model, (GeneralType, HierarchicalType)): + yield k + def update_search_vector(self, save=True): """ Update the search vector @@ -1275,7 +1285,7 @@ class BaseHistorizedItem(DocumentItem, FullSearch, Imported, JsonData, FixAssociated): """ Historized item with external ID management. - All historized items are searcheable and have a data json field + All historized items are searchable and have a data json field. """ IS_BASKET = False EXTERNAL_ID_KEY = '' diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py index f5e47a832..a2cc0762c 100644 --- a/ishtar_common/views_item.py +++ b/ishtar_common/views_item.py @@ -312,8 +312,16 @@ FORBIDDEN_CHAR = [u":"] RESERVED_CHAR = [u"|", u"&"] -def _parse_query_string(string): +def _parse_query_string(string, request_keys, current_dct): string = string.strip().lower() + + if u"=" in string: + splited = string.split(u"=") + if len(splited) == 2 and splited[0] in request_keys: + term, query = splited + term = request_keys[term] + current_dct[term] = query + return "" for reserved_char in FORBIDDEN_CHAR: string = string.replace(reserved_char, u"") if len(string) != 1: @@ -326,22 +334,53 @@ def _parse_query_string(string): return string -def _parse_parentheses_groups(groups): +def _parse_parentheses_groups(groups, request_keys, current_dct=None): """ Transform parentheses groups to query + + :param groups: groups to transform (list) + :param request_keys: request keys for facet search + :param current_dct: + :return: query string, query dict """ + if not current_dct: + current_dct = {} if type(groups) is not list: string = groups.strip() # split into many groups if spaces - if ' ' not in string: - return _parse_query_string(groups) - return _parse_parentheses_groups(string.split(u" ")) + + # do not split inside quotes + current_index = 0 + found = string.find('"', current_index) + SEP = u"?ç;?" # replace spaces inside quote with this characters + previous_quote = None + while found != -1: + if previous_quote: + string = string[0:previous_quote] + \ + string[previous_quote:found].replace(u' ', SEP) + \ + string[found:] + previous_quote = None + # SEP is larger than a space + found = string.find('"', current_index) + else: + previous_quote = found + current_index = found + 1 + found = string.find('"', current_index) + + string_groups = [gp.replace(SEP, u" ") for gp in string.split(u" ")] + if len(string_groups) == 1: + return _parse_query_string(string_groups[0], request_keys, + current_dct), current_dct + return _parse_parentheses_groups(string_groups, + request_keys, current_dct) if not groups: # empty list - return "" + return "", current_dct query = u"(" previous_sep, has_item = None, False for item in groups: - q = _parse_parentheses_groups(item).strip() + q, current_dct = _parse_parentheses_groups(item, request_keys, + current_dct) + q = q.strip() if not q: continue if q in (u"|", u"&"): @@ -358,17 +397,24 @@ def _parse_parentheses_groups(groups): has_item = True previous_sep = None query += u")" - return unidecode(query) + if query == u"()": + query = u"" + return unidecode(query), current_dct -def _search_manage_search_vector(dct): - if 'search_vector' in dct: - parentheses_groups = _parse_parentheses(dct['search_vector'].strip()) - query = _parse_parentheses_groups(parentheses_groups) +def _search_manage_search_vector(dct, request_keys): + if 'search_vector' not in dct: + return dct + + parentheses_groups = _parse_parentheses(dct['search_vector'].strip()) + search_query, extra_dct = _parse_parentheses_groups(parentheses_groups, + request_keys) + dct.update(extra_dct) + if search_query: dct['extras'].append( {'where': ["search_vector @@ (to_tsquery(%s, %s)) = true"], 'params': [settings.ISHTAR_SEARCH_LANGUAGE, - query]} + search_query]} ) return dct @@ -478,6 +524,8 @@ def get_item(model, func_name, default_name, extra_request_keys=[], else: my_relation_types_prefix = copy(relation_types_prefix) + general_types = model.general_types() + fields = [model._meta.get_field(k) for k in get_all_field_names(model)] @@ -599,6 +647,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[], dct = request.session[func_name] else: request.session[func_name] = dct + + dct['extras'] = [] + dct = _search_manage_search_vector(dct, request_keys) + search_vector = "" + if 'search_vector' in dct: + search_vector = dct.pop('search_vector') + for k in (list(my_bool_fields) + list(my_reversed_bool_fields)): if k in dct: if dct[k] == u"1": @@ -682,19 +737,67 @@ def get_item(model, func_name, default_name, extra_request_keys=[], break elif req.endswith(k_hr + '__pk'): val = dct.pop(req) - reqs = Q(**{req: val}) - req = req[:-2] + '__' - for idx in range(HIERARCHIC_LEVELS): - req = req[:-2] + 'parent__pk' - q = Q(**{req: val}) - reqs |= q - and_reqs.append(reqs) + + if u";" in val: + # OR request + values = val.split(u";") + else: + values = [val] + base_req = req[:] + reqs = None + for val in values: + suffix = "pk" + req = base_req[:] + + if val.startswith(u'"') and val.startswith(u'"'): + # manage search text by label + if u"%" in val: + suffix = "label__icontains" + else: + suffix = "label__iexact" + val = val[1:-1] + req = req[:-2] + suffix + + if not reqs: + reqs = Q(**{req: val}) + else: + reqs |= Q(**{req: val}) + for idx in range(HIERARCHIC_LEVELS): + req = req[:-(len(suffix))] + 'parent__' + suffix + q = Q(**{req: val}) + reqs |= q + if reqs: + and_reqs.append(reqs) break - dct['extras'] = [] - dct = _search_manage_search_vector(dct) - search_vector = "" - if 'search_vector' in dct: - search_vector = dct.pop('search_vector') + + # manage search text by label + for base_k in general_types: + if base_k in HIERARCHIC_FIELDS: + continue + k = base_k + "__pk" + if k not in dct or not dct[k].startswith(u'"') \ + or not dct[k].startswith(u'"'): + continue + val = dct.pop(k) + if u";" in val: + # OR request + values = val.split(u";") + else: + values = [val] + reqs = None + for val in values: + if not val.endswith(u'"') or not val.startswith(u""): + continue + query = val[1:-1] + suffix = "__label__icontains" if u"%" in val else \ + "__label__iexact" + if not reqs: + reqs = Q(**{base_k + suffix: query}) + else: + reqs |= Q(**{base_k + suffix: query}) + if reqs: + and_reqs.append(reqs) + extras = dct.pop('extras') query = Q(**dct) |