diff options
| -rw-r--r-- | archaeological_operations/tests.py | 65 | ||||
| -rw-r--r-- | ishtar_common/views_item.py | 117 |
2 files changed, 170 insertions, 12 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 9f08f1e48..381efd070 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -1456,8 +1456,11 @@ class OperationSearchTest(TestCase, OperationInitTest): profile.areas.add(area) self.orgas = self.create_orgas(self.user) - self.operations = self.create_operation(self.user, self.orgas[0]) - self.operations += self.create_operation(self.alt_user, self.orgas[0]) + self.create_operation(self.user, self.orgas[0]) + self.create_operation(self.alt_user, self.orgas[0]) + self.operations = self.create_operation(self.alt_user, self.orgas[0]) + self.operations[2].year = 2018 + self.operations[2].save() self.item = self.operations[0] def test_base_search(self): @@ -1471,11 +1474,11 @@ class OperationSearchTest(TestCase, OperationInitTest): response = c.get(reverse('get-operation'), {'operator': self.orgas[0].pk}) result = json.loads(response.content) - self.assertEqual(result['recordsTotal'], 2) + self.assertEqual(result['recordsTotal'], 3) - def test_search_vector(self): + def test_base_search_vector(self): c = Client() - response = c.get(reverse('get-operation'), {'year': '2010'}) + response = c.get(reverse('get-operation'), {'search_vector': 'chaTEAU'}) # no result when no authentication self.assertTrue(not json.loads(response.content)) c.login(username=self.username, password=self.password) @@ -1490,6 +1493,56 @@ class OperationSearchTest(TestCase, OperationInitTest): result = json.loads(response.content) self.assertEqual(result['recordsTotal'], 1) + def test_complex_search_vector(self): + c = Client() + c.login(username=self.username, password=self.password) + operation_1 = models.Operation.objects.get(pk=self.operations[0].pk) + operation_2 = models.Operation.objects.get(pk=self.operations[1].pk) + operation_3 = models.Operation.objects.get(pk=self.operations[2].pk) + operation_1.common_name = u"Opération : Château de Fougères" + operation_1.save() + operation_2.common_name = u"Opération : Fougère filicophyta et " \ + u"herbe à chat" + operation_2.save() + operation_3.common_name = u"Opération : Château Filicophyta" + operation_3.save() + + # simple separation + response = c.get(reverse('get-operation'), + {'search_vector': 'chaTEAU fougere'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 1) + + # explicit AND + response = c.get(reverse('get-operation'), + {'search_vector': 'chaTEAU & fougere'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 1) + + # explicit OR + response = c.get(reverse('get-operation'), + {'search_vector': 'chaTEAU | fougere'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 3) + + # query with parenthesis + response = c.get(reverse('get-operation'), + {'search_vector': '2010 & (fougere | filicophyta)'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 2) + + # query with mistmatch parenthesis + response = c.get(reverse('get-operation'), + {'search_vector': ')) 2010 &) ((chaTEAU | fougere)'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 2) + + # open search + response = c.get(reverse('get-operation'), + {'search_vector': 'cha*'}) + result = json.loads(response.content) + self.assertEqual(result['recordsTotal'], 3) + def create_relations(self): rel1 = models.RelationType.objects.create( symmetrical=True, label='Include', txt_idx='include') @@ -1513,7 +1566,7 @@ class OperationSearchTest(TestCase, OperationInitTest): self.assertTrue(not json.loads(response.content)) c.login(username=self.username, password=self.password) response = c.get(reverse('get-operation'), search) - self.assertTrue(json.loads(response.content)['recordsTotal'] == 2) + self.assertEqual(json.loads(response.content)['recordsTotal'], 2) def testHierarchicSearch(self): ope = self.operations[1] diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py index 513035903..f5e47a832 100644 --- a/ishtar_common/views_item.py +++ b/ishtar_common/views_item.py @@ -268,11 +268,107 @@ def _get_values(request, val): return new_vals +def _push_to_list(obj, current_group, depth): + """ + parse_parentheses helper function + """ + try: + while depth > 0: + current_group = current_group[-1] + depth -= 1 + except IndexError: + # tolerant to parentheses mismatch + pass + if current_group and type(obj) in (unicode, str) and \ + type(current_group[-1]) in (unicode, str): + current_group[-1] += obj + else: + current_group.append(obj) + + +def _parse_parentheses(s): + """ + Parse parentheses into list. + (OA01 & (pierre | ciseau)) -> ["0A01 &", ["pierre | ciseau"]] + """ + + groups = [] + depth = 0 + + for char in s: + if char == u'(': + _push_to_list([], groups, depth) + depth += 1 + elif char == ')': + if depth > 0: + depth -= 1 + else: + _push_to_list(char, groups, depth) + # for non tolerant to parentheses mismatch check depth is equal to 0 + return groups + + +FORBIDDEN_CHAR = [u":"] +RESERVED_CHAR = [u"|", u"&"] + + +def _parse_query_string(string): + string = string.strip().lower() + for reserved_char in FORBIDDEN_CHAR: + string = string.replace(reserved_char, u"") + if len(string) != 1: + for reserved_char in RESERVED_CHAR: + string = string.replace(reserved_char, u"") + # like search + if string.endswith(u'*'): + string = string[:-1] + u':*' + + return string + + +def _parse_parentheses_groups(groups): + """ + Transform parentheses groups to query + """ + if type(groups) is not list: + string = groups.strip() + # split into many groups if spaces + if ' ' not in string: + return _parse_query_string(groups) + return _parse_parentheses_groups(string.split(u" ")) + if not groups: # empty list + return "" + query = u"(" + previous_sep, has_item = None, False + for item in groups: + q = _parse_parentheses_groups(item).strip() + if not q: + continue + if q in (u"|", u"&"): + if previous_sep or not has_item: + continue # multiple sep is not relevant + previous_sep = q + continue + if has_item: + if previous_sep: + query += previous_sep + else: + query += u" & " + query += q + has_item = True + previous_sep = None + query += u")" + return unidecode(query) + + def _search_manage_search_vector(dct): if 'search_vector' in dct: - dct['search_vector'] = SearchQuery( - unidecode(dct['search_vector']), - config=settings.ISHTAR_SEARCH_LANGUAGE + parentheses_groups = _parse_parentheses(dct['search_vector'].strip()) + query = _parse_parentheses_groups(parentheses_groups) + dct['extras'].append( + {'where': ["search_vector @@ (to_tsquery(%s, %s)) = true"], + 'params': [settings.ISHTAR_SEARCH_LANGUAGE, + query]} ) return dct @@ -594,7 +690,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[], reqs |= q and_reqs.append(reqs) break + dct['extras'] = [] dct = _search_manage_search_vector(dct) + search_vector = "" + if 'search_vector' in dct: + search_vector = dct.pop('search_vector') + extras = dct.pop('extras') + query = Q(**dct) for k, or_req in or_reqs: alt_dct = dct.copy() @@ -665,11 +767,14 @@ def get_item(model, func_name, default_name, extra_request_keys=[], dct = {upper_key: current} query &= Q(**dct) - items = model.objects.filter(query).distinct() + items = model.objects.filter(query) + for extra in extras: + items = items.extra(**extra) + items = items.distinct() # print(items.query) - if 'search_vector' in dct: # for serialization - dct['search_vector'] = dct['search_vector'].value + if search_vector: # for serialization + dct['search_vector'] = search_vector # table cols if own_table_cols: |