summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorÉtienne Loks <etienne.loks@iggdrasil.net>2018-06-29 18:45:21 +0200
committerÉtienne Loks <etienne.loks@iggdrasil.net>2018-08-13 18:26:03 +0200
commit9de2c94a7a528e1ae24bc2a0a9bb9354329d0a93 (patch)
tree3acc5fe0568d4bc91e6f38abbadff7e05af35aa7
parentf94890fe2d2ab241ee1cd5b980bf88409b47a998 (diff)
downloadIshtar-9de2c94a7a528e1ae24bc2a0a9bb9354329d0a93.tar.bz2
Ishtar-9de2c94a7a528e1ae24bc2a0a9bb9354329d0a93.zip
Searches: manage AND, OR, parentheses and open search '*' (refs #4180)
-rw-r--r--archaeological_operations/tests.py65
-rw-r--r--ishtar_common/views_item.py117
2 files changed, 170 insertions, 12 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 9f08f1e48..381efd070 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -1456,8 +1456,11 @@ class OperationSearchTest(TestCase, OperationInitTest):
profile.areas.add(area)
self.orgas = self.create_orgas(self.user)
- self.operations = self.create_operation(self.user, self.orgas[0])
- self.operations += self.create_operation(self.alt_user, self.orgas[0])
+ self.create_operation(self.user, self.orgas[0])
+ self.create_operation(self.alt_user, self.orgas[0])
+ self.operations = self.create_operation(self.alt_user, self.orgas[0])
+ self.operations[2].year = 2018
+ self.operations[2].save()
self.item = self.operations[0]
def test_base_search(self):
@@ -1471,11 +1474,11 @@ class OperationSearchTest(TestCase, OperationInitTest):
response = c.get(reverse('get-operation'),
{'operator': self.orgas[0].pk})
result = json.loads(response.content)
- self.assertEqual(result['recordsTotal'], 2)
+ self.assertEqual(result['recordsTotal'], 3)
- def test_search_vector(self):
+ def test_base_search_vector(self):
c = Client()
- response = c.get(reverse('get-operation'), {'year': '2010'})
+ response = c.get(reverse('get-operation'), {'search_vector': 'chaTEAU'})
# no result when no authentication
self.assertTrue(not json.loads(response.content))
c.login(username=self.username, password=self.password)
@@ -1490,6 +1493,56 @@ class OperationSearchTest(TestCase, OperationInitTest):
result = json.loads(response.content)
self.assertEqual(result['recordsTotal'], 1)
+ def test_complex_search_vector(self):
+ c = Client()
+ c.login(username=self.username, password=self.password)
+ operation_1 = models.Operation.objects.get(pk=self.operations[0].pk)
+ operation_2 = models.Operation.objects.get(pk=self.operations[1].pk)
+ operation_3 = models.Operation.objects.get(pk=self.operations[2].pk)
+ operation_1.common_name = u"Opération : Château de Fougères"
+ operation_1.save()
+ operation_2.common_name = u"Opération : Fougère filicophyta et " \
+ u"herbe à chat"
+ operation_2.save()
+ operation_3.common_name = u"Opération : Château Filicophyta"
+ operation_3.save()
+
+ # simple separation
+ response = c.get(reverse('get-operation'),
+ {'search_vector': 'chaTEAU fougere'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 1)
+
+ # explicit AND
+ response = c.get(reverse('get-operation'),
+ {'search_vector': 'chaTEAU & fougere'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 1)
+
+ # explicit OR
+ response = c.get(reverse('get-operation'),
+ {'search_vector': 'chaTEAU | fougere'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 3)
+
+ # query with parenthesis
+ response = c.get(reverse('get-operation'),
+ {'search_vector': '2010 & (fougere | filicophyta)'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 2)
+
+ # query with mistmatch parenthesis
+ response = c.get(reverse('get-operation'),
+ {'search_vector': ')) 2010 &) ((chaTEAU | fougere)'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 2)
+
+ # open search
+ response = c.get(reverse('get-operation'),
+ {'search_vector': 'cha*'})
+ result = json.loads(response.content)
+ self.assertEqual(result['recordsTotal'], 3)
+
def create_relations(self):
rel1 = models.RelationType.objects.create(
symmetrical=True, label='Include', txt_idx='include')
@@ -1513,7 +1566,7 @@ class OperationSearchTest(TestCase, OperationInitTest):
self.assertTrue(not json.loads(response.content))
c.login(username=self.username, password=self.password)
response = c.get(reverse('get-operation'), search)
- self.assertTrue(json.loads(response.content)['recordsTotal'] == 2)
+ self.assertEqual(json.loads(response.content)['recordsTotal'], 2)
def testHierarchicSearch(self):
ope = self.operations[1]
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 513035903..f5e47a832 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -268,11 +268,107 @@ def _get_values(request, val):
return new_vals
+def _push_to_list(obj, current_group, depth):
+ """
+ parse_parentheses helper function
+ """
+ try:
+ while depth > 0:
+ current_group = current_group[-1]
+ depth -= 1
+ except IndexError:
+ # tolerant to parentheses mismatch
+ pass
+ if current_group and type(obj) in (unicode, str) and \
+ type(current_group[-1]) in (unicode, str):
+ current_group[-1] += obj
+ else:
+ current_group.append(obj)
+
+
+def _parse_parentheses(s):
+ """
+ Parse parentheses into list.
+ (OA01 & (pierre | ciseau)) -> ["0A01 &", ["pierre | ciseau"]]
+ """
+
+ groups = []
+ depth = 0
+
+ for char in s:
+ if char == u'(':
+ _push_to_list([], groups, depth)
+ depth += 1
+ elif char == ')':
+ if depth > 0:
+ depth -= 1
+ else:
+ _push_to_list(char, groups, depth)
+ # for non tolerant to parentheses mismatch check depth is equal to 0
+ return groups
+
+
+FORBIDDEN_CHAR = [u":"]
+RESERVED_CHAR = [u"|", u"&"]
+
+
+def _parse_query_string(string):
+ string = string.strip().lower()
+ for reserved_char in FORBIDDEN_CHAR:
+ string = string.replace(reserved_char, u"")
+ if len(string) != 1:
+ for reserved_char in RESERVED_CHAR:
+ string = string.replace(reserved_char, u"")
+ # like search
+ if string.endswith(u'*'):
+ string = string[:-1] + u':*'
+
+ return string
+
+
+def _parse_parentheses_groups(groups):
+ """
+ Transform parentheses groups to query
+ """
+ if type(groups) is not list:
+ string = groups.strip()
+ # split into many groups if spaces
+ if ' ' not in string:
+ return _parse_query_string(groups)
+ return _parse_parentheses_groups(string.split(u" "))
+ if not groups: # empty list
+ return ""
+ query = u"("
+ previous_sep, has_item = None, False
+ for item in groups:
+ q = _parse_parentheses_groups(item).strip()
+ if not q:
+ continue
+ if q in (u"|", u"&"):
+ if previous_sep or not has_item:
+ continue # multiple sep is not relevant
+ previous_sep = q
+ continue
+ if has_item:
+ if previous_sep:
+ query += previous_sep
+ else:
+ query += u" & "
+ query += q
+ has_item = True
+ previous_sep = None
+ query += u")"
+ return unidecode(query)
+
+
def _search_manage_search_vector(dct):
if 'search_vector' in dct:
- dct['search_vector'] = SearchQuery(
- unidecode(dct['search_vector']),
- config=settings.ISHTAR_SEARCH_LANGUAGE
+ parentheses_groups = _parse_parentheses(dct['search_vector'].strip())
+ query = _parse_parentheses_groups(parentheses_groups)
+ dct['extras'].append(
+ {'where': ["search_vector @@ (to_tsquery(%s, %s)) = true"],
+ 'params': [settings.ISHTAR_SEARCH_LANGUAGE,
+ query]}
)
return dct
@@ -594,7 +690,13 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
reqs |= q
and_reqs.append(reqs)
break
+ dct['extras'] = []
dct = _search_manage_search_vector(dct)
+ search_vector = ""
+ if 'search_vector' in dct:
+ search_vector = dct.pop('search_vector')
+ extras = dct.pop('extras')
+
query = Q(**dct)
for k, or_req in or_reqs:
alt_dct = dct.copy()
@@ -665,11 +767,14 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
dct = {upper_key: current}
query &= Q(**dct)
- items = model.objects.filter(query).distinct()
+ items = model.objects.filter(query)
+ for extra in extras:
+ items = items.extra(**extra)
+ items = items.distinct()
# print(items.query)
- if 'search_vector' in dct: # for serialization
- dct['search_vector'] = dct['search_vector'].value
+ if search_vector: # for serialization
+ dct['search_vector'] = search_vector
# table cols
if own_table_cols: