diff options
| -rw-r--r-- | archaeological_operations/tests.py | 15 | ||||
| -rw-r--r-- | ishtar_common/views_item.py | 298 |
2 files changed, 226 insertions, 87 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index dc98ba25e..00eacbca4 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -3149,14 +3149,17 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText, StatisticsTes response = c.get(reverse("get-operation"), {"search_vector": request}) self.assertEqual(json.loads(response.content.decode())["recordsTotal"], 2) - def test_and_search_vector(self): + def test_and_or_search_vector(self): operation_1 = models.Operation.objects.get(pk=self.operations[0].pk) operation_2 = models.Operation.objects.get(pk=self.operations[1].pk) + operation_3 = models.Operation.objects.get(pk=self.operations[2].pk) villa = models.RemainType.objects.get(txt_idx="villa") cairn = models.RemainType.objects.get(txt_idx="cairn") + statue_menhir = models.RemainType.objects.get(txt_idx="statue-menhir") operation_1.remains.add(villa) operation_1.remains.add(cairn) operation_2.remains.add(cairn) + operation_3.remains.add(statue_menhir) c = Client() c.login(username=self.username, password=self.password) @@ -3169,6 +3172,16 @@ class OperationSearchTest(TestCase, OperationInitTest, SearchText, StatisticsTes response = c.get(reverse("get-operation"), {"search_vector": request}) self.assertEqual(json.loads(response.content.decode())["recordsTotal"], 1) + # test parenthesis and mixed search AND and OR operators + request = f' (( {remain_key}="{cairn.label}" && {remain_key}="{villa.label}" )) ' + request += f' || {remain_key}="{statue_menhir.label}"' + response = c.get(reverse("get-operation"), {"search_vector": request}) + self.assertEqual(json.loads(response.content.decode())["recordsTotal"], 2) + request = f' {remain_key}="{statue_menhir.label}" || ' + request += f' (( {remain_key}="{cairn.label}" && {remain_key}="{villa.label}" )) ' + response = c.get(reverse("get-operation"), {"search_vector": request}) + self.assertEqual(json.loads(response.content.decode())["recordsTotal"], 2) + def test_complex_search_vector(self): c = Client() c.login(username=self.username, password=self.password) diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py index 40734a087..631a46c96 100644 --- a/ishtar_common/views_item.py +++ b/ishtar_common/views_item.py @@ -2278,22 +2278,200 @@ def _get_table_cols(request, data_type, own_table_cols, full, model): filtered_table_cols.append(col_name) return filtered_table_cols +AND = " && " +OR = " || " +RE_GROUPS = re.compile(r"(.*?)\(\( (.*) \)\)(.*)") -def split_dict(dct): - if not dct.get("search_vector", None): - return [("OR", dct)] - new_dcts = [] - # TODO: manage || and && syntax in the same query - # example: to extract [[]] parenthesis re.findall(r"(.*)\[\[ (.*?) \]\](.*)", s) - split_key, split_type = " || ", "OR" - if " && " in dct["search_vector"]: - split_key, split_type = " && ", "AND" - for vector in dct["search_vector"].split(split_key): - new_dct = deepcopy(dct) - new_dct["search_vector"] = vector - new_dcts.append((split_type, new_dct)) - return new_dcts +def _parse_operator_simple(query): + """ + "aaa || bbb || ccc" -> ("OR", ("aaa", "bbb", "ccc")) + "aaa && bbb && ccc" -> ("AND", ("aaa", "bbb", "ccc")) + "aaa || bbb && ccc" -> ("OR", ("aaa", ("AND", "bbb", "ccc"))) + """ + if AND in query: + return ("AND", [_parse_operator_simple(v) for v in query.split(AND)]) + if OR in query: + return ("OR", [_parse_operator_simple(v) for v in query.split(OR)]) + return query.strip() + + +def _parse_operator_query_string(query): + """ + "|| query" -> ("OR", query, "OR") + "|| query &&" -> ("OR", query, "AND") + """ + start_operator = "OR" + if query.startswith(AND): + start_operator = "AND" + query = query[4:] + if query.startswith(OR): + query = query[4:] + end_operator = "OR" + if query.endswith(AND): + end_operator = AND + query = query[:-4] + if query.endswith(OR): + query = query[:-4] + query = _parse_logic_query_string(query) + return (start_operator, query, end_operator) + + +def _parse_logic_query_string(query): + """ + (( (( ccea && MAC )) || materiau="céramique" || materiau="fer*" )) -> + [('OR', [('AND', ['ccea', 'MAc']), ('OR', ['materiau="céramique"', 'materiau="fer*"'])])] + tuple is for (operator, query) + list is for operand + """ + current_group = [] + m = RE_GROUPS.match(query) + if not m: + return _parse_operator_simple(query) + start, middle, end = m.groups() + end_operator = "OR" + if start: + start_operator, query, end_operator = _parse_operator_query_string( + start) + if isinstance(query, (tuple, str)): # single operand, multiple operand is list + current_group.append(query) + else: + current_group.append((start_operator, query)) + q = _parse_logic_query_string(middle) + if isinstance(q, (tuple, str)): # single operand + current_group.append(q) + else: + current_group.append((end_operator, q)) + end_operator = "OR" + if end: + start_operator, query, end_operator = _parse_operator_query_string( + end) + if isinstance(query, (tuple, str)): # single operand + current_group.append(query) + else: + current_group.append((start_operator, query)) + return (end_operator, current_group) + + +def __operator_query(model, items, sub_items, operator): + if not items: + items = sub_items + else: + if not sub_items.exists(): + if operator == "AND": + items = model.objects.filter(pk__isnull=True) + return items + if operator == "OR": + items |= sub_items + else: + # in Django m2m queries use the same JOIN + # items &= sub_items do not work + items &= model.objects.filter( + id__in=Subquery(sub_items.values("id")) + ) + return items + + +def execute_queries( + request, request_items, groups, model, base_query, base_dct, query_own, full, + distinct_queries, query_parameters, my_relation_types_prefix, my_bool_fields, + my_reversed_bool_fields, related_name_fields, many_counted_fields, + reversed_many_counted_fields, my_dated_fields, datetime_fields, + my_number_fields, and_reqs, data_type, operator + ): + if isinstance(groups, tuple): + operator, groups = groups + elif isinstance(groups, str): + groups = [groups] + + items = None + for term in groups: + if not term: + continue + if term == "~~~ALL~~~": # get all items, no filter + term = "" + if isinstance(term, tuple): + # subquery + sub_items = execute_queries( + request, request_items, term, model, base_query, base_dct, + query_own, full, distinct_queries, query_parameters, + my_relation_types_prefix, my_bool_fields, + my_reversed_bool_fields, related_name_fields, + many_counted_fields, reversed_many_counted_fields, + my_dated_fields, datetime_fields, my_number_fields, and_reqs, + data_type, operator) + items = __operator_query(model, items, sub_items, operator) + continue + + sub_dct = base_dct.copy() if base_dct else {} + sub_dct["search_vector"] = term + items = execute_query( + request, request_items, items, model, base_query, query_own, full, + sub_dct, distinct_queries, query_parameters, my_relation_types_prefix, + my_bool_fields, my_reversed_bool_fields, related_name_fields, + many_counted_fields, reversed_many_counted_fields, my_dated_fields, + datetime_fields, my_number_fields, and_reqs, data_type, operator) + return items + + +def execute_query( + request, request_items, items, model, base_query, query_own, full, sub_dct, + distinct_queries, query_parameters, my_relation_types_prefix, + my_bool_fields, my_reversed_bool_fields, related_name_fields, + many_counted_fields, reversed_many_counted_fields, my_dated_fields, + datetime_fields, my_number_fields, and_reqs, data_type, operator + ): + query, exc_query, extras = main_manager( + request, model, query_own, full, sub_dct, distinct_queries, + query_parameters, my_relation_types_prefix, my_bool_fields, + my_reversed_bool_fields, related_name_fields, many_counted_fields, + reversed_many_counted_fields, my_dated_fields, datetime_fields, + my_number_fields, and_reqs[:] + ) + sub_items = model.objects.filter(query) + for d_q in distinct_queries: + sub_items = sub_items.filter(d_q) + + if base_query: + sub_items = sub_items.filter(base_query) + if exc_query: + sub_items = sub_items.exclude(exc_query) + + stats_modality_1, stats_modality_2 = None, None + if data_type == "json-stats": + stats_modality_1 = request_items.get("stats_modality_1", None) + stats_modality_2 = request_items.get("stats_modality_2", None) + if ( + not stats_modality_1 + or stats_modality_1 not in model.STATISTIC_MODALITIES + ): + stats_modality_1 = model.STATISTIC_MODALITIES[0] + if stats_modality_2 not in model.STATISTIC_MODALITIES: + stats_modality_2 = None + if getattr(model, "STATISTIC_MODALITIES_QUERY", False): + if stats_modality_1 in model.STATISTIC_MODALITIES_QUERY and \ + "query" in model.STATISTIC_MODALITIES_QUERY[stats_modality_1]: + sub_items = sub_items.filter( + **model.STATISTIC_MODALITIES_QUERY[stats_modality_1]["query"]) + if stats_modality_2 in model.STATISTIC_MODALITIES_QUERY and \ + "query" in model.STATISTIC_MODALITIES_QUERY[stats_modality_2]: + sub_items = sub_items.filter( + **model.STATISTIC_MODALITIES_QUERY[stats_modality_2]["query"]) + + for extra in extras: + sub_items = sub_items.extra(**extra) + + items = __operator_query(model, items, sub_items, operator) + """ + print("ishtar_common/views_item.py - 2458") + print(f"operator: {operator}", f"query: {query}", + f"distinct_queries: {distinct_queries}", + f"base_query: {base_query}", f"exc_query: {exc_query}", + f"extras: {extras}", f"sub count: {sub_items.count()}", + f"all count: {items.count()}" + ) + """ + return items def main_manager( @@ -2878,78 +3056,24 @@ def get_item( pk__in=[int(pk) for pk in q.values_list("object_pk", flat=True)] ) - items = None - for split_type, sub_dct in split_dict(dct): - query, exc_query, extras = main_manager( - request, - model, - query_own, - full, - sub_dct, - distinct_queries, - query_parameters, - my_relation_types_prefix, - my_bool_fields, - my_reversed_bool_fields, - related_name_fields, - many_counted_fields, - reversed_many_counted_fields, - my_dated_fields, - datetime_fields, - my_number_fields, - and_reqs[:] - ) - - # print("ishtar_common/views_item.py - 2745") - # print(f"query: {query}", f"distinct_queries: {distinct_queries}", - # f"base_query: {base_query}", f"exc_query: {exc_query}", - # f"extras: {extras}") - sub_items = model.objects.filter(query) - for d_q in distinct_queries: - sub_items = sub_items.filter(d_q) - - if base_query: - sub_items = sub_items.filter(base_query) - if exc_query: - sub_items = sub_items.exclude(exc_query) - stats_modality_1, stats_modality_2 = None, None - if data_type == "json-stats": - stats_modality_1 = request_items.get("stats_modality_1", None) - stats_modality_2 = request_items.get("stats_modality_2", None) - if ( - not stats_modality_1 - or stats_modality_1 not in model.STATISTIC_MODALITIES - ): - stats_modality_1 = model.STATISTIC_MODALITIES[0] - if stats_modality_2 not in model.STATISTIC_MODALITIES: - stats_modality_2 = None - if getattr(model, "STATISTIC_MODALITIES_QUERY", False): - if stats_modality_1 in model.STATISTIC_MODALITIES_QUERY and \ - "query" in model.STATISTIC_MODALITIES_QUERY[stats_modality_1]: - sub_items = sub_items.filter( - **model.STATISTIC_MODALITIES_QUERY[stats_modality_1]["query"]) - if stats_modality_2 in model.STATISTIC_MODALITIES_QUERY and \ - "query" in model.STATISTIC_MODALITIES_QUERY[stats_modality_2]: - sub_items = sub_items.filter( - **model.STATISTIC_MODALITIES_QUERY[stats_modality_2]["query"]) - - for extra in extras: - sub_items = sub_items.extra(**extra) - if not items: - items = sub_items - else: - if not sub_items.exists(): - if split_type == "AND": - items = model.objects.filter(pk__isnull=True) - continue - if split_type == "OR": - items |= sub_items - else: - # in Django m2m queries use the same JOIN - # items &= sub_items do not work - items &= model.objects.filter( - id__in=Subquery(sub_items.values("id")) - ) + operator = "OR" + base_dct = dct.copy() + if "search_vector" in base_dct: + base_dct.pop("search_vector") + if not dct.get("search_vector", None): + groups = "~~~ALL~~~" # get all items + else: + groups = _parse_logic_query_string(dct["search_vector"]) + if isinstance(groups, tuple): + operator, groups = groups + + items = execute_queries( + request, request_items, groups, model, base_query, base_dct, + query_own, full, distinct_queries, query_parameters, + my_relation_types_prefix, my_bool_fields, my_reversed_bool_fields, + related_name_fields, many_counted_fields, + reversed_many_counted_fields, my_dated_fields, datetime_fields, + my_number_fields, and_reqs, data_type, operator) if return_query: return items @@ -3021,6 +3145,8 @@ def get_item( ): stats_sum_variable = stats_sum_variable_keys[0] multiply = model.STATISTIC_SUM_VARIABLE[stats_sum_variable][1] + stats_modality_1 = request_items.get("stats_modality_1", None) + stats_modality_2 = request_items.get("stats_modality_2", None) return _get_json_stats( items, stats_sum_variable, |