summaryrefslogtreecommitdiff
path: root/ishtar_common
diff options
context:
space:
mode:
Diffstat (limited to 'ishtar_common')
-rw-r--r--ishtar_common/management/commands/ishtar_maintenance.py2
-rw-r--r--ishtar_common/models.py6
-rw-r--r--ishtar_common/models_common.py59
-rw-r--r--ishtar_common/utils.py11
-rw-r--r--ishtar_common/views_item.py6
5 files changed, 65 insertions, 19 deletions
diff --git a/ishtar_common/management/commands/ishtar_maintenance.py b/ishtar_common/management/commands/ishtar_maintenance.py
index c4ad52ab6..4f050088e 100644
--- a/ishtar_common/management/commands/ishtar_maintenance.py
+++ b/ishtar_common/management/commands/ishtar_maintenance.py
@@ -131,7 +131,7 @@ def _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols):
writer.writerow(csv_cols)
writer.writerows(store_results)
if not quiet:
- sys.stdout.write(f"log: {path} written.")
+ sys.stdout.write(f"log: {path} written.\n")
def task_main_image(quiet=False, log=False):
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 442e0c687..fc2487d7b 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -3780,9 +3780,9 @@ class Document(
BASE_SEARCH_VECTORS = [
SearchVectorConfig("title"),
SearchVectorConfig("source_type__label"),
- SearchVectorConfig("external_id"),
- SearchVectorConfig("reference"),
- SearchVectorConfig("internal_reference"),
+ SearchVectorConfig("external_id", "raw"),
+ SearchVectorConfig("reference", "raw"),
+ SearchVectorConfig("internal_reference", "raw"),
SearchVectorConfig("description", "local"),
SearchVectorConfig("comment", "local"),
SearchVectorConfig("additional_information", "local"),
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index a6f9ac0e8..03fe26f0c 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -884,8 +884,36 @@ class FullSearch(models.Model):
deactivate()
return query_parameters
+ @classmethod
+ def _update_raw_search_field(cls, value):
+ result = []
+ if not value:
+ value = ""
+ for val in value.split("'"):
+ result.append(f"'{val.lower()}':1")
+ SEPS = [" ", "-", "/"]
+ values = []
+ # split ID terms
+ for idx, sep in enumerate(SEPS):
+ if not idx:
+ values = value.split(sep)
+ continue
+ new_values = []
+ for val in values:
+ new_values += val.split(sep)
+ values = new_values
+ for val in values:
+ if len(val) < 2:
+ continue
+ val = val.replace("'", "").lower()
+ result.append(f"'{val}':1")
+ return result
+
def _update_search_field(self, search_vector_conf, search_vectors, data):
for value in search_vector_conf.format(data):
+ if search_vector_conf.language == "raw":
+ search_vectors += self._update_raw_search_field(value)
+ continue
with connection.cursor() as cursor:
cursor.execute(
"SELECT to_tsvector(%s, %s)", [search_vector_conf.language, value]
@@ -921,7 +949,7 @@ class FullSearch(models.Model):
logger.warning("No search_vectors defined for {}".format(self.__class__))
return
if getattr(self, "_search_updated", None):
- return
+ return self.search_vector
JsonDataField = apps.get_model("ishtar_common", "JsonDataField")
self._search_updated = True
@@ -938,6 +966,12 @@ class FullSearch(models.Model):
for item in rel_key.values("pk").all():
query_dct = {key + "__pk": item["pk"]}
q = copy.copy(base_q).filter(**query_dct)
+ if m2m_search_vector.language == "raw":
+ q = q.values_list(m2m_search_vector.key, flat=True)
+ search_vectors += self._update_raw_search_field(q.all()[0])
+ continue
+ query_dct = {key + "__pk": item["pk"]}
+ q = copy.copy(base_q).filter(**query_dct)
q = q.annotate(
search=SearchVector(
m2m_search_vector.key, config=m2m_search_vector.language
@@ -961,17 +995,17 @@ class FullSearch(models.Model):
elif parent:
search_vectors.append(parent.search_vector)
- for PARENT_ONLY_SEARCH_VECTOR in self.PARENT_ONLY_SEARCH_VECTORS:
- parent = getattr(self, PARENT_ONLY_SEARCH_VECTOR)
- if hasattr(parent, "all"): # m2m
- for p in parent.all():
+ for PARENT_ONLY_SEARCH_VECTOR in self.PARENT_ONLY_SEARCH_VECTORS:
+ parent = getattr(self, PARENT_ONLY_SEARCH_VECTOR)
+ if hasattr(parent, "all"): # m2m
+ for p in parent.all():
+ search_vectors.append(
+ p.update_search_vector(save=False, exclude_parent=True)
+ )
+ elif parent:
search_vectors.append(
- p.update_search_vector(save=False, exclude_parent=True)
+ parent.update_search_vector(save=False, exclude_parent=True)
)
- elif parent:
- search_vectors.append(
- parent.update_search_vector(save=False, exclude_parent=True)
- )
if self.BASE_SEARCH_VECTORS:
# query "simple" fields
@@ -4110,7 +4144,10 @@ class SearchVectorConfig:
value = ""
if not self.func:
return [value]
- return self.func(value)
+ value = self.func(value)
+ if not isinstance(value, list):
+ return [value]
+ return value
class ShortMenuItem:
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 340fb9ee0..1219dd454 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -934,6 +934,9 @@ def num2col(n):
return string
+RE_TSVECTOR = re.compile(r"('[^']+':\d+(?:,\d+)*)")
+
+
def merge_tsvectors(vectors):
"""
Parse tsvector to merge them in one string
@@ -952,16 +955,20 @@ def merge_tsvectors(vectors):
if max_position > current_position:
current_position = max_position
- for dct_member in vector.split(" "):
+ for dct_member in RE_TSVECTOR.findall(vector):
splitted = dct_member.split(":")
key = ":".join(splitted[:-1])
- positions = splitted[-1]
key = key[1:-1] # remove quotes
+ result_dict[key] = [1]
+ """
+ # position is not used today - simplify
+ positions = splitted[-1]
positions = [int(pos) + current_position for pos in positions.split(",")]
if key in result_dict:
result_dict[key] += positions
else:
result_dict[key] = positions
+ """
# {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"}
result_dict = {
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 821a6635f..f905d677c 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -835,9 +835,11 @@ def _search_manage_search_vector(
{
"where": [
f"{model._meta.db_table}.search_vector @@ (to_tsquery(%s, %s)) = true OR " +
- f"{model._meta.db_table}.search_vector @@ (to_tsquery('simple', %s)) = true"
+ f"{model._meta.db_table}.search_vector @@ (to_tsquery('simple', %s)) = true OR "
+ f"{model._meta.db_table}.search_vector::tsvector @@ %s::tsquery = true"
],
- "params": [settings.ISHTAR_SEARCH_LANGUAGE, search_query, search_query],
+ "params": [settings.ISHTAR_SEARCH_LANGUAGE, search_query, search_query,
+ search_query],
}
)
return dct, exc_dct, distinct_queries
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-16 19:50:55 +0000