⚡ improve performance on search vector generation

- prevent multi-save
- simplify parent search vector
- improve in obtaining M2M attribute

5 files changed, 67 insertions, 33 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index c54256cf6..ccc145c48 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -70,6 +70,7 @@ from ishtar_common.models import (
 from ishtar_common.models_common import GeoVectorData, HistoricalRecords,\
     SerializeItem, geodata_attached_changed
 from archaeological_operations.models import (
+    add_oa_prefix,
     ArchaeologicalSite,
     CulturalAttributionType,
     Operation,
@@ -745,7 +746,6 @@ class ContextRecord(
     ALT_NAMES.update(GeoItem.ALT_NAMES)
     ALT_NAMES.update(Imported.ALT_NAMES)
 
-    PARENT_ONLY_SEARCH_VECTORS = ["operation", "archaeological_site", "parcel"]
     BASE_SEARCH_VECTORS = [
         SearchVectorConfig("label", "raw"),
         SearchVectorConfig("location"),
@@ -757,6 +757,12 @@ class ContextRecord(
         SearchVectorConfig("unit__label"),
         SearchVectorConfig("activity__label"),
         SearchVectorConfig("excavator__cached_label", "raw"),
+        SearchVectorConfig("operation__code_patriarche", "raw"),
+        SearchVectorConfig("operation__code_patriarche", "raw", func=add_oa_prefix),
+        SearchVectorConfig("archaeological_site__name", "raw"),
+        SearchVectorConfig("archaeological_site__other_reference", "raw"),
+        SearchVectorConfig("archaeological_site__reference", "raw"),
+        SearchVectorConfig("parcel__cached_label", "raw"),
     ]
     M2M_SEARCH_VECTORS = [
         SearchVectorConfig("datings__period__label", "local"),
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index 9ad128432..9a287f516 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -616,7 +616,7 @@ class BaseFind(
     RELATED_POST_PROCESS = ["find"]
     CACHED_LABELS = ["cache_short_id", "cache_complete_id"]
     CACHED_COMPLETE_ID = "cache_complete_id"
-    PARENT_SEARCH_VECTORS = ["context_record"]
+    PARENT_ONLY_SEARCH_VECTORS = ["context_record"]
     BASE_SEARCH_VECTORS = [
         SearchVectorConfig("label", "raw"),
         SearchVectorConfig("description", "local"),
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index 63904bebe..9119a5c72 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -336,6 +336,18 @@ class GeographicTownItem(GeoItem):
         return changed
 
 
+def add_oa_prefix(value):
+    if not value:
+        return []
+    profile = get_current_profile()
+    values = []
+    if profile.operation_prefix:
+        values.append(profile.operation_prefix + value)
+    if profile.default_operation_prefix:
+        values.append(profile.default_operation_prefix + value)
+    return values
+
+
 class SiteManager(models.Manager):
     def get_by_natural_key(self, txt_idx):
         return self.get(reference=txt_idx)
@@ -390,12 +402,12 @@ class ArchaeologicalSite(
         SearchVectorConfig("precise_locality", "local"),
         SearchVectorConfig("locality_cadastral", "local"),
         SearchVectorConfig("locality_ngi", "local"),
-        SearchVectorConfig("name"),
+        SearchVectorConfig("name", "raw"),
         SearchVectorConfig("oceanographic_service_localisation"),
         SearchVectorConfig("reference", "raw"),
         SearchVectorConfig("other_reference", "raw"),
         SearchVectorConfig("shipwreck_code", "raw"),
-        SearchVectorConfig("shipwreck_name"),
+        SearchVectorConfig("shipwreck_name", "raw"),
         SearchVectorConfig("drassm_number", "raw"),
         SearchVectorConfig("affmar_number", "raw"),
     ]
@@ -403,10 +415,11 @@ class ArchaeologicalSite(
         SearchVectorConfig("periods__label", "local"),
         SearchVectorConfig("remains__label", "local"),
         SearchVectorConfig("types__label", "local"),
-        SearchVectorConfig("towns__name"),
+        SearchVectorConfig("towns__name", "raw"),
         SearchVectorConfig("towns__numero_insee", "raw"),
+        SearchVectorConfig("operations__code_patriarche", "raw"),
+        SearchVectorConfig("operations__code_patriarche", "raw", func=add_oa_prefix),
     ]
-    PARENT_ONLY_SEARCH_VECTORS = ["operations"]
     GET_VALUES_M2M = [
         "periods",
         "remains",
@@ -1095,15 +1108,6 @@ class ParcelItem:
                 parcels[key] = p
 
 
-def add_oa_prefix(value):
-    if not value:
-        return ""
-    profile = get_current_profile()
-    if not profile.operation_prefix:
-        return ""
-    return profile.operation_prefix + value
-
-
 class Operation(
     ClosedItem,
     DocumentItem,
@@ -1254,6 +1258,9 @@ class Operation(
         SearchVectorConfig("scientific_documentation_comment", "local"),
         SearchVectorConfig("seizure_name"),
         SearchVectorConfig("drassm_code", "raw"),
+        SearchVectorConfig("associated_file__name", "raw"),
+        SearchVectorConfig("associated_file__operation_name", "raw"),
+        SearchVectorConfig("associated_file__internal_reference", "raw"),
     ]
     PROPERTY_SEARCH_VECTORS = [
         SearchVectorConfig("full_reference", "raw"),
@@ -1265,11 +1272,12 @@ class Operation(
     M2M_SEARCH_VECTORS = [
         SearchVectorConfig("periods__label", "local"),
         SearchVectorConfig("remains__label", "local"),
-        SearchVectorConfig("towns__name"),
-        SearchVectorConfig("towns__numero_insee"),
+        SearchVectorConfig("towns__name", "raw"),
+        SearchVectorConfig("towns__numero_insee", "raw"),
+        SearchVectorConfig("archaeological_sites__name", "raw"),
+        SearchVectorConfig("archaeological_sites__other_reference", "raw"),
+        SearchVectorConfig("archaeological_sites__reference", "raw"),
     ]
-    PARENT_SEARCH_VECTORS = ["associated_file"]
-    PARENT_ONLY_SEARCH_VECTORS = ["archaeological_sites"]
     ASSOCIATED = {
         "scientist": {("person_types", PersonType): ("head_scientist", "sra_agent")},
     }
diff --git a/archaeological_operations/views.py b/archaeological_operations/views.py
index bf33e45b6..9ebd0a371 100644
--- a/archaeological_operations/views.py
+++ b/archaeological_operations/views.py
@@ -684,6 +684,7 @@ def operation_site_modify(model, related_model, related_key, formset_class, url_
                     q_relations.add(new_item)
                 initial, __, __ = get_initial(q_relations)
                 formset = formset_class(initial=initial)
+                item.update_search_vector()
         else:
             formset = formset_class(initial=initial)
 
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 403c020dc..7b84b4dbd 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -975,6 +975,9 @@ class FullSearch(models.Model):
         :param save: True if you want to save the object immediately
         :return: True if modified
         """
+        if getattr(self, "_search_vector_updated", None):
+            return
+        self._search_vector_updated = True
         if not hasattr(self, "search_vector"):
             return
         if not self.pk:
@@ -1005,23 +1008,38 @@ class FullSearch(models.Model):
 
         # many to many have to be queried one by one otherwise only one is fetch
         for m2m_search_vector in self.M2M_SEARCH_VECTORS:
-            key = m2m_search_vector.key.split("__")[0]
+            key_splitted = m2m_search_vector.key.split("__")
+            key = key_splitted[0]
             rel_key = getattr(self, key)
-            for item in rel_key.values("pk").all():
-                query_dct = {key + "__pk": item["pk"]}
-                q = copy.copy(base_q).filter(**query_dct)
+            if len(key_splitted) == 2:
+                attr = key_splitted[1]
                 if m2m_search_vector.language == "raw":
-                    q = q.values_list(m2m_search_vector.key, flat=True)
-                    search_vectors += self._update_raw_search_field(q.all()[0])
-                    continue
-                query_dct = {key + "__pk": item["pk"]}
-                q = copy.copy(base_q).filter(**query_dct)
-                q = q.annotate(
-                    search=SearchVector(
-                        m2m_search_vector.key, config=m2m_search_vector.language
+                    values = list(rel_key.values_list(attr, flat=True))
+                else:
+                    values = list(
+                        rel_key.annotate(
+                            search=SearchVector(
+                                attr, config=m2m_search_vector.language
+                            )).values_list("search", flat=True)
                     )
-                ).values("search")
-                search_vectors.append(q.all()[0]["search"])
+                for value in values:
+                    search_vectors += self._update_raw_search_field(value)
+            else:
+                for item in rel_key.values("pk").all():
+                    query_dct = {key + "__pk": item["pk"]}
+                    q = copy.copy(base_q).filter(**query_dct)
+                    if m2m_search_vector.language == "raw":
+                        q = q.values_list(m2m_search_vector.key, flat=True)
+                        search_vectors += self._update_raw_search_field(q.all()[0])
+                        continue
+                    query_dct = {key + "__pk": item["pk"]}
+                    q = copy.copy(base_q).filter(**query_dct)
+                    q = q.annotate(
+                        search=SearchVector(
+                            m2m_search_vector.key, config=m2m_search_vector.language
+                        )
+                    ).values("search")
+                    search_vectors.append(q.all()[0]["search"])
 
         # int/float are not well managed by the SearchVector
         for int_search_vector in self.INT_SEARCH_VECTORS:
@@ -1117,6 +1135,7 @@ class FullSearch(models.Model):
                             cursor.execute("SELECT to_tsvector(%s, %s)", [lang, d])
                             row = cursor.fetchone()
                             search_vectors.append(row[0])
+        # TODO - performance: could be very slow -> cf. DGM CD17
         new_search_vector = merge_tsvectors(search_vectors)
         changed = old_search != new_search_vector
         self.search_vector = new_search_vector