⚡️ improve post-treatment (specialy for operations)develop-4.4-optimization-test

7 files changed, 156 insertions, 65 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index 29e8b56a8..406c50057 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -18,6 +18,7 @@
 # See the file COPYING for details.
 
 from collections import OrderedDict
+import datetime
 import uuid
 
 from django.apps import apps
@@ -435,11 +436,15 @@ class GeographicSubTownItem(GeoItem):
     def _get_geo_town(self):
         raise NotImplementedError()
 
-    def post_save_geo(self, save=True):
+    def post_save_geo(self, save=True, timestamp=None):
         # manage geodata towns
         if getattr(self, "_post_save_geo_ok", False) or not self.pk:
             # prevent infinite loop - should not happen, but...
             return
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        if hasattr(self, "timestamp_geo") and (self.timestamp_geo or 0) >= timestamp:
+            return
         self._post_save_geo_ok = True
 
         q_geodata_current_town = self.geodata.filter(
@@ -512,7 +517,8 @@ class GeographicSubTownItem(GeoItem):
             return False
         if save:
             post_save_geo(self.__class__, instance=self, created=False,
-                          update_fields=False, raw=False, using="default")
+                          update_fields=False, raw=False, using="default",
+                          timestamp=timestamp)
         return True
 
 
@@ -1431,20 +1437,22 @@ class ContextRecord(
 
 
 def context_record_post_save(sender, **kwargs):
-    cached_label_changed(sender=sender, **kwargs)
-    post_save_geo(sender=sender, **kwargs)
     instance = kwargs.get("instance", None)
     if not instance or not instance.pk:
         return
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
+    cached_label_changed(sender=sender, **kwargs)
+    post_save_geo(sender=sender, **kwargs)
     profile = get_current_profile()
     if profile.parent_relations_engine == "T":
         ContextRecordTree._update_self_relation(instance.pk)  # on creation: manage self relation
     BaseFind = apps.get_model("archaeological_finds", "BaseFind")
     Find = apps.get_model("archaeological_finds", "Find")
     for bf in instance.base_finds.all():
-        cached_label_changed(BaseFind, instance=bf)
+        cached_label_changed(BaseFind, instance=bf, timestamp=timestamp)
         for f in bf.find.all():
-            cached_label_changed(Find, instance=f)
+            cached_label_changed(Find, instance=f, timestamp=timestamp)
 
 
 post_save.connect(context_record_post_save, sender=ContextRecord)
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index fa38a6dbb..7fc27c7cc 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -886,13 +886,15 @@ class BaseFind(
 
 
 def post_save_basefind(sender, **kwargs):
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     cached_label_changed(sender, **kwargs)
     post_save_geo(sender, **kwargs)
     instance = kwargs.get("instance", None)
     if not instance or not instance.pk:
         return
     for f in instance.find.all():
-        cached_label_changed(Find, instance=f)
+        cached_label_changed(Find, instance=f, timestamp=timestamp)
 
 
 def pre_delete_basefind(sender, **kwargs):
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index d933944d1..74efe2104 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -223,11 +223,15 @@ class GeographicTownItem(GeoItem):
     class Meta:
         abstract = True
 
-    def post_save_geo(self, save=True):
+    def post_save_geo(self, save=True, timestamp=None):
         # manage geodata towns
         if getattr(self, "_post_save_geo_ok", False):
             # prevent infinite loop - should not happen, but...
             return
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        if hasattr(self, "timestamp_geo") and (self.timestamp_geo or 0) >= timestamp:
+            return
         self._post_save_geo_ok = True
         q_towns = self.towns.filter(main_geodata__multi_polygon__isnull=False)
         q_towns_nb = q_towns.count()
@@ -330,9 +334,11 @@ class GeographicTownItem(GeoItem):
 
         if changed and save:
             self.no_post_process()
+            self.timestamp_geo = timestamp
             self.save()
             post_save_geo(self.__class__, instance=self, created=False,
-                          update_fields=False, raw=False, using="default")
+                          update_fields=False, raw=False, using="default",
+                          timestamp=timestamp)
         return changed
 
 
@@ -1048,6 +1054,8 @@ class ArchaeologicalSite(
 
 
 def site_post_save(sender, **kwargs):
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     cached_label_changed(sender=sender, **kwargs)
     post_save_geo(sender=sender, **kwargs)
 
@@ -1546,9 +1554,11 @@ class Operation(
     POST_PROCESS_REQUEST = {
         "towns__numero_insee__startswith": "_get_department_code",
     }
-
     DOWN_MODEL_UPDATE = ["parcels", "context_record"]
-
+    DOWN_MODEL_REVERSE_QUERY = {
+        "context_record": "operation_id",
+        "parcels": "operation_id"
+    }
     HISTORICAL_M2M = [
         "remains",
         "towns",
@@ -2675,6 +2685,8 @@ for attr in Operation.HISTORICAL_M2M:
 def operation_post_save(sender, **kwargs):
     if not kwargs["instance"]:
         return
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     post_save_geo(sender=sender, **kwargs)
 
     operation = kwargs["instance"]
@@ -2702,39 +2714,27 @@ def operation_post_save(sender, **kwargs):
         for parcel in operation.parcels.all():
             parcel.copy_to_file()
 
+    """
+    # TODO: delete - should be managed with DOWN_MODEL_UPDATE
     # external id, cached_labels update
     for parcel in operation.parcels.all():
-        cached_label_changed(Parcel, instance=parcel)
+        cached_label_changed(Parcel, instance=parcel, timestamp=timestamp)
 
     ContextRecord = apps.get_model("archaeological_context_records", "ContextRecord")
     BaseFind = apps.get_model("archaeological_finds", "BaseFind")
     Find = apps.get_model("archaeological_finds", "Find")
     for cr in operation.context_record.all():
-        cached_label_changed(ContextRecord, instance=cr)
+        cached_label_changed(ContextRecord, instance=cr, timestamp=timestamp)
         for bf in cr.base_finds.all():
-            cached_label_changed(BaseFind, instance=bf)
+            cached_label_changed(BaseFind, instance=bf, timestamp=timestamp)
             for f in bf.find.all():
-                cached_label_changed(Find, instance=f)
+                cached_label_changed(Find, instance=f, timestamp=timestamp)
+    """
 
 
 post_save.connect(operation_post_save, sender=Operation)
 
 
-def operation_town_m2m_changed(sender, **kwargs):
-    operation = kwargs.get("instance", None)
-    if not operation:
-        return
-    operation._prevent_loop = False
-    operation.regenerate_all_ids()
-    geotown_attached_changed(sender, **kwargs)
-    force_cached_label_changed(sender, **kwargs)
-
-
-m2m_changed.connect(
-    operation_town_m2m_changed, sender=Operation.towns.through
-)
-
-
 class RelationType(GeneralRelationType):
     class Meta:
         verbose_name = _("Operation relation type")
@@ -3706,6 +3706,8 @@ def parcel_post_save(sender, **kwargs):
     if not kwargs["instance"]:
         return
     parcel = kwargs["instance"]
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     cached_label_changed(sender, **kwargs)
 
     if (
diff --git a/archaeological_operations/wizards.py b/archaeological_operations/wizards.py
index 74840e552..94d6b49f9 100644
--- a/archaeological_operations/wizards.py
+++ b/archaeological_operations/wizards.py
@@ -87,7 +87,6 @@ class OperationWizard(Wizard):
         Return extra context for templates
         """
         context = super(OperationWizard, self).get_context_data(form, **kwargs)
-        step = self.steps.current
         # reminder of the current file
         reminder = self.get_reminder()
         if reminder:
@@ -198,15 +197,6 @@ class OperationModificationWizard(OperationWizard):
     modification = True
     filter_owns = {"selec-operation_modification": ["pk"]}
 
-    def get_form_kwargs(self, step, **kwargs):
-        kwargs = super(OperationModificationWizard, self).get_form_kwargs(
-            step, **kwargs
-        )
-        if step != "relations-operation_modification":
-            return kwargs
-        kwargs["left_record"] = self.get_current_object()
-        return kwargs
-
 
 class OperationClosingWizard(ClosingWizard):
     model = models.Operation
diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py
index 121e2bbb9..15cf1384c 100644
--- a/archaeological_warehouse/models.py
+++ b/archaeological_warehouse/models.py
@@ -2143,6 +2143,8 @@ class ContainerLocalisation(models.Model):
     def save(self, *args, **kwargs):
         super(ContainerLocalisation, self).save(*args, **kwargs)
         self.container.skip_history_when_saving = True
-        cached_label_changed(Container, instance=self.container, force_update=True)
+        timestamp = int(datetime.datetime.now().timestamp())
+        cached_label_changed(Container, instance=self.container, force_update=True,
+                             timestamp=timestamp)
 
 
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index e2495bcd5..d09d64d88 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -66,6 +66,7 @@ from ishtar_common.model_merging import merge_model_objects
 from ishtar_common.models_imports import Import
 from ishtar_common.templatetags.link_to_window import simple_link_to_window
 from ishtar_common.utils import (
+    bulk_item_changed,
     cached_label_changed,
     disable_for_loaddata,
     duplicate_item,
@@ -1906,6 +1907,8 @@ class BaseHistorizedItem(
         )
         if not created and not external_id_updated:
             self.update_external_id()
+        if not hasattr(self, "_timestamp"):
+            self._timestamp = int(datetime.datetime.now().timestamp())
         super(BaseHistorizedItem, self).save(*args, **kwargs)
         if created and self.update_external_id():
             # force resave for external ID creation
@@ -3131,11 +3134,11 @@ class GeographicItem(models.Model):
         if getattr(self, "_geodata_list", None):
             return self._geodata_list
         lst = []
+        q = self.geodata
         if self.main_geodata:
             lst.append(self.main_geodata)
-        for geo in self.geodata.all():
-            if geo != self.main_geodata:
-                lst.append(geo)
+            q = q.exclude(id=self.main_geodata_id)
+        lst += [geo for geo in self.geodata.all()]
         self._geodata_list = lst
         return lst
 
@@ -3363,6 +3366,7 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
     SLUG = ""
     SHOW_URL = None
     DOWN_MODEL_UPDATE = []
+    DOWN_MODEL_REVERSE_QUERY = {}
     INITIAL_VALUES = []  # list of field checkable if changed on save
     OLD_SHEET_EXPORT = True
 
@@ -3385,11 +3389,15 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
                 self._initial_values[field_name] = value
         return changed
 
-    def cascade_update(self, changed=True):
+    def cascade_update(self, changed=True, timestamp=None):
         if not changed:
             return
         if getattr(self, "_no_down_model_update", False):
             return
+
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+
         queue = getattr(self, "_queue", settings.CELERY_DEFAULT_QUEUE)
         for down_model in self.DOWN_MODEL_UPDATE:
             if not settings.USE_BACKGROUND_TASK:
@@ -3397,14 +3405,27 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
                 if hasattr(rel.model, "need_update"):
                     rel.update(need_update=True)
                     continue
+            if down_model in self.DOWN_MODEL_REVERSE_QUERY:
+                key = self.DOWN_MODEL_REVERSE_QUERY[down_model]
+                new_down_model = getattr(self, down_model).model
+                if hasattr(new_down_model, "bulk_item_changed"):
+                    new_down_model.bulk_item_changed({key: self.pk})
+                    continue
             for item in getattr(self, down_model).all():
                 if hasattr(self, "_timestamp"):
                     item._timestamp = self._timestamp
                 item._queue = queue
                 if hasattr(item, "cached_label_changed"):
-                    item.cached_label_changed()
+                    if hasattr(item, "timestamp_label") and \
+                            (item.timestamp_label or 0) >= timestamp:
+                        pass
+                    else:
+                        item.cached_label_changed()
+                if hasattr(item, "timestamp_geo") and \
+                        (item.timestamp_geo or 0) >= timestamp:
+                    continue
                 if hasattr(item, "main_geodata"):
-                    item.post_save_geo()
+                    item.post_save_geo(timestamp=timestamp)
 
     def no_post_process(self):
         self.skip_history_when_saving = True
@@ -3490,16 +3511,30 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
         self._no_move = True
         self.save()
 
-    def cached_label_changed(self):
+    @classmethod
+    def bulk_item_changed(cls, query_dict, timestamp=None):
+        """
+        Trigger changes in a bulk item list set by a query_dict (must be serializabled)
+        """
+        if not settings.USE_BACKGROUND_TASK:
+            for item in cls.objects.filter(**query_dict).all():
+                item.cached_label_changed(timestamp=timestamp)
+            return
+        bulk_item_changed(cls, query=query_dict, timestamp=timestamp)
+
+    def cached_label_changed(self, timestamp=None):
         self.no_post_process()
         self._cached_label_checked = False
-        cached_label_changed(self.__class__, instance=self, created=False)
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        cached_label_changed(self.__class__, instance=self, created=False,
+                             timestamp=timestamp)
 
-    def post_save_geo(self, save=True):
+    def post_save_geo(self, save=True, timestamp=None):
         if getattr(self, "_post_saved_geo", False):
             return
         self.no_post_process()
-        post_save_geo(self.__class__, instance=self, created=False)
+        post_save_geo(self.__class__, instance=self, created=False, timestamp=timestamp)
         return False
 
     def external_id_changed(self):
@@ -3784,6 +3819,8 @@ class Town(GeographicItem, Imported, DocumentItem, MainItem, models.Model):
 
 
 def post_save_town(sender, **kwargs):
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     cached_label_changed(sender, **kwargs)
     town = kwargs["instance"]
     town.generate_geo()
@@ -3812,7 +3849,6 @@ def geotown_attached_changed(sender, **kwargs):
         return
     instance = kwargs.get("instance", None)
     model = kwargs.get("model", None)
-    pk_set = kwargs.get("pk_set", None)
     action = kwargs.get("action", None)
     if not instance or not model or not hasattr(instance, "post_save_geo"):
         return
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index cf58216a4..bcba9c52a 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -1123,14 +1123,51 @@ def load_task(task_func, task_name, checks, sender, queue=None, **kwargs):
     return task_item
 
 
+def load_query_task(task_func, task_name, sender, queue=None, **kwargs):
+    if not queue:
+        queue = settings.CELERY_DEFAULT_QUEUE
+    query = kwargs.get("query", None)
+    if not query:
+        return
+
+    if not settings.USE_BACKGROUND_TASK:
+        # no background task
+        return task_func(sender, **kwargs)
+
+    sender, kwargs = serialize_args_for_tasks(
+        sender, query, kwargs, EXTRA_KWARGS_TRIGGER
+    )
+    kwargs["queue"] = queue
+    task_item = task_func.apply_async([sender], kwargs, queue=queue)
+    revoke_old_task(kwargs, task_name, task_item.id, sender)
+    return task_item
+
+
+def bulk_item_changed(sender, **kwargs):
+    load_query_task(_bulk_item_changed, "bulk_item_changed", sender, "low_priority")
+
+
+@task()
+def _bulk_item_changed(sender, **kwargs):
+    sender, query = deserialize_args_for_tasks(sender, kwargs, EXTRA_KWARGS_TRIGGER)
+    if not query:
+        return
+    for item in sender.objects.filter(**query).all():
+        kwargs["instance"] = item
+        cached_label_changed(sender, **kwargs)
+        post_save_geo(sender, **kwargs)
+
+
 def cached_label_changed(sender, **kwargs):
     if "instance" not in kwargs:
         return
     instance = kwargs["instance"]
     if not instance:
         return
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label") and (
-            instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    if hasattr(instance, "timestamp_label") and (instance.timestamp_label or 0) >= timestamp:
         return
     queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE)
     if hasattr(instance, "external_id") and hasattr(instance, "auto_external_id") \
@@ -1160,10 +1197,15 @@ def _cached_label_changed(sender, **kwargs):
     if not force_update and getattr(instance, "_cached_label_checked", False):
         return
 
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label"):
-        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    if hasattr(instance, "timestamp_label"):
+        if (instance.timestamp_label or 0) >= timestamp:
             return
-        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_label=instance._timestamp)
+        instance.__class__.objects.filter(pk=instance.pk).update(
+            timestamp_label=timestamp
+        )
 
     instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE)
     logger.debug(f"[ishtar] ishtar_common.utils._cached_label_changed - {instance.__class__.__name__} - {instance.pk} - {instance}")
@@ -1197,7 +1239,7 @@ def _cached_label_changed(sender, **kwargs):
         instance.__class__.objects.filter(pk=instance.pk).update(**dict(changed))
     if ((getattr(instance, "check_cascade_update", False) and instance.check_cascade_update())
             or changed or not cached_labels) and hasattr(instance, "cascade_update"):
-        instance.cascade_update()
+        instance.cascade_update(timestamp=timestamp)
     updated = False
     if force_update or hasattr(instance, "update_search_vector"):
         updated = instance.update_search_vector()
@@ -1206,9 +1248,7 @@ def _cached_label_changed(sender, **kwargs):
             item._cascade_change = True
             if hasattr(instance, "test_obj"):
                 item.test_obj = instance.test_obj
-            if instance.timestamp_label:
-                item._timestamp = instance.timestamp_label
-            cached_label_changed(item.__class__, instance=item)
+            cached_label_changed(item.__class__, instance=item, timestamp=timestamp)
     cache_key, __ = get_cache(sender, ["cached_label_changed", instance.pk])
     cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT)
     if cached_labels:
@@ -1495,6 +1535,12 @@ def post_save_geo(sender, **kwargs):
         return
     if getattr(instance, "_post_saved_geo", False):
         return
+    timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    elif (instance.timestamp_geo or 0) >= timestamp:
+        return
+
     queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE)
     return load_task(_post_save_geo, "post_save_geo", ["_no_geo_check"],
                      sender, queue=queue, **kwargs)
@@ -1515,10 +1561,15 @@ def _post_save_geo(sender, **kwargs):
     if getattr(instance, "_post_saved_geo", False):
         return
 
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_geo"):
-        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None)
+    if hasattr(instance, "timestamp_geo"):
+        if timestamp and (instance.timestamp_geo or 0) >= timestamp:
             return
-        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_geo=instance._timestamp)
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        instance.__class__.objects.filter(pk=instance.pk).update(
+            timestamp_geo=timestamp
+        )
 
     instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE)
     logger.debug(f"[ishtar] ishtar_common.utils._post_save_geo - {instance.__class__.__name__} - {instance.pk} - {instance}")
@@ -1541,7 +1592,7 @@ def _post_save_geo(sender, **kwargs):
         instance._cached_label_checked = False
         instance.save()
         if hasattr(instance, "cascade_update"):
-            instance.cascade_update()
+            instance.cascade_update(timestamp=timestamp)
     cache_key, __ = get_cache(sender, ["post_save_geo", instance.pk])
     cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT)
     return