⚡️ improve post-treatment (specialy for operations)develop-4.4-optimization-test

2 files changed, 111 insertions, 24 deletions

diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index e2495bcd5..d09d64d88 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -66,6 +66,7 @@ from ishtar_common.model_merging import merge_model_objects
 from ishtar_common.models_imports import Import
 from ishtar_common.templatetags.link_to_window import simple_link_to_window
 from ishtar_common.utils import (
+    bulk_item_changed,
     cached_label_changed,
     disable_for_loaddata,
     duplicate_item,
@@ -1906,6 +1907,8 @@ class BaseHistorizedItem(
         )
         if not created and not external_id_updated:
             self.update_external_id()
+        if not hasattr(self, "_timestamp"):
+            self._timestamp = int(datetime.datetime.now().timestamp())
         super(BaseHistorizedItem, self).save(*args, **kwargs)
         if created and self.update_external_id():
             # force resave for external ID creation
@@ -3131,11 +3134,11 @@ class GeographicItem(models.Model):
         if getattr(self, "_geodata_list", None):
             return self._geodata_list
         lst = []
+        q = self.geodata
         if self.main_geodata:
             lst.append(self.main_geodata)
-        for geo in self.geodata.all():
-            if geo != self.main_geodata:
-                lst.append(geo)
+            q = q.exclude(id=self.main_geodata_id)
+        lst += [geo for geo in self.geodata.all()]
         self._geodata_list = lst
         return lst
 
@@ -3363,6 +3366,7 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
     SLUG = ""
     SHOW_URL = None
     DOWN_MODEL_UPDATE = []
+    DOWN_MODEL_REVERSE_QUERY = {}
     INITIAL_VALUES = []  # list of field checkable if changed on save
     OLD_SHEET_EXPORT = True
 
@@ -3385,11 +3389,15 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
                 self._initial_values[field_name] = value
         return changed
 
-    def cascade_update(self, changed=True):
+    def cascade_update(self, changed=True, timestamp=None):
         if not changed:
             return
         if getattr(self, "_no_down_model_update", False):
             return
+
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+
         queue = getattr(self, "_queue", settings.CELERY_DEFAULT_QUEUE)
         for down_model in self.DOWN_MODEL_UPDATE:
             if not settings.USE_BACKGROUND_TASK:
@@ -3397,14 +3405,27 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
                 if hasattr(rel.model, "need_update"):
                     rel.update(need_update=True)
                     continue
+            if down_model in self.DOWN_MODEL_REVERSE_QUERY:
+                key = self.DOWN_MODEL_REVERSE_QUERY[down_model]
+                new_down_model = getattr(self, down_model).model
+                if hasattr(new_down_model, "bulk_item_changed"):
+                    new_down_model.bulk_item_changed({key: self.pk})
+                    continue
             for item in getattr(self, down_model).all():
                 if hasattr(self, "_timestamp"):
                     item._timestamp = self._timestamp
                 item._queue = queue
                 if hasattr(item, "cached_label_changed"):
-                    item.cached_label_changed()
+                    if hasattr(item, "timestamp_label") and \
+                            (item.timestamp_label or 0) >= timestamp:
+                        pass
+                    else:
+                        item.cached_label_changed()
+                if hasattr(item, "timestamp_geo") and \
+                        (item.timestamp_geo or 0) >= timestamp:
+                    continue
                 if hasattr(item, "main_geodata"):
-                    item.post_save_geo()
+                    item.post_save_geo(timestamp=timestamp)
 
     def no_post_process(self):
         self.skip_history_when_saving = True
@@ -3490,16 +3511,30 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem):
         self._no_move = True
         self.save()
 
-    def cached_label_changed(self):
+    @classmethod
+    def bulk_item_changed(cls, query_dict, timestamp=None):
+        """
+        Trigger changes in a bulk item list set by a query_dict (must be serializabled)
+        """
+        if not settings.USE_BACKGROUND_TASK:
+            for item in cls.objects.filter(**query_dict).all():
+                item.cached_label_changed(timestamp=timestamp)
+            return
+        bulk_item_changed(cls, query=query_dict, timestamp=timestamp)
+
+    def cached_label_changed(self, timestamp=None):
         self.no_post_process()
         self._cached_label_checked = False
-        cached_label_changed(self.__class__, instance=self, created=False)
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        cached_label_changed(self.__class__, instance=self, created=False,
+                             timestamp=timestamp)
 
-    def post_save_geo(self, save=True):
+    def post_save_geo(self, save=True, timestamp=None):
         if getattr(self, "_post_saved_geo", False):
             return
         self.no_post_process()
-        post_save_geo(self.__class__, instance=self, created=False)
+        post_save_geo(self.__class__, instance=self, created=False, timestamp=timestamp)
         return False
 
     def external_id_changed(self):
@@ -3784,6 +3819,8 @@ class Town(GeographicItem, Imported, DocumentItem, MainItem, models.Model):
 
 
 def post_save_town(sender, **kwargs):
+    timestamp = int(datetime.datetime.now().timestamp())
+    kwargs["timestamp"] = timestamp
     cached_label_changed(sender, **kwargs)
     town = kwargs["instance"]
     town.generate_geo()
@@ -3812,7 +3849,6 @@ def geotown_attached_changed(sender, **kwargs):
         return
     instance = kwargs.get("instance", None)
     model = kwargs.get("model", None)
-    pk_set = kwargs.get("pk_set", None)
     action = kwargs.get("action", None)
     if not instance or not model or not hasattr(instance, "post_save_geo"):
         return
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index cf58216a4..bcba9c52a 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -1123,14 +1123,51 @@ def load_task(task_func, task_name, checks, sender, queue=None, **kwargs):
     return task_item
 
 
+def load_query_task(task_func, task_name, sender, queue=None, **kwargs):
+    if not queue:
+        queue = settings.CELERY_DEFAULT_QUEUE
+    query = kwargs.get("query", None)
+    if not query:
+        return
+
+    if not settings.USE_BACKGROUND_TASK:
+        # no background task
+        return task_func(sender, **kwargs)
+
+    sender, kwargs = serialize_args_for_tasks(
+        sender, query, kwargs, EXTRA_KWARGS_TRIGGER
+    )
+    kwargs["queue"] = queue
+    task_item = task_func.apply_async([sender], kwargs, queue=queue)
+    revoke_old_task(kwargs, task_name, task_item.id, sender)
+    return task_item
+
+
+def bulk_item_changed(sender, **kwargs):
+    load_query_task(_bulk_item_changed, "bulk_item_changed", sender, "low_priority")
+
+
+@task()
+def _bulk_item_changed(sender, **kwargs):
+    sender, query = deserialize_args_for_tasks(sender, kwargs, EXTRA_KWARGS_TRIGGER)
+    if not query:
+        return
+    for item in sender.objects.filter(**query).all():
+        kwargs["instance"] = item
+        cached_label_changed(sender, **kwargs)
+        post_save_geo(sender, **kwargs)
+
+
 def cached_label_changed(sender, **kwargs):
     if "instance" not in kwargs:
         return
     instance = kwargs["instance"]
     if not instance:
         return
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label") and (
-            instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    if hasattr(instance, "timestamp_label") and (instance.timestamp_label or 0) >= timestamp:
         return
     queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE)
     if hasattr(instance, "external_id") and hasattr(instance, "auto_external_id") \
@@ -1160,10 +1197,15 @@ def _cached_label_changed(sender, **kwargs):
     if not force_update and getattr(instance, "_cached_label_checked", False):
         return
 
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label"):
-        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    if hasattr(instance, "timestamp_label"):
+        if (instance.timestamp_label or 0) >= timestamp:
             return
-        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_label=instance._timestamp)
+        instance.__class__.objects.filter(pk=instance.pk).update(
+            timestamp_label=timestamp
+        )
 
     instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE)
     logger.debug(f"[ishtar] ishtar_common.utils._cached_label_changed - {instance.__class__.__name__} - {instance.pk} - {instance}")
@@ -1197,7 +1239,7 @@ def _cached_label_changed(sender, **kwargs):
         instance.__class__.objects.filter(pk=instance.pk).update(**dict(changed))
     if ((getattr(instance, "check_cascade_update", False) and instance.check_cascade_update())
             or changed or not cached_labels) and hasattr(instance, "cascade_update"):
-        instance.cascade_update()
+        instance.cascade_update(timestamp=timestamp)
     updated = False
     if force_update or hasattr(instance, "update_search_vector"):
         updated = instance.update_search_vector()
@@ -1206,9 +1248,7 @@ def _cached_label_changed(sender, **kwargs):
             item._cascade_change = True
             if hasattr(instance, "test_obj"):
                 item.test_obj = instance.test_obj
-            if instance.timestamp_label:
-                item._timestamp = instance.timestamp_label
-            cached_label_changed(item.__class__, instance=item)
+            cached_label_changed(item.__class__, instance=item, timestamp=timestamp)
     cache_key, __ = get_cache(sender, ["cached_label_changed", instance.pk])
     cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT)
     if cached_labels:
@@ -1495,6 +1535,12 @@ def post_save_geo(sender, **kwargs):
         return
     if getattr(instance, "_post_saved_geo", False):
         return
+    timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None)
+    if not timestamp:
+        timestamp = int(datetime.datetime.now().timestamp())
+    elif (instance.timestamp_geo or 0) >= timestamp:
+        return
+
     queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE)
     return load_task(_post_save_geo, "post_save_geo", ["_no_geo_check"],
                      sender, queue=queue, **kwargs)
@@ -1515,10 +1561,15 @@ def _post_save_geo(sender, **kwargs):
     if getattr(instance, "_post_saved_geo", False):
         return
 
-    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_geo"):
-        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+    timestamp = kwargs.get("timestamp", None)
+    if hasattr(instance, "timestamp_geo"):
+        if timestamp and (instance.timestamp_geo or 0) >= timestamp:
             return
-        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_geo=instance._timestamp)
+        if not timestamp:
+            timestamp = int(datetime.datetime.now().timestamp())
+        instance.__class__.objects.filter(pk=instance.pk).update(
+            timestamp_geo=timestamp
+        )
 
     instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE)
     logger.debug(f"[ishtar] ishtar_common.utils._post_save_geo - {instance.__class__.__name__} - {instance.pk} - {instance}")
@@ -1541,7 +1592,7 @@ def _post_save_geo(sender, **kwargs):
         instance._cached_label_checked = False
         instance.save()
         if hasattr(instance, "cascade_update"):
-            instance.cascade_update()
+            instance.cascade_update(timestamp=timestamp)
     cache_key, __ = get_cache(sender, ["post_save_geo", instance.pk])
     cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT)
     return