From 53bea3ddd326510f7658286b394fbdd2b0f4b39a Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 19 Mar 2025 17:55:43 +0100 Subject: ⚡️ improve post-treatment (specialy for operations) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- archaeological_context_records/models.py | 20 ++++++--- archaeological_finds/models_finds.py | 4 +- archaeological_operations/models.py | 48 ++++++++++---------- archaeological_operations/wizards.py | 10 ----- archaeological_warehouse/models.py | 4 +- ishtar_common/models_common.py | 58 +++++++++++++++++++----- ishtar_common/utils.py | 77 ++++++++++++++++++++++++++------ 7 files changed, 156 insertions(+), 65 deletions(-) diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py index 29e8b56a8..406c50057 100644 --- a/archaeological_context_records/models.py +++ b/archaeological_context_records/models.py @@ -18,6 +18,7 @@ # See the file COPYING for details. from collections import OrderedDict +import datetime import uuid from django.apps import apps @@ -435,11 +436,15 @@ class GeographicSubTownItem(GeoItem): def _get_geo_town(self): raise NotImplementedError() - def post_save_geo(self, save=True): + def post_save_geo(self, save=True, timestamp=None): # manage geodata towns if getattr(self, "_post_save_geo_ok", False) or not self.pk: # prevent infinite loop - should not happen, but... return + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + if hasattr(self, "timestamp_geo") and (self.timestamp_geo or 0) >= timestamp: + return self._post_save_geo_ok = True q_geodata_current_town = self.geodata.filter( @@ -512,7 +517,8 @@ class GeographicSubTownItem(GeoItem): return False if save: post_save_geo(self.__class__, instance=self, created=False, - update_fields=False, raw=False, using="default") + update_fields=False, raw=False, using="default", + timestamp=timestamp) return True @@ -1431,20 +1437,22 @@ class ContextRecord( def context_record_post_save(sender, **kwargs): - cached_label_changed(sender=sender, **kwargs) - post_save_geo(sender=sender, **kwargs) instance = kwargs.get("instance", None) if not instance or not instance.pk: return + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp + cached_label_changed(sender=sender, **kwargs) + post_save_geo(sender=sender, **kwargs) profile = get_current_profile() if profile.parent_relations_engine == "T": ContextRecordTree._update_self_relation(instance.pk) # on creation: manage self relation BaseFind = apps.get_model("archaeological_finds", "BaseFind") Find = apps.get_model("archaeological_finds", "Find") for bf in instance.base_finds.all(): - cached_label_changed(BaseFind, instance=bf) + cached_label_changed(BaseFind, instance=bf, timestamp=timestamp) for f in bf.find.all(): - cached_label_changed(Find, instance=f) + cached_label_changed(Find, instance=f, timestamp=timestamp) post_save.connect(context_record_post_save, sender=ContextRecord) diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py index fa38a6dbb..7fc27c7cc 100644 --- a/archaeological_finds/models_finds.py +++ b/archaeological_finds/models_finds.py @@ -886,13 +886,15 @@ class BaseFind( def post_save_basefind(sender, **kwargs): + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp cached_label_changed(sender, **kwargs) post_save_geo(sender, **kwargs) instance = kwargs.get("instance", None) if not instance or not instance.pk: return for f in instance.find.all(): - cached_label_changed(Find, instance=f) + cached_label_changed(Find, instance=f, timestamp=timestamp) def pre_delete_basefind(sender, **kwargs): diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py index d933944d1..74efe2104 100644 --- a/archaeological_operations/models.py +++ b/archaeological_operations/models.py @@ -223,11 +223,15 @@ class GeographicTownItem(GeoItem): class Meta: abstract = True - def post_save_geo(self, save=True): + def post_save_geo(self, save=True, timestamp=None): # manage geodata towns if getattr(self, "_post_save_geo_ok", False): # prevent infinite loop - should not happen, but... return + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + if hasattr(self, "timestamp_geo") and (self.timestamp_geo or 0) >= timestamp: + return self._post_save_geo_ok = True q_towns = self.towns.filter(main_geodata__multi_polygon__isnull=False) q_towns_nb = q_towns.count() @@ -330,9 +334,11 @@ class GeographicTownItem(GeoItem): if changed and save: self.no_post_process() + self.timestamp_geo = timestamp self.save() post_save_geo(self.__class__, instance=self, created=False, - update_fields=False, raw=False, using="default") + update_fields=False, raw=False, using="default", + timestamp=timestamp) return changed @@ -1048,6 +1054,8 @@ class ArchaeologicalSite( def site_post_save(sender, **kwargs): + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp cached_label_changed(sender=sender, **kwargs) post_save_geo(sender=sender, **kwargs) @@ -1546,9 +1554,11 @@ class Operation( POST_PROCESS_REQUEST = { "towns__numero_insee__startswith": "_get_department_code", } - DOWN_MODEL_UPDATE = ["parcels", "context_record"] - + DOWN_MODEL_REVERSE_QUERY = { + "context_record": "operation_id", + "parcels": "operation_id" + } HISTORICAL_M2M = [ "remains", "towns", @@ -2675,6 +2685,8 @@ for attr in Operation.HISTORICAL_M2M: def operation_post_save(sender, **kwargs): if not kwargs["instance"]: return + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp post_save_geo(sender=sender, **kwargs) operation = kwargs["instance"] @@ -2702,39 +2714,27 @@ def operation_post_save(sender, **kwargs): for parcel in operation.parcels.all(): parcel.copy_to_file() + """ + # TODO: delete - should be managed with DOWN_MODEL_UPDATE # external id, cached_labels update for parcel in operation.parcels.all(): - cached_label_changed(Parcel, instance=parcel) + cached_label_changed(Parcel, instance=parcel, timestamp=timestamp) ContextRecord = apps.get_model("archaeological_context_records", "ContextRecord") BaseFind = apps.get_model("archaeological_finds", "BaseFind") Find = apps.get_model("archaeological_finds", "Find") for cr in operation.context_record.all(): - cached_label_changed(ContextRecord, instance=cr) + cached_label_changed(ContextRecord, instance=cr, timestamp=timestamp) for bf in cr.base_finds.all(): - cached_label_changed(BaseFind, instance=bf) + cached_label_changed(BaseFind, instance=bf, timestamp=timestamp) for f in bf.find.all(): - cached_label_changed(Find, instance=f) + cached_label_changed(Find, instance=f, timestamp=timestamp) + """ post_save.connect(operation_post_save, sender=Operation) -def operation_town_m2m_changed(sender, **kwargs): - operation = kwargs.get("instance", None) - if not operation: - return - operation._prevent_loop = False - operation.regenerate_all_ids() - geotown_attached_changed(sender, **kwargs) - force_cached_label_changed(sender, **kwargs) - - -m2m_changed.connect( - operation_town_m2m_changed, sender=Operation.towns.through -) - - class RelationType(GeneralRelationType): class Meta: verbose_name = _("Operation relation type") @@ -3706,6 +3706,8 @@ def parcel_post_save(sender, **kwargs): if not kwargs["instance"]: return parcel = kwargs["instance"] + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp cached_label_changed(sender, **kwargs) if ( diff --git a/archaeological_operations/wizards.py b/archaeological_operations/wizards.py index 74840e552..94d6b49f9 100644 --- a/archaeological_operations/wizards.py +++ b/archaeological_operations/wizards.py @@ -87,7 +87,6 @@ class OperationWizard(Wizard): Return extra context for templates """ context = super(OperationWizard, self).get_context_data(form, **kwargs) - step = self.steps.current # reminder of the current file reminder = self.get_reminder() if reminder: @@ -198,15 +197,6 @@ class OperationModificationWizard(OperationWizard): modification = True filter_owns = {"selec-operation_modification": ["pk"]} - def get_form_kwargs(self, step, **kwargs): - kwargs = super(OperationModificationWizard, self).get_form_kwargs( - step, **kwargs - ) - if step != "relations-operation_modification": - return kwargs - kwargs["left_record"] = self.get_current_object() - return kwargs - class OperationClosingWizard(ClosingWizard): model = models.Operation diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py index 121e2bbb9..15cf1384c 100644 --- a/archaeological_warehouse/models.py +++ b/archaeological_warehouse/models.py @@ -2143,6 +2143,8 @@ class ContainerLocalisation(models.Model): def save(self, *args, **kwargs): super(ContainerLocalisation, self).save(*args, **kwargs) self.container.skip_history_when_saving = True - cached_label_changed(Container, instance=self.container, force_update=True) + timestamp = int(datetime.datetime.now().timestamp()) + cached_label_changed(Container, instance=self.container, force_update=True, + timestamp=timestamp) diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index e2495bcd5..d09d64d88 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -66,6 +66,7 @@ from ishtar_common.model_merging import merge_model_objects from ishtar_common.models_imports import Import from ishtar_common.templatetags.link_to_window import simple_link_to_window from ishtar_common.utils import ( + bulk_item_changed, cached_label_changed, disable_for_loaddata, duplicate_item, @@ -1906,6 +1907,8 @@ class BaseHistorizedItem( ) if not created and not external_id_updated: self.update_external_id() + if not hasattr(self, "_timestamp"): + self._timestamp = int(datetime.datetime.now().timestamp()) super(BaseHistorizedItem, self).save(*args, **kwargs) if created and self.update_external_id(): # force resave for external ID creation @@ -3131,11 +3134,11 @@ class GeographicItem(models.Model): if getattr(self, "_geodata_list", None): return self._geodata_list lst = [] + q = self.geodata if self.main_geodata: lst.append(self.main_geodata) - for geo in self.geodata.all(): - if geo != self.main_geodata: - lst.append(geo) + q = q.exclude(id=self.main_geodata_id) + lst += [geo for geo in self.geodata.all()] self._geodata_list = lst return lst @@ -3363,6 +3366,7 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem): SLUG = "" SHOW_URL = None DOWN_MODEL_UPDATE = [] + DOWN_MODEL_REVERSE_QUERY = {} INITIAL_VALUES = [] # list of field checkable if changed on save OLD_SHEET_EXPORT = True @@ -3385,11 +3389,15 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem): self._initial_values[field_name] = value return changed - def cascade_update(self, changed=True): + def cascade_update(self, changed=True, timestamp=None): if not changed: return if getattr(self, "_no_down_model_update", False): return + + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + queue = getattr(self, "_queue", settings.CELERY_DEFAULT_QUEUE) for down_model in self.DOWN_MODEL_UPDATE: if not settings.USE_BACKGROUND_TASK: @@ -3397,14 +3405,27 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem): if hasattr(rel.model, "need_update"): rel.update(need_update=True) continue + if down_model in self.DOWN_MODEL_REVERSE_QUERY: + key = self.DOWN_MODEL_REVERSE_QUERY[down_model] + new_down_model = getattr(self, down_model).model + if hasattr(new_down_model, "bulk_item_changed"): + new_down_model.bulk_item_changed({key: self.pk}) + continue for item in getattr(self, down_model).all(): if hasattr(self, "_timestamp"): item._timestamp = self._timestamp item._queue = queue if hasattr(item, "cached_label_changed"): - item.cached_label_changed() + if hasattr(item, "timestamp_label") and \ + (item.timestamp_label or 0) >= timestamp: + pass + else: + item.cached_label_changed() + if hasattr(item, "timestamp_geo") and \ + (item.timestamp_geo or 0) >= timestamp: + continue if hasattr(item, "main_geodata"): - item.post_save_geo() + item.post_save_geo(timestamp=timestamp) def no_post_process(self): self.skip_history_when_saving = True @@ -3490,16 +3511,30 @@ class MainItem(ShortMenuItem, SerializeItem, SheetItem): self._no_move = True self.save() - def cached_label_changed(self): + @classmethod + def bulk_item_changed(cls, query_dict, timestamp=None): + """ + Trigger changes in a bulk item list set by a query_dict (must be serializabled) + """ + if not settings.USE_BACKGROUND_TASK: + for item in cls.objects.filter(**query_dict).all(): + item.cached_label_changed(timestamp=timestamp) + return + bulk_item_changed(cls, query=query_dict, timestamp=timestamp) + + def cached_label_changed(self, timestamp=None): self.no_post_process() self._cached_label_checked = False - cached_label_changed(self.__class__, instance=self, created=False) + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + cached_label_changed(self.__class__, instance=self, created=False, + timestamp=timestamp) - def post_save_geo(self, save=True): + def post_save_geo(self, save=True, timestamp=None): if getattr(self, "_post_saved_geo", False): return self.no_post_process() - post_save_geo(self.__class__, instance=self, created=False) + post_save_geo(self.__class__, instance=self, created=False, timestamp=timestamp) return False def external_id_changed(self): @@ -3784,6 +3819,8 @@ class Town(GeographicItem, Imported, DocumentItem, MainItem, models.Model): def post_save_town(sender, **kwargs): + timestamp = int(datetime.datetime.now().timestamp()) + kwargs["timestamp"] = timestamp cached_label_changed(sender, **kwargs) town = kwargs["instance"] town.generate_geo() @@ -3812,7 +3849,6 @@ def geotown_attached_changed(sender, **kwargs): return instance = kwargs.get("instance", None) model = kwargs.get("model", None) - pk_set = kwargs.get("pk_set", None) action = kwargs.get("action", None) if not instance or not model or not hasattr(instance, "post_save_geo"): return diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index cf58216a4..bcba9c52a 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -1123,14 +1123,51 @@ def load_task(task_func, task_name, checks, sender, queue=None, **kwargs): return task_item +def load_query_task(task_func, task_name, sender, queue=None, **kwargs): + if not queue: + queue = settings.CELERY_DEFAULT_QUEUE + query = kwargs.get("query", None) + if not query: + return + + if not settings.USE_BACKGROUND_TASK: + # no background task + return task_func(sender, **kwargs) + + sender, kwargs = serialize_args_for_tasks( + sender, query, kwargs, EXTRA_KWARGS_TRIGGER + ) + kwargs["queue"] = queue + task_item = task_func.apply_async([sender], kwargs, queue=queue) + revoke_old_task(kwargs, task_name, task_item.id, sender) + return task_item + + +def bulk_item_changed(sender, **kwargs): + load_query_task(_bulk_item_changed, "bulk_item_changed", sender, "low_priority") + + +@task() +def _bulk_item_changed(sender, **kwargs): + sender, query = deserialize_args_for_tasks(sender, kwargs, EXTRA_KWARGS_TRIGGER) + if not query: + return + for item in sender.objects.filter(**query).all(): + kwargs["instance"] = item + cached_label_changed(sender, **kwargs) + post_save_geo(sender, **kwargs) + + def cached_label_changed(sender, **kwargs): if "instance" not in kwargs: return instance = kwargs["instance"] if not instance: return - if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label") and ( - instance.timestamp_label or 0) >= (instance._timestamp or 0): + timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None) + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + if hasattr(instance, "timestamp_label") and (instance.timestamp_label or 0) >= timestamp: return queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE) if hasattr(instance, "external_id") and hasattr(instance, "auto_external_id") \ @@ -1160,10 +1197,15 @@ def _cached_label_changed(sender, **kwargs): if not force_update and getattr(instance, "_cached_label_checked", False): return - if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label"): - if (instance.timestamp_label or 0) >= (instance._timestamp or 0): + timestamp = kwargs.get("timestamp", None) + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + if hasattr(instance, "timestamp_label"): + if (instance.timestamp_label or 0) >= timestamp: return - instance.__class__.objects.filter(pk=instance.pk).update(timestamp_label=instance._timestamp) + instance.__class__.objects.filter(pk=instance.pk).update( + timestamp_label=timestamp + ) instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE) logger.debug(f"[ishtar] ishtar_common.utils._cached_label_changed - {instance.__class__.__name__} - {instance.pk} - {instance}") @@ -1197,7 +1239,7 @@ def _cached_label_changed(sender, **kwargs): instance.__class__.objects.filter(pk=instance.pk).update(**dict(changed)) if ((getattr(instance, "check_cascade_update", False) and instance.check_cascade_update()) or changed or not cached_labels) and hasattr(instance, "cascade_update"): - instance.cascade_update() + instance.cascade_update(timestamp=timestamp) updated = False if force_update or hasattr(instance, "update_search_vector"): updated = instance.update_search_vector() @@ -1206,9 +1248,7 @@ def _cached_label_changed(sender, **kwargs): item._cascade_change = True if hasattr(instance, "test_obj"): item.test_obj = instance.test_obj - if instance.timestamp_label: - item._timestamp = instance.timestamp_label - cached_label_changed(item.__class__, instance=item) + cached_label_changed(item.__class__, instance=item, timestamp=timestamp) cache_key, __ = get_cache(sender, ["cached_label_changed", instance.pk]) cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT) if cached_labels: @@ -1495,6 +1535,12 @@ def post_save_geo(sender, **kwargs): return if getattr(instance, "_post_saved_geo", False): return + timestamp = kwargs.get("timestamp", None) or getattr(instance, "_timestamp", None) + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + elif (instance.timestamp_geo or 0) >= timestamp: + return + queue = getattr(instance, "_queue", settings.CELERY_DEFAULT_QUEUE) return load_task(_post_save_geo, "post_save_geo", ["_no_geo_check"], sender, queue=queue, **kwargs) @@ -1515,10 +1561,15 @@ def _post_save_geo(sender, **kwargs): if getattr(instance, "_post_saved_geo", False): return - if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_geo"): - if (instance.timestamp_label or 0) >= (instance._timestamp or 0): + timestamp = kwargs.get("timestamp", None) + if hasattr(instance, "timestamp_geo"): + if timestamp and (instance.timestamp_geo or 0) >= timestamp: return - instance.__class__.objects.filter(pk=instance.pk).update(timestamp_geo=instance._timestamp) + if not timestamp: + timestamp = int(datetime.datetime.now().timestamp()) + instance.__class__.objects.filter(pk=instance.pk).update( + timestamp_geo=timestamp + ) instance._queue = kwargs.get("queue", settings.CELERY_DEFAULT_QUEUE) logger.debug(f"[ishtar] ishtar_common.utils._post_save_geo - {instance.__class__.__name__} - {instance.pk} - {instance}") @@ -1541,7 +1592,7 @@ def _post_save_geo(sender, **kwargs): instance._cached_label_checked = False instance.save() if hasattr(instance, "cascade_update"): - instance.cascade_update() + instance.cascade_update(timestamp=timestamp) cache_key, __ = get_cache(sender, ["post_save_geo", instance.pk]) cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT) return -- cgit v1.2.3