⚡️ improve parcel post-treatments - add timestamp to prevent multiple geo and cached_label edition

5 files changed, 183 insertions, 18 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 8c46cadb6..0f447f957 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -873,12 +873,14 @@ class Importer(object):
             self.import_instance.state = "PP"
             self.import_instance.imported_line_numbers = ""
             self.import_instance.save()
+        self.timestamp = int(datetime.datetime.now().timestamp())
         for cls_pk, idx_line in self.post_save_items.items():
             if self.import_instance and not self.line_to_process:
                 self.import_instance.add_imported_line(idx_line)
             cls, pk = cls_pk
             # force django based post-processing for the item
             item = cls.objects.get(pk=pk)
+            item._timestamp = self.timestamp
             item.save()
             if hasattr(item, "RELATED_POST_PROCESS"):
                 for related_key in item.RELATED_POST_PROCESS:
@@ -892,6 +894,7 @@ class Importer(object):
         for cls, pk in related_list.keys():
             try:
                 item = cls.objects.get(pk=pk)
+                item._timestamp = self.timestamp
                 item.save()
                 if hasattr(item, "fix"):
                     # post save/m2m specific fix
@@ -1105,6 +1108,7 @@ class Importer(object):
         return obj
 
     def _line_processing(self, idx_line, line):
+        self.timestamp = int(datetime.datetime.now().timestamp())
         for item in self.to_be_close:
             item.close()
         self.to_be_close = []
@@ -1212,6 +1216,7 @@ class Importer(object):
             for k in data["defaults"]:
                 setattr(obj, k, data["defaults"][k])
             obj._no_post_save = True
+            obj._timestamp = self.timestamp
             obj.save()
             self._add_to_post_save(obj.__class__, obj.pk, idx_line)
 
@@ -1250,6 +1255,7 @@ class Importer(object):
                 if item:
                     for k in geodata:
                         setattr(item, k, geodata[k])
+                    item._timestamp = self.timestamp
                     item.save()
                 else:
                     item = GeoVectorData.objects.create(**geodata)
@@ -1260,6 +1266,7 @@ class Importer(object):
             if self.import_instance and created:
                 item.imports.add(self.import_instance)
             if self.MAIN_GEO:
+                obj._timestamp = self.timestamp
                 obj.main_geodata = item
                 obj._post_saved_geo = True
                 obj._no_move = True
@@ -1335,6 +1342,7 @@ class Importer(object):
                 for k in data["defaults"]:
                     setattr(t_obj, k, data["defaults"][k])
                 t_obj._no_post_save = True
+                t_obj._timestamp = self.timestamp
                 t_obj.save()
                 self._add_to_post_save(t_obj.__class__, t_obj.pk, idx_line)
             if self.import_instance and hasattr(t_obj, "imports") and created:
@@ -1674,6 +1682,7 @@ class Importer(object):
                             changed = True
                             setattr(v, k, extra_fields[k])
                     if changed:
+                        v._timestamp = self.timestamp
                         v.save()
                 for att, objs in m2m_m2ms:
                     if type(objs) not in (list, tuple):
@@ -2022,6 +2031,7 @@ class Importer(object):
                             else:
                                 for k in updated_dct:
                                     setattr(obj, k, updated_dct[k])
+                                obj._timestamp = self.timestamp
                                 obj.save()
                 if (
                     not self.simulate
@@ -2090,6 +2100,7 @@ class Importer(object):
                         self._add_to_post_save(v.__class__, v.pk, idx_line)
                         v._no_post_save = True
                         try:
+                            v._timestamp = self.timestamp
                             v.save()
                         except DatabaseError as e:
                             raise IntegrityError(e.message)
diff --git a/ishtar_common/migrations/0230_auto_20231114_1334.py b/ishtar_common/migrations/0230_auto_20231114_1334.py
new file mode 100644
index 000000000..15da327d4
--- /dev/null
+++ b/ishtar_common/migrations/0230_auto_20231114_1334.py
@@ -0,0 +1,128 @@
+# Generated by Django 2.2.24 on 2023-11-14 13:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0229_auto_20230608_1303'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='geovectordata',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='geovectordata',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='historicaldocument',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='historicaldocument',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='historicalorganization',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='historicalorganization',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='historicalperson',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='historicalperson',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='person',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='person',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AddField(
+            model_name='town',
+            name='timestamp_geo',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp geo'),
+        ),
+        migrations.AddField(
+            model_name='town',
+            name='timestamp_label',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Timestamp label'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='account_naming_style',
+            field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='basefind_cached_label',
+            field=models.TextField(blank=True, default='{context_record__cached_label} | {label}', help_text='Formula to manage cached label. If not set a default formula is used.', verbose_name='Base find cached label'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='contextrecord_cached_label',
+            field=models.TextField(default='{parcel__cached_label} | {label}', help_text='Formula to manage cached label. If not set a default formula is used.', verbose_name='Context record cached label'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='find_cached_label',
+            field=models.TextField(default='{get_first_base_find__context_record__operation__complete_identifier}-{index:0>5} | {label}', help_text='Formula to manage cached label. If not set a default formula is used.', verbose_name='Find cached label'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='operation_cached_label',
+            field=models.TextField(default='{{cached_towns_label}} | {% if code_patriarche %}OA{{code_patriarche}}{% else %}{% if operation_code and year %}OP{{year}}-{{operation_code}}{% endif %}{% endif %}', help_text='Formula to manage cached label. If not set a default formula is used.', verbose_name='Operation cached label'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='operation_complete_identifier',
+            field=models.TextField(default='{% if code_patriarche %}OA{{code_patriarche}}{% else %}{% if operation_code and year %}{{year}}-{{operation_code}}{% else %}-{% endif %}{% endif %}', help_text='Formula to manage operation complete identifier.', verbose_name='Operation complete identifier'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='parcel_cached_label',
+            field=models.TextField(default='{associated_file__external_id}{operation__complete_identifier} | {town__numero_insee} | {section}{parcel_number}', help_text='Formula to manage cached label. If not set a default formula is used.', verbose_name='Parcel cached label'),
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 75e07d7cb..a1fd5e714 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -560,6 +560,7 @@ class BulkUpdatedItem(object):
         :param extra_args: arguments dealing with
         :return: (transaction ID, is a recursion)
         """
+        # TODO: no more bulk update - should be removed
         if not transaction_id:
             transaction_id = str(time.time())
         args = ["cached_label_bulk_update", transaction_id] + extra_args
@@ -1169,8 +1170,9 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     operation_complete_identifier = models.TextField(
         _("Operation complete identifier"),
-        default="",
-        blank=True,
+        default="{% if code_patriarche %}OA{{code_patriarche}}{% else %}"
+                "{% if operation_code and year %}{{year}}-{{operation_code}}"
+                "{% else %}-{% endif %}{% endif %}",
         help_text=_("Formula to manage operation complete identifier."),
     )
     operation_custom_index = models.TextField(
@@ -1184,8 +1186,8 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     operation_cached_label = models.TextField(
         _("Operation cached label"),
-        default="",
-        blank=True,
+        default="{{cached_towns_label}} | {% if code_patriarche %}OA{{code_patriarche}}{% else %}"
+                "{% if operation_code and year %}OP{{year}}-{{operation_code}}{% endif %}{% endif %}",
         help_text=_(
             "Formula to manage cached label. If not set a default formula is used."
         ),
@@ -1227,7 +1229,7 @@ class IshtarSiteProfile(models.Model, Cached):
         _("Archaeological file complete identifier"),
         default="",
         blank=True,
-        help_text=_("Formula to manage archaeological file complete " "identifier."),
+        help_text=_("Formula to manage archaeological file complete identifier."),
     )
     file_custom_index = models.TextField(
         _("Archaeological file custom index key"),
@@ -1259,8 +1261,8 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     parcel_cached_label = models.TextField(
         _("Parcel cached label"),
-        default="",
-        blank=True,
+        default="{associated_file__external_id}{operation__complete_identifier} | "
+                "{town__numero_insee} | {section}{parcel_number}",
         help_text=_(
             "Formula to manage cached label. If not set a default formula is used."
         ),
@@ -1292,8 +1294,7 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     contextrecord_cached_label = models.TextField(
         _("Context record cached label"),
-        default="",
-        blank=True,
+        default="{parcel__cached_label} | {label}",
         help_text=_(
             "Formula to manage cached label. If not set a default formula is used."
         ),
@@ -1325,7 +1326,7 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     basefind_cached_label = models.TextField(
         _("Base find cached label"),
-        default="",
+        default="{context_record__cached_label} | {label}",
         blank=True,
         help_text=_(
             "Formula to manage cached label. If not set a default formula is used."
@@ -1358,8 +1359,7 @@ class IshtarSiteProfile(models.Model, Cached):
     )
     find_cached_label = models.TextField(
         _("Find cached label"),
-        default="",
-        blank=True,
+        default="{get_first_base_find__context_record__operation__complete_identifier}-{index:0>5} | {label}",
         help_text=_(
             "Formula to manage cached label. If not set a default formula is used."
         ),
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 2074aa523..0345eada2 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -1080,6 +1080,8 @@ class Imported(models.Model):
     imports = models.ManyToManyField(
         Import, blank=True, related_name="imported_%(app_label)s_%(class)s"
     )
+    timestamp_geo = models.IntegerField(_("Timestamp geo"), null=True, blank=True)
+    timestamp_label = models.IntegerField(_("Timestamp label"), null=True, blank=True)
 
     class Meta:
         abstract = True
@@ -3329,6 +3331,8 @@ class MainItem(ShortMenuItem, SerializeItem):
                     rel.update(need_update=True)
                     continue
             for item in getattr(self, down_model).all():
+                if hasattr(self, "_timestamp"):
+                    item._timestamp = self._timestamp
                 if hasattr(item, "cached_label_changed"):
                     item.cached_label_changed()
                 if hasattr(item, "main_geodata"):
@@ -3417,10 +3421,13 @@ class MainItem(ShortMenuItem, SerializeItem):
     def regenerate_external_id(self):
         if not hasattr(self, "external_id"):
             return
+        try:
+            self.external_id = ""
+            self.auto_external_id = True
+        except AttributeError:
+            return
         self.skip_history_when_saving = True
         self._no_move = True
-        self.external_id = ""
-        self.auto_external_id = True
         self.save()
 
     def cached_label_changed(self):
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index bbefd085e..d5ace858d 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -487,6 +487,7 @@ EXTRA_KWARGS_TRIGGER = [
     "_post_saved_geo",
     "_search_updated",
     "_cached_label_checked",
+    "_timestamp",
 ]
 
 
@@ -549,6 +550,9 @@ def cached_label_changed(sender, **kwargs):
     instance = kwargs["instance"]
     if not instance:
         return
+    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label") and (
+            instance.timestamp_label or 0) >= (instance._timestamp or 0):
+        return
     if hasattr(instance, "external_id") and hasattr(instance, "auto_external_id") \
             and hasattr(instance, "SLUG") and not getattr(instance, "_external_id_checked", None):
         changed = load_task(_external_id_changed, "external_id_changed", None, sender, **kwargs)
@@ -581,6 +585,11 @@ def _cached_label_changed(sender, **kwargs):
     if not force_update and getattr(instance, "_cached_label_checked", False):
         return
 
+    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_label"):
+        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+            return
+        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_label=instance._timestamp)
+
     logger.debug(f"[ishtar] ishtar_common.utils._cached_label_changed - {instance.__class__.__name__} - {instance.pk} - {instance}")
     if hasattr(instance, "refresh_cache"):
         instance.refresh_cache()
@@ -625,6 +634,8 @@ def _cached_label_changed(sender, **kwargs):
             item._cascade_change = True
             if hasattr(instance, "test_obj"):
                 item.test_obj = instance.test_obj
+            if instance.timestamp_label:
+                item._timestamp = instance.timestamp_label
             cached_label_changed(item.__class__, instance=item)
             
     cache_key, __ = get_cache(sender, ["cached_label_changed", instance.pk])
@@ -672,9 +683,12 @@ def _external_id_changed(sender, **kwargs):
     if not instance.external_id or instance.auto_external_id:
         external_id = get_generated_id(instance.SLUG + "_external_id", instance)
         if external_id != instance.external_id:
-            updated = True
-            instance.auto_external_id = True
-            instance.external_id = external_id
+            try:
+                instance.auto_external_id = True
+                instance.external_id = external_id
+                updated = True
+            except AttributeError:
+                pass
     if hasattr(instance, "regenerate_all_ids"):
         updated |= instance.regenerate_all_ids(save=False) or False
     instance._external_id_checked = True
@@ -909,6 +923,11 @@ def _post_save_geo(sender, **kwargs):
     if getattr(instance, "_post_saved_geo", False):
         return
 
+    if hasattr(instance, "_timestamp") and hasattr(instance, "timestamp_geo"):
+        if (instance.timestamp_label or 0) >= (instance._timestamp or 0):
+            return
+        instance.__class__.objects.filter(pk=instance.pk).update(timestamp_geo=instance._timestamp)
+
     logger.debug(f"[ishtar] ishtar_common.utils._post_save_geo - {instance.__class__.__name__} - {instance.pk} - {instance}")
     instance._post_saved_geo = True
 
@@ -1964,7 +1983,7 @@ def get_current_profile(force=False):
     return IshtarSiteProfile.get_current_profile(force=force)
 
 
-PARSE_FORMULA = re.compile(r"{([^}^\:]*)(?::.*)?}")
+PARSE_FORMULA = re.compile(r"{([^}^\\:]*)(?::[^}]*)?}")
 PARSE_JINJA = re.compile("{{([^}]*)}")
 PARSE_JINJA_IF = re.compile("{% if ([^}]*)}")