Better management of cascade updates

4 files changed, 142 insertions, 3 deletions

diff --git a/ishtar_common/management/commands/process_pending_update.py b/ishtar_common/management/commands/process_pending_update.py
new file mode 100644
index 000000000..f0e908d42
--- /dev/null
+++ b/ishtar_common/management/commands/process_pending_update.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013-2018 Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import sys
+
+from django.core.management.base import BaseCommand
+
+from django.apps import apps
+
+
+APPS = ['ishtar_common', 'archaeological_operations',
+        'archaeological_context_records', 'archaeological_finds',
+        'archaeological_warehouse']
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Process pending update of cached labels and geo position'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--quiet', dest='quiet', action='store_true',
+            help='Quiet output')
+
+    def handle(self, *args, **options):
+        quiet = options['quiet']
+        for app in APPS:
+            if not quiet:
+                print(u"* app: {}".format(app))
+            for model in apps.get_app_config(app).get_models():
+                if model.__name__.startswith('Historical'):
+                    continue
+                if not bool(
+                        [True for k in model._meta.get_fields()
+                         if k == "need_update"]):
+                    continue
+                msg = u"-> processing {}: ".format(model._meta.verbose_name)
+                ln = model.objects.count()
+                for idx, obj_id in enumerate(
+                        model.objects.filter(
+                            need_update=True).values('pk').all()):
+                    obj = model.objects.get(pk=obj_id['pk'])
+                    obj.skip_history_when_saving = True
+                    obj._no_move = True
+                    cmsg = u"\r{} {}/{}".format(msg, idx + 1, ln)
+                    if not quiet:
+                        sys.stdout.write(cmsg)
+                        sys.stdout.flush()
+                    obj.save()
+                if not quiet:
+                    sys.stdout.write("\n")
diff --git a/ishtar_common/migrations/0097_auto_20190628_1256.py b/ishtar_common/migrations/0097_auto_20190628_1256.py
new file mode 100644
index 000000000..f8713d64d
--- /dev/null
+++ b/ishtar_common/migrations/0097_auto_20190628_1256.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-06-28 12:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0096_tinyurl'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='need_update',
+            field=models.BooleanField(default=False, verbose_name='Need update'),
+        ),
+        migrations.AddField(
+            model_name='historicalorganization',
+            name='need_update',
+            field=models.BooleanField(default=False, verbose_name='Need update'),
+        ),
+        migrations.AddField(
+            model_name='historicalperson',
+            name='need_update',
+            field=models.BooleanField(default=False, verbose_name='Need update'),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='need_update',
+            field=models.BooleanField(default=False, verbose_name='Need update'),
+        ),
+        migrations.AddField(
+            model_name='person',
+            name='need_update',
+            field=models.BooleanField(default=False, verbose_name='Need update'),
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a03f9f387..93a099b74 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -87,7 +87,7 @@ from ishtar_common.models_imports import ImporterModel, ImporterType, \
     Import, TargetKeyGroup, ValueFormater
 from ishtar_common.templatetags.link_to_window import simple_link_to_window
 from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug, \
-    get_all_field_names, merge_tsvectors, cached_label_changed, \
+    get_all_field_names, merge_tsvectors, cached_label_changed, post_save_geo, \
     generate_relation_graph, max_size_help, task, SecretaryRenderer
 
 __all__ = [
@@ -1963,6 +1963,7 @@ class BaseHistorizedItem(StatisticItem, TemplateItem, FullSearch, Imported,
     SHOW_URL = None
     EXTERNAL_ID_KEY = ''
     EXTERNAL_ID_DEPENDENCIES = []
+    DOWN_MODEL_UPDATE = []
     HISTORICAL_M2M = []
 
     history_modifier = models.ForeignKey(
@@ -1973,6 +1974,8 @@ class BaseHistorizedItem(StatisticItem, TemplateItem, FullSearch, Imported,
         verbose_name=_("Creator"), blank=True, null=True)
     last_modified = models.DateTimeField(auto_now=True)
     history_m2m = JSONField(default={}, blank=True)
+    need_update = models.BooleanField(
+        verbose_name=_("Need update"), default=False)
 
     ALT_NAMES = {
         'history_creator': SearchAltName(
@@ -1992,6 +1995,15 @@ class BaseHistorizedItem(StatisticItem, TemplateItem, FullSearch, Imported,
     class Meta:
         abstract = True
 
+    def cascade_update(self):
+        for down_model in self.DOWN_MODEL_UPDATE:
+            if not settings.USE_BACKGROUND_TASK:
+                getattr(self, down_model).update(need_update=True)
+                continue
+            for item in getattr(self, down_model).all():
+                cached_label_changed(item.model, instance=item)
+                post_save_geo(item.model, instance=item)
+
     @classmethod
     def get_verbose_name(cls):
         return cls._meta.verbose_name
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index a2fdd36f3..f3df5c282 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -419,6 +419,11 @@ EXTRA_KWARGS_TRIGGER = [
 ]
 
 
+def cached_label_and_geo_changed(sender, **kwargs):
+    cached_label_changed(sender=sender, **kwargs)
+    post_save_geo(sender=sender, **kwargs)
+
+
 def cached_label_changed(sender, **kwargs):
     if not kwargs.get('instance'):
         return
@@ -454,11 +459,13 @@ def _cached_label_changed(sender, **kwargs):
         return
     force_update = kwargs.get('force_update', False)
 
+    if hasattr(instance, "need_update") and instance.need_update:
+        force_update = True
+        instance.skip_history_when_saving = True
+
     if not force_update and getattr(instance, '_cached_label_checked', False):
         return
 
-    force_update = kwargs.get('force_update', False)
-
     if hasattr(instance, "refresh_cache"):
         instance.refresh_cache()
 
@@ -475,12 +482,19 @@ def _cached_label_changed(sender, **kwargs):
         if lbl != getattr(instance, cached_label):
             changed.append((cached_label, lbl))
             setattr(instance, cached_label, lbl)  # update for cache
+
+    if hasattr(instance, "need_update") and instance.need_update:
+        changed.append(("need_update", False))
+        instance.need_update = False
+
     if changed:
         instance._search_updated = False
         if hasattr(instance, '_cascade_change') and instance._cascade_change:
             instance.skip_history_when_saving = True
         instance.__class__.objects.filter(pk=instance.pk).update(
             **dict(changed))
+    if (changed or not cached_labels) and hasattr(instance, "cascade_update"):
+        instance.cascade_update()
     updated = False
     if force_update or hasattr(instance, 'update_search_vector'):
         updated = instance.update_search_vector()
@@ -837,11 +851,17 @@ def _post_save_geo(sender, **kwargs):
                 instance.point_source = None
                 modified = True
 
+    if hasattr(instance, "need_update") and instance.need_update:
+        instance.need_update = False
+        modified = True
+
     if modified:
         instance.skip_history_when_saving = True
         instance._post_saved_geo = True
         instance._cached_label_checked = False
         instance.save()
+        if hasattr(instance, "cascade_update"):
+            instance.cascade_update()
     cache_key, __ = get_cache(
         sender, ["post_save_geo", instance.pk]
     )