Celery: manage stats generation on sheets

3 files changed, 262 insertions, 14 deletions

diff --git a/ishtar_common/migrations/0089_auto_20190315_2114.py b/ishtar_common/migrations/0089_auto_20190315_2114.py
new file mode 100644
index 000000000..e6a16b4fc
--- /dev/null
+++ b/ishtar_common/migrations/0089_auto_20190315_2114.py
@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-03-15 21:14
+from __future__ import unicode_literals
+
+import django.contrib.gis.db.models.fields
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import virtualtime
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0088_auto_20190218_1808'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StatsCache',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('model', models.CharField(max_length=200, verbose_name='Model name')),
+                ('model_pk', models.IntegerField(verbose_name='Associated primary key')),
+                ('values', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default={})),
+                ('updated', models.DateTimeField(default=virtualtime.virtual_datetime.now)),
+                ('update_requested', models.DateTimeField(blank=True, null=True)),
+            ],
+            options={
+                'verbose_name_plural': 'Caches for stats',
+                'verbose_name': 'Cache for stats',
+            },
+        ),
+        migrations.AlterField(
+            model_name='administrationtask',
+            name='state',
+            field=models.CharField(choices=[('S', 'Planifié'), ('P', 'En cours'), ('FE', 'Terminé avec des erreurs'), ('F', 'Terminé')], default='S', max_length=2, verbose_name='État'),
+        ),
+        migrations.AlterField(
+            model_name='customformjsonfield',
+            name='label',
+            field=models.CharField(blank=True, default='', max_length=200, verbose_name='Dénomination'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='title',
+            field=models.TextField(blank=True, default='', verbose_name='Titre'),
+        ),
+        migrations.AlterField(
+            model_name='documenttemplate',
+            name='associated_object_name',
+            field=models.CharField(choices=[('archaeological_operations.models.AdministrativeAct', 'Acte administratif')], max_length=100, verbose_name='Objet associé'),
+        ),
+        migrations.AlterField(
+            model_name='documenttemplate',
+            name='template',
+            field=models.FileField(help_text='La taille maximale supportée pour le fichier est de 100 Mo.', upload_to='templates/%Y/', verbose_name='Patron'),
+        ),
+        migrations.AlterField(
+            model_name='formatertype',
+            name='formater_type',
+            field=models.CharField(choices=[('IntegerFormater', 'Entier'), ('FloatFormater', 'Nombre à virgule'), ('UnicodeFormater', 'Chaîne de caractères'), ('DateFormater', 'Date'), ('TypeFormater', 'Type'), ('YearFormater', 'Année'), ('InseeFormater', 'Code INSEE'), ('StrToBoolean', 'Chaîne de caractères vers booléen'), ('FileFormater', 'Fichier'), ('UnknowType', 'Type inconnu')], max_length=20, verbose_name='Formater type'),
+        ),
+        migrations.AlterField(
+            model_name='historicalorganization',
+            name='town',
+            field=models.CharField(blank=True, max_length=150, null=True, verbose_name='Commune (saisie libre)'),
+        ),
+        migrations.AlterField(
+            model_name='historicalperson',
+            name='old_title',
+            field=models.CharField(blank=True, choices=[('Mr', 'M.'), ('Ms', 'Mlle'), ('Mr and Miss', 'M. et Mme'), ('Md', 'Mme'), ('Dr', 'Dr.')], max_length=100, null=True, verbose_name='Titre'),
+        ),
+        migrations.AlterField(
+            model_name='historicalperson',
+            name='town',
+            field=models.CharField(blank=True, max_length=150, null=True, verbose_name='Commune (saisie libre)'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='encoding',
+            field=models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', max_length=15, verbose_name='Codage'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='error_file',
+            field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Fichier erreur'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='imported_file',
+            field=models.FileField(help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, upload_to='upload/imports/%Y/%m/', verbose_name='Fichier importé'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='imported_images',
+            field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Images associées (fichier zip)'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='match_file',
+            field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Fichier de correspondance'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='result_file',
+            field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Fichier résultant'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='state',
+            field=models.CharField(choices=[('C', 'Créé'), ('AP', 'Analyse en cours'), ('A', 'Analysé'), ('HQ', 'Vérification des modifications dans la file'), ('IQ', "Import en file d'attente"), ('HP', 'Vérification des modifications en cours'), ('IP', 'Import en cours'), ('PI', 'Importé partiellement'), ('FE', 'Terminé avec des erreurs'), ('F', 'Terminé'), ('AC', 'Archivé')], default='C', max_length=2, verbose_name='État'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='archaeological_site_label',
+            field=models.CharField(choices=[('site', 'Site'), ('entity', 'Entité (EA)')], default='site', max_length=200, verbose_name='Type de site archéologique'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='config',
+            field=models.CharField(blank=True, choices=[('DRASSM', 'DRASSM')], help_text='Choisir une configuration alternative pour les libellés, gestion des index', max_length=200, null=True, verbose_name='Configuration alternative'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='default_center',
+            field=django.contrib.gis.db.models.fields.PointField(default='SRID=4326;POINT(2.4397 46.5528)', srid=4326, verbose_name='Cartes - centre par défaut'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='default_zoom',
+            field=models.IntegerField(default=6, verbose_name='Cartes - zoom par défaut'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='find_index',
+            field=models.CharField(choices=[('O', 'Opérations'), ('CR', "Unités d'Enregistrement")], default='O', help_text="Pour éviter des index non pertinents, ne changer ce paramètre que s'il n'y a pas encore de mobilier dans cette base de données", max_length=2, verbose_name='Index mobilier basé sur'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='use_town_for_geo',
+            field=models.BooleanField(default=True, verbose_name='Utiliser la commune pour la localisation quand les coordonnées manques'),
+        ),
+        migrations.AlterField(
+            model_name='jsondatafield',
+            name='value_type',
+            field=models.CharField(choices=[('T', 'Texte'), ('LT', 'Texte long'), ('I', 'Entier'), ('B', 'Booléen'), ('F', 'Nombre à virgule'), ('D', 'Date'), ('C', 'Choix')], default='T', max_length=10, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='precise_town',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Town', verbose_name='Commune (précis)'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='town',
+            field=models.CharField(blank=True, max_length=150, null=True, verbose_name='Commune (saisie libre)'),
+        ),
+        migrations.AlterField(
+            model_name='person',
+            name='old_title',
+            field=models.CharField(blank=True, choices=[('Mr', 'M.'), ('Ms', 'Mlle'), ('Mr and Miss', 'M. et Mme'), ('Md', 'Mme'), ('Dr', 'Dr.')], max_length=100, null=True, verbose_name='Titre'),
+        ),
+        migrations.AlterField(
+            model_name='person',
+            name='precise_town',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Town', verbose_name='Commune (précis)'),
+        ),
+        migrations.AlterField(
+            model_name='person',
+            name='town',
+            field=models.CharField(blank=True, max_length=150, null=True, verbose_name='Commune (saisie libre)'),
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index b535d9bf7..bacfd2147 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -83,7 +83,7 @@ from ishtar_common.models_imports import ImporterModel, ImporterType, \
 from ishtar_common.templatetags.link_to_window import simple_link_to_window
 from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug, \
     get_all_field_names, merge_tsvectors, cached_label_changed, \
-    generate_relation_graph, max_size_help
+    generate_relation_graph, max_size_help, task
 
 __all__ = [
     'ImporterModel', 'ImporterType', 'ImporterDefault', 'ImporterDefaultValues',
@@ -2769,6 +2769,57 @@ class UserDashboard:
             .order_by('person__person_types')
 
 
+class StatsCache(models.Model):
+    model = models.CharField(_("Model name"), max_length=200)
+    model_pk = models.IntegerField(_("Associated primary key"))
+    values = JSONField(default={}, blank=True)
+    updated = models.DateTimeField(default=datetime.datetime.now)
+    update_requested = models.DateTimeField(blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Cache for stats")
+        verbose_name_plural = _("Caches for stats")
+
+
+def update_stats(statscache, item, funcname):
+    if not settings.USE_BACKGROUND_TASK:
+        statscache.values = getattr(item, funcname)()
+        statscache.updated = datetime.datetime.now()
+        statscache.save()
+        return statscache.values
+
+    now = datetime.datetime.now()
+    if statscache.update_requested and (
+            statscache.update_requested + datetime.timedelta(hours=12) > now):
+        return statscache.values
+    app_name = item._meta.app_label
+    model_name = item._meta.model_name
+    statscache.update_requested = now.isoformat()
+    statscache.save()
+    _update_stats.delay(app_name, model_name, item.pk, funcname)
+    return statscache.values
+
+
+@task()
+def _update_stats(app, model, model_pk, funcname):
+    model_name = app + "." + model
+    sc, __ = StatsCache.objects.get_or_create(
+        model=model_name, model_pk=model_pk
+    )
+    model = apps.get_model(app, model)
+    try:
+        item = model.objects.get(pk=model_pk)
+    except model.DoesNotExist:
+        return
+    sc.values = getattr(item, funcname)()
+    sc.update_requested = None
+    sc.updated = datetime.datetime.now()
+    sc.save()
+    sc, __ = StatsCache.objects.get_or_create(
+        model=model_name, model_pk=model_pk
+    )
+
+
 class DashboardFormItem(object):
     """
     Provide methods to manage statistics
@@ -2776,12 +2827,15 @@ class DashboardFormItem(object):
 
     def _get_or_set_stats(self, funcname, update,
                           timeout=settings.CACHE_TIMEOUT):
-        key, val = get_cache(self.__class__, [funcname, self.pk])
-        if not update and val is not None:
-            return val
-        val = getattr(self, funcname)()
-        cache.set(key, val, timeout)
-        return val
+        model_name = self._meta.app_label + "." + self._meta.model_name
+        sc, __ = StatsCache.objects.get_or_create(
+            model=model_name, model_pk=self.pk
+        )
+        now = datetime.datetime.now()
+        if sc.values and (
+                sc.updated + datetime.timedelta(seconds=timeout)) > now:
+            return sc.values
+        return update_stats(sc, self, funcname)
 
     @classmethod
     def get_periods(cls, slice='month', fltr={}, date_source='creation'):
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 55301af14..3d9cfc13f 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -324,16 +324,20 @@ def force_cached_label_changed(sender, **kwargs):
     cached_label_changed(sender, **kwargs)
 
 
-def serialize_args_for_tasks(sender, **kwargs):
+def serialize_args_for_tasks(sender, instance, kwargs, extra_kwargs=None):
     if 'signal' in kwargs:
         kwargs.pop('signal')
     if 'instance' in kwargs:
         kwargs['instance'] = kwargs["instance"].pk
     sender = (sender._meta.app_label, sender._meta.object_name)
+    if extra_kwargs:
+        for kw in extra_kwargs:
+            if getattr(instance, kw, None):
+                kwargs[kw] = getattr(instance, kw)
     return sender, kwargs
 
 
-def deserialize_args_for_tasks(sender, **kwargs):
+def deserialize_args_for_tasks(sender, kwargs, extra_kwargs=None):
     if "instance" not in kwargs:
         return sender, None
     if not isinstance(sender, (tuple, list)):  # not task
@@ -343,9 +347,19 @@ def deserialize_args_for_tasks(sender, **kwargs):
         instance = sender.objects.get(pk=kwargs['instance'])
     except sender.DoesNotExist:
         return sender, None  # object is not here anymore
+    if extra_kwargs:
+        for kw in extra_kwargs:
+            if kw in kwargs:
+                setattr(instance, kw, kwargs[kw])
     return sender, instance
 
 
+EXTRA_KWARGS_TRIGGER = [
+    "_cascade_change", "_cached_labels_bulk_update", "skip_history_when_saving",
+    "_post_saved_geo", "_search_updated", "_cached_label_checked"
+]
+
+
 def cached_label_changed(sender, **kwargs):
     if not kwargs.get('instance'):
         return
@@ -361,13 +375,18 @@ def cached_label_changed(sender, **kwargs):
     if not settings.USE_BACKGROUND_TASK:
         return _cached_label_changed(sender, **kwargs)
 
-    sender, kwargs = serialize_args_for_tasks(sender, **kwargs)
+    if getattr(instance, "_cascade_change", False):
+        kwargs["cascade_change"] = True
+
+    sender, kwargs = serialize_args_for_tasks(
+        sender, instance, kwargs, EXTRA_KWARGS_TRIGGER)
     return _cached_label_changed.delay(sender, **kwargs)
 
 
 @task()
 def _cached_label_changed(sender, **kwargs):
-    sender, instance = deserialize_args_for_tasks(sender, **kwargs)
+    sender, instance = deserialize_args_for_tasks(sender, kwargs,
+                                                  EXTRA_KWARGS_TRIGGER)
     if not instance:
         return
 
@@ -376,7 +395,6 @@ def _cached_label_changed(sender, **kwargs):
     if hasattr(instance, "refresh_cache"):
         instance.refresh_cache()
 
-    # print(sender, "cached_label_changed")
     instance._cached_label_checked = True
     cached_labels = ['cached_label']
     if hasattr(sender, 'CACHED_LABELS'):
@@ -558,7 +576,8 @@ def post_save_geo(sender, **kwargs):
 
     if not settings.USE_BACKGROUND_TASK:
         return _post_save_geo(sender, **kwargs)
-    sender, kwargs = serialize_args_for_tasks(sender, **kwargs)
+    sender, kwargs = serialize_args_for_tasks(sender, kwargs.get('instance'),
+                                              kwargs, EXTRA_KWARGS_TRIGGER)
     return _post_save_geo.delay(sender, **kwargs)
 
 
@@ -573,7 +592,8 @@ def _post_save_geo(sender, **kwargs):
     profile = get_current_profile()
     if not profile.mapping:
         return
-    sender, instance = deserialize_args_for_tasks(sender, **kwargs)
+    sender, instance = deserialize_args_for_tasks(sender, kwargs,
+                                                  EXTRA_KWARGS_TRIGGER)
     if not instance:
         return