Cache: manage long keys - immediatly refresh cache after types modifications

8 files changed, 127 insertions, 18 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index 0123dd2ed..af042bf45 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -29,7 +29,7 @@ from ishtar_common.utils import cached_label_changed
 from ishtar_common.models import GeneralType, BaseHistorizedItem, \
     HistoricalRecords, OwnPerms, ShortMenuItem, Source, GeneralRelationType,\
     GeneralRecordRelations, post_delete_record_relation, get_external_id, \
-    ImageModel
+    ImageModel, post_save_cache
 from archaeological_operations.models import Operation, Period, Parcel
 
 
@@ -38,6 +38,8 @@ class DatingType(GeneralType):
         verbose_name = _(u"Dating type")
         verbose_name_plural = _(u"Dating types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=DatingType)
+post_delete.connect(post_save_cache, sender=DatingType)
 
 
 class DatingQuality(GeneralType):
@@ -45,6 +47,8 @@ class DatingQuality(GeneralType):
         verbose_name = _(u"Dating quality")
         verbose_name_plural = _(u"Dating qualities")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=DatingQuality)
+post_delete.connect(post_save_cache, sender=DatingQuality)
 
 
 class Dating(models.Model):
@@ -82,6 +86,8 @@ class Unit(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=Unit)
+post_delete.connect(post_save_cache, sender=Unit)
 
 
 class ActivityType(GeneralType):
@@ -94,6 +100,8 @@ class ActivityType(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=ActivityType)
+post_delete.connect(post_save_cache, sender=ActivityType)
 
 
 class IdentificationType(GeneralType):
@@ -106,6 +114,8 @@ class IdentificationType(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=IdentificationType)
+post_delete.connect(post_save_cache, sender=IdentificationType)
 
 
 class ContextRecord(BaseHistorizedItem, ImageModel, OwnPerms, ShortMenuItem):
diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index 42520769a..b116d1f1c 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -23,7 +23,7 @@ from django.conf import settings
 from django.contrib.gis.db import models
 from django.core.cache import cache
 from django.db.models import Q, Count, Sum
-from django.db.models.signals import post_save, m2m_changed
+from django.db.models.signals import post_save, m2m_changed, post_delete
 from django.utils.translation import ugettext_lazy as _, ugettext
 
 from ishtar_common.utils import cached_label_changed, get_cache
@@ -31,7 +31,7 @@ from ishtar_common.utils import cached_label_changed, get_cache
 from ishtar_common.models import GeneralType, BaseHistorizedItem, \
     HistoricalRecords, OwnPerms, Person, Organization, Department, Town, \
     Dashboard, DashboardFormItem, ValueGetter, ShortMenuItem, \
-    OperationType, get_external_id
+    OperationType, get_external_id, post_save_cache
 
 from archaeological_operations.models import get_values_town_related, \
     ClosedItem
@@ -51,6 +51,8 @@ class FileType(GeneralType):
             return file_type_id == preventive
         except FileType.DoesNotExist:
             return False
+post_save.connect(post_save_cache, sender=FileType)
+post_delete.connect(post_save_cache, sender=FileType)
 
 
 class PermitType(GeneralType):
@@ -58,6 +60,8 @@ class PermitType(GeneralType):
         verbose_name = _(u"Permit type")
         verbose_name_plural = _(u"Permit types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=PermitType)
+post_delete.connect(post_save_cache, sender=PermitType)
 
 if settings.COUNTRY == 'fr':
     class SaisineType(GeneralType, ValueGetter):
@@ -67,6 +71,8 @@ if settings.COUNTRY == 'fr':
             verbose_name = u"Type Saisine"
             verbose_name_plural = u"Types Saisine"
             ordering = ('label',)
+    post_save.connect(post_save_cache, sender=SaisineType)
+    post_delete.connect(post_save_cache, sender=SaisineType)
 
 
 class File(ClosedItem, BaseHistorizedItem, OwnPerms, ValueGetter,
diff --git a/archaeological_finds/models.py b/archaeological_finds/models.py
index be9817287..722d165ce 100644
--- a/archaeological_finds/models.py
+++ b/archaeological_finds/models.py
@@ -23,14 +23,14 @@ from django.conf import settings
 from django.contrib.gis.db import models
 from django.core.urlresolvers import reverse
 from django.db.models import Max, Q
-from django.db.models.signals import m2m_changed, post_save
+from django.db.models.signals import m2m_changed, post_save, post_delete
 from django.utils.translation import ugettext_lazy as _, ugettext
 
 from ishtar_common.utils import cached_label_changed
 
 from ishtar_common.models import GeneralType, ImageModel, BaseHistorizedItem, \
     ShortMenuItem, LightHistorizedItem, HistoricalRecords, OwnPerms, Source, \
-    Person, Basket, get_external_id
+    Person, Basket, get_external_id, post_save_cache
 
 from archaeological_operations.models import AdministrativeAct
 from archaeological_context_records.models import ContextRecord, Dating
@@ -50,6 +50,8 @@ class MaterialType(GeneralType):
         verbose_name = _(u"Material type")
         verbose_name_plural = _(u"Material types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=MaterialType)
+post_delete.connect(post_save_cache, sender=MaterialType)
 
 
 class ConservatoryState(GeneralType):
@@ -60,6 +62,8 @@ class ConservatoryState(GeneralType):
         verbose_name = _(u"Conservatory state")
         verbose_name_plural = _(u"Conservatory states")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=ConservatoryState)
+post_delete.connect(post_save_cache, sender=ConservatoryState)
 
 
 class PreservationType(GeneralType):
@@ -67,6 +71,8 @@ class PreservationType(GeneralType):
         verbose_name = _(u"Preservation type")
         verbose_name_plural = _(u"Preservation types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=PreservationType)
+post_delete.connect(post_save_cache, sender=PreservationType)
 
 
 class IntegrityType(GeneralType):
@@ -74,6 +80,8 @@ class IntegrityType(GeneralType):
         verbose_name = _(u"Integrity / interest type")
         verbose_name_plural = _(u"Integrity / interest types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=IntegrityType)
+post_delete.connect(post_save_cache, sender=IntegrityType)
 
 
 class RemarkabilityType(GeneralType):
@@ -81,6 +89,8 @@ class RemarkabilityType(GeneralType):
         verbose_name = _(u"Remarkability type")
         verbose_name_plural = _(u"Remarkability types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=RemarkabilityType)
+post_delete.connect(post_save_cache, sender=RemarkabilityType)
 
 
 class ObjectType(GeneralType):
@@ -102,6 +112,8 @@ class ObjectType(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=ObjectType)
+post_delete.connect(post_save_cache, sender=ObjectType)
 
 IS_ISOLATED_CHOICES = (
     ('U', _(u"Unknow")),
@@ -693,6 +705,8 @@ class TreatmentType(GeneralType):
         verbose_name = _(u"Treatment type")
         verbose_name_plural = _(u"Treatment types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=TreatmentType)
+post_delete.connect(post_save_cache, sender=TreatmentType)
 
 
 class Treatment(BaseHistorizedItem, OwnPerms):
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index 042cfce2d..926e27b3b 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -36,7 +36,7 @@ from ishtar_common.models import GeneralType, BaseHistorizedItem, \
     SourceType, Person, Organization, Town, Dashboard, IshtarUser, ValueGetter,\
     DocumentTemplate, ShortMenuItem, DashboardFormItem, GeneralRelationType,\
     GeneralRecordRelations, post_delete_record_relation, OperationType, \
-    get_external_id, ImageModel
+    get_external_id, ImageModel, post_save_cache
 
 
 class RemainType(GeneralType):
@@ -44,6 +44,8 @@ class RemainType(GeneralType):
         verbose_name = _(u"Remain type")
         verbose_name_plural = _(u"Remain types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=RemainType)
+post_delete.connect(post_save_cache, sender=RemainType)
 
 
 class Period(GeneralType):
@@ -60,6 +62,8 @@ class Period(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=Period)
+post_delete.connect(post_save_cache, sender=Period)
 
 
 class ReportState(GeneralType):
@@ -72,6 +76,8 @@ class ReportState(GeneralType):
 
     def __unicode__(self):
         return self.label
+post_save.connect(post_save_cache, sender=ReportState)
+post_delete.connect(post_save_cache, sender=ReportState)
 
 
 class ArchaeologicalSite(BaseHistorizedItem):
@@ -792,6 +798,8 @@ class ActType(GeneralType):
         verbose_name = _(u"Act type")
         verbose_name_plural = _(u"Act types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=ActType)
+post_delete.connect(post_save_cache, sender=ActType)
 
 
 class AdministrativeAct(BaseHistorizedItem, OwnPerms, ValueGetter):
diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py
index 3591f554a..17495bf72 100644
--- a/archaeological_warehouse/models.py
+++ b/archaeological_warehouse/models.py
@@ -17,11 +17,12 @@
 
 # See the file COPYING for details.
 
+from django.db.models.signals import post_save, post_delete
 from django.contrib.gis.db import models
 from django.utils.translation import ugettext_lazy as _, ugettext
 
 from ishtar_common.models import GeneralType, \
-    LightHistorizedItem, OwnPerms, Address, Person
+    LightHistorizedItem, OwnPerms, Address, Person, post_save_cache
 
 
 class WarehouseType(GeneralType):
@@ -29,6 +30,8 @@ class WarehouseType(GeneralType):
         verbose_name = _(u"Warehouse type")
         verbose_name_plural = _(u"Warehouse types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=WarehouseType)
+post_delete.connect(post_save_cache, sender=WarehouseType)
 
 
 class Warehouse(Address, OwnPerms):
@@ -66,6 +69,8 @@ class ContainerType(GeneralType):
         verbose_name = _(u"Container type")
         verbose_name_plural = _(u"Container types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=ContainerType)
+post_delete.connect(post_save_cache, sender=ContainerType)
 
 
 class Container(LightHistorizedItem):
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 932beaf98..95d995a91 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -260,8 +260,33 @@ class Cached(object):
     slug_field = 'txt_idx'
 
     @classmethod
+    def refresh_cache(cls):
+        cache_ckey, current_keys = get_cache(cls, ['_current_keys'])
+        if not current_keys:
+            return
+        for keys in current_keys:
+            if len(keys) == 2 and keys[0] == '__slug':
+                cls.get_cache(keys[1], force=True)
+            if len(keys) >= 2 and keys[0] == '__get_types':
+                default = keys.pop()
+                empty_first = bool(keys.pop())
+                exclude = keys[1:]
+                cls.get_types(
+                    exclude=exclude, empty_first=empty_first, default=default,
+                    force=True)
+
+    @classmethod
+    def _add_cache_key_to_refresh(cls, keys):
+        cache_ckey, current_keys = get_cache(cls, ['_current_keys'])
+        if type(current_keys) != list:
+            current_keys = []
+        if keys not in current_keys:
+            current_keys.append(keys)
+            cache.set(cache_ckey, current_keys, settings.CACHE_TIMEOUT)
+
+    @classmethod
     def get_cache(cls, slug, force=False):
-        cache_key, value = get_cache(cls, slug)
+        cache_key, value = get_cache(cls, ['__slug', slug])
         if not force and value:
             return value
         try:
@@ -270,12 +295,12 @@ class Cached(object):
             cache.set(cache_key, obj, settings.CACHE_TIMEOUT)
             return obj
         except cls.DoesNotExist:
+            cache.set(cache_key, None, settings.CACHE_TIMEOUT)
             return None
 
-    def save(self, *args, **kwargs):
-        ret = super(Cached, self).save(*args, **kwargs)
-        self.get_cache(getattr(self, self.slug_field), force=True)
-        return ret
+
+def post_save_cache(sender, **kwargs):
+    sender.refresh_cache()
 
 
 class GeneralType(Cached, models.Model):
@@ -351,9 +376,9 @@ class GeneralType(Cached, models.Model):
 
     @classmethod
     def get_types(cls, dct={}, instances=False, exclude=[], empty_first=True,
-                  default=None, initial=None):
+                  default=None, initial=None, force=False):
         types = cls._pre_get_types(dct, instances, exclude, empty_first,
-                                   default)
+                                   default, force)
         if not initial:
             return types
         new_vals = cls._get_initial_types(initial, [idx for idx, lbl in types])
@@ -362,14 +387,15 @@ class GeneralType(Cached, models.Model):
 
     @classmethod
     def _pre_get_types(cls, dct={}, instances=False, exclude=[],
-                       empty_first=True, default=None):
+                       empty_first=True, default=None, force=False):
         # cache
         cache_key = None
         if not instances:
-            keys = [u"{}".format(ex) for ex in exclude] + [
+            keys = ['__get_types']
+            keys += [u"{}".format(ex) for ex in exclude] + [
                 empty_first and 'empty_first' or ''] + [u"{}".format(default)]
             cache_key, value = get_cache(cls, keys)
-            if value:
+            if value and not force:
                 return value
         base_dct = dct.copy()
         if hasattr(cls, 'parent'):
@@ -405,7 +431,7 @@ class GeneralType(Cached, models.Model):
             except cls.DoesNotExist:
                 pass
         items = cls.objects.filter(**dct)
-        if default:
+        if default and default != "None":
             exclude.append(default.txt_idx)
         if exclude:
             items = items.exclude(txt_idx__in=exclude)
@@ -1499,6 +1525,8 @@ class OrganizationType(GeneralType):
         verbose_name = _(u"Organization type")
         verbose_name_plural = _(u"Organization types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=OrganizationType)
+post_delete.connect(post_save_cache, sender=OrganizationType)
 
 IMPORTER_CLASSES = {}
 
@@ -2279,6 +2307,8 @@ class PersonType(GeneralType):
         verbose_name = _(u"Person type")
         verbose_name_plural = _(u"Person types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=PersonType)
+post_delete.connect(post_save_cache, sender=PersonType)
 
 
 class TitleType(GeneralType):
@@ -2286,6 +2316,8 @@ class TitleType(GeneralType):
         verbose_name = _(u"Title type")
         verbose_name_plural = _(u"Title types")
         ordering = ('label',)
+post_save.connect(post_save_cache, sender=TitleType)
+post_delete.connect(post_save_cache, sender=TitleType)
 
 
 class Person(Address, Merge, OwnPerms, ValueGetter):
@@ -2547,6 +2579,8 @@ class AuthorType(GeneralType):
     class Meta:
         verbose_name = _(u"Author type")
         verbose_name_plural = _(u"Author types")
+post_save.connect(post_save_cache, sender=AuthorType)
+post_delete.connect(post_save_cache, sender=AuthorType)
 
 
 class Author(models.Model):
@@ -2577,18 +2611,24 @@ class SourceType(GeneralType):
     class Meta:
         verbose_name = _(u"Source type")
         verbose_name_plural = _(u"Source types")
+post_save.connect(post_save_cache, sender=SourceType)
+post_delete.connect(post_save_cache, sender=SourceType)
 
 
 class SupportType(GeneralType):
     class Meta:
         verbose_name = _(u"Support type")
         verbose_name_plural = _(u"Support types")
+post_save.connect(post_save_cache, sender=SupportType)
+post_delete.connect(post_save_cache, sender=SupportType)
 
 
 class Format(GeneralType):
     class Meta:
         verbose_name = _(u"Format")
         verbose_name_plural = _(u"Formats")
+post_save.connect(post_save_cache, sender=Format)
+post_delete.connect(post_save_cache, sender=Format)
 
 
 class Source(ImageModel, models.Model):
@@ -2737,3 +2777,5 @@ class OperationType(GeneralType):
         if not key:
             return op_type.preventive
         return key == op_type.txt_idx
+post_save.connect(post_save_cache, sender=OperationType)
+post_delete.connect(post_save_cache, sender=OperationType)
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 8c918b5a9..0c4bbda08 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -143,6 +143,7 @@ class WizardTest(object):
 
 class CacheTest(TestCase):
     def testAdd(self):
+        models.OrganizationType.refresh_cache()
         cached = models.OrganizationType.get_cache('test')
         self.assertEqual(cached, None)
         orga = models.OrganizationType.objects.create(
@@ -154,6 +155,21 @@ class CacheTest(TestCase):
         cached = models.OrganizationType.get_cache('testy')
         self.assertEqual(cached.pk, orga.pk)
 
+    def testList(self):
+        models.OrganizationType.refresh_cache()
+        types = models.OrganizationType.get_types()
+        # only empty
+        self.assertTrue(len(types), 1)
+        org = models.OrganizationType.objects.create(
+            txt_idx='test', label='testy')
+        types = [
+            unicode(lbl) for idx, lbl in models.OrganizationType.get_types()]
+        self.assertTrue('testy' in types)
+        org.delete()
+        types = [
+            unicode(lbl) for idx, lbl in models.OrganizationType.get_types()]
+        self.assertFalse('testy' in types)
+
 
 class MergeTest(TestCase):
     def setUp(self):
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index ca9193204..60c3ac7ef 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -17,6 +17,7 @@
 
 # See the file COPYING for details.
 
+import hashlib
 import random
 
 from django.conf import settings
@@ -41,6 +42,13 @@ def get_cache(cls, extra_args=[]):
             else:
                 cache_key += '-' + unicode(arg)
     cache_key = slugify(cache_key)
+    if not cache_key.endswith('_current_keys') \
+            and hasattr(cls, '_add_cache_key_to_refresh'):
+        cls._add_cache_key_to_refresh(extra_args)
+    if len(cache_key) >= 250:
+        m = hashlib.md5()
+        m.update(cache_key)
+        cache_key = m.hexdigest()
     return cache_key, cache.get(cache_key)