Celery: manage cached_label update with tasks

7 files changed, 84 insertions, 13 deletions

diff --git a/.gitignore b/.gitignore
index 04e2482f5..68f835be4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,7 +16,7 @@ local_settings.py
 fabfile.py
 example_project/media
 example_project/static
-example_project/celery.py
+example_project/celery_app.py
 example_project/__init__.py
 docs/*/build
 Makefile
diff --git a/Makefile.example b/Makefile.example
index fc6af35e2..3ad0fb71c 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -19,6 +19,8 @@ help:
 	#	* pep8: 
 	#	* pylint: 
 	#	* syncdb: 
+	#	* run: 
+	#	* runcelery: 
 	#	* shell: 
 	#	* makemessages: 
 	#	* compilemessages: 
@@ -82,6 +84,9 @@ shell:
 run:
 	cd $(project); $(PYTHON) manage.py runserver 0.0.0.0:8000
 
+runcelery:
+	celery -A example_project worker -l info
+
 makemessages:
 	for DIR in $(apps); do \
 		cd $(CURDIR)/$$DIR; \
diff --git a/example_project/__init__.py.celery.sample b/example_project/__init__.py.celery.sample
index ebae6a3c3..5dad2ced8 100644
--- a/example_project/__init__.py.celery.sample
+++ b/example_project/__init__.py.celery.sample
@@ -1,7 +1,7 @@
 # force the retranslation of generated strings and external module
 from django.utils.translation import ugettext_lazy as _
 
-from .celery import app as celery_app
+from .celery_app import app as celery_app
 
 __all__ = ('celery_app',)
 
diff --git a/example_project/celery.py.sample b/example_project/celery_app.py.sample
index 3aed98038..3aed98038 100644
--- a/example_project/celery.py.sample
+++ b/example_project/celery_app.py.sample
diff --git a/example_project/settings.py b/example_project/settings.py
index 09590f807..bf2b26c31 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -273,9 +273,12 @@ DOT_BINARY = ""
 TEST_RUNNER = 'ishtar_common.tests.ManagedModelTestRunner'
 
 try:
-    from .local_settings import *
+    from local_settings import *
 except ImportError as e:
-    print('Unable to load local_settings.py:', e)
+    try:
+        from .local_settings import *
+    except ImportError as e:
+        print('Unable to load local_settings.py:', e)
 
 if LANGUAGE_CODE == "fr-fr" and SRID == 4326:
     SRID = 27572  # Lambert zone II - France
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 904f3f013..20c3bf9c4 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -47,11 +47,7 @@ from ishtar_common.data_importer import Importer, ImportFormater, \
     IntegerFormater, FloatFormater, UnicodeFormater, DateFormater, \
     TypeFormater, YearFormater, StrToBoolean, FileFormater, InseeFormater, \
     ImporterError
-
-if settings.USE_BACKGROUND_TASK:
-    from background_task import background
-else:
-    background = lambda x: x
+from ishtar_common.utils import task
 
 logger = logging.getLogger(__name__)
 
@@ -859,7 +855,7 @@ delayed_check = None
 
 
 if settings.USE_BACKGROUND_TASK:
-    @background(schedule=1)
+    @task()
     def delayed_import(import_pk, session_key):
         try:
             imp = Import.objects.get(pk=import_pk)
@@ -867,7 +863,7 @@ if settings.USE_BACKGROUND_TASK:
             pass
         imp.importation(session_key=session_key)
 
-    @background(schedule=1)
+    @task()
     def delayed_check(import_pk, session_key):
         try:
             imp = Import.objects.get(pk=import_pk)
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index d0bfee02a..55301af14 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -62,10 +62,22 @@ def dict_to_tuple(dct):
 
 
 def debug_line_no():
-    cf = currentframe()
     return currentframe().f_back.f_lineno
 
 
+def fake_task(*args):
+    def fake(func):
+        return func
+    return fake
+
+
+if settings.USE_BACKGROUND_TASK:
+    from celery import shared_task
+    task = shared_task
+else:
+    task = fake_task
+
+
 class BColors:
     """
     Bash colors. Don't forget to finish your colored string with ENDC.
@@ -312,6 +324,28 @@ def force_cached_label_changed(sender, **kwargs):
     cached_label_changed(sender, **kwargs)
 
 
+def serialize_args_for_tasks(sender, **kwargs):
+    if 'signal' in kwargs:
+        kwargs.pop('signal')
+    if 'instance' in kwargs:
+        kwargs['instance'] = kwargs["instance"].pk
+    sender = (sender._meta.app_label, sender._meta.object_name)
+    return sender, kwargs
+
+
+def deserialize_args_for_tasks(sender, **kwargs):
+    if "instance" not in kwargs:
+        return sender, None
+    if not isinstance(sender, (tuple, list)):  # not task
+        return sender, kwargs["instance"]
+    sender = apps.get_model(*sender)
+    try:
+        instance = sender.objects.get(pk=kwargs['instance'])
+    except sender.DoesNotExist:
+        return sender, None  # object is not here anymore
+    return sender, instance
+
+
 def cached_label_changed(sender, **kwargs):
     if not kwargs.get('instance'):
         return
@@ -324,9 +358,25 @@ def cached_label_changed(sender, **kwargs):
     if not force_update and hasattr(instance, '_cached_label_checked') \
             and instance._cached_label_checked:
         return
+    if not settings.USE_BACKGROUND_TASK:
+        return _cached_label_changed(sender, **kwargs)
+
+    sender, kwargs = serialize_args_for_tasks(sender, **kwargs)
+    return _cached_label_changed.delay(sender, **kwargs)
+
+
+@task()
+def _cached_label_changed(sender, **kwargs):
+    sender, instance = deserialize_args_for_tasks(sender, **kwargs)
+    if not instance:
+        return
+
+    force_update = kwargs.get('force_update', False)
+
     if hasattr(instance, "refresh_cache"):
         instance.refresh_cache()
 
+    # print(sender, "cached_label_changed")
     instance._cached_label_checked = True
     cached_labels = ['cached_label']
     if hasattr(sender, 'CACHED_LABELS'):
@@ -505,19 +555,36 @@ def post_save_geo(sender, **kwargs):
     if not kwargs.get('instance') or getattr(kwargs['instance'],
                                              '_post_saved_geo', False):
         return
+
+    if not settings.USE_BACKGROUND_TASK:
+        return _post_save_geo(sender, **kwargs)
+    sender, kwargs = serialize_args_for_tasks(sender, **kwargs)
+    return _post_save_geo.delay(sender, **kwargs)
+
+
+@task()
+def _post_save_geo(sender, **kwargs):
+    """
+    Convert raw x, y, z point to real geo field
+    """
     from ishtar_common.models import get_current_profile  # not clean but utils
     # is loaded before models
 
     profile = get_current_profile()
     if not profile.mapping:
         return
-    instance = kwargs.get('instance')
+    sender, instance = deserialize_args_for_tasks(sender, **kwargs)
+    if not instance:
+        return
+
     kls_name = instance.__class__.__name__
 
     if not profile.locate_warehouses and ("Container" in kls_name
                                           or "Warehouse" in kls_name):
         return
 
+    # print(sender, "post_save_geo")
+
     current_source = "default"
     if hasattr(instance.__class__, "_meta"):
         current_source = str(instance.__class__._meta.verbose_name)