Add import task using celery

8 files changed, 279 insertions, 7 deletions

diff --git a/chimere/admin.py b/chimere/admin.py
index 4e632b8..889f4c2 100644
--- a/chimere/admin.py
+++ b/chimere/admin.py
@@ -23,12 +23,14 @@ Settings for administration pages
 
 from django.conf import settings
 from django.contrib import admin
+from django.utils.translation import ugettext_lazy as _
 
+from chimere import tasks
+from chimere.forms import MarkerAdminForm, RouteAdminForm, AreaAdminForm,\
+                          NewsAdminForm, CategoryAdminForm
 from chimere.models import Category, Icon, SubCategory, Marker, \
            PropertyModel, News, Route, Area, ColorTheme, Color, RouteFile,\
            MultimediaType, MultimediaFile, PictureFile, Importer
-from chimere.forms import MarkerAdminForm, RouteAdminForm, AreaAdminForm,\
-                          NewsAdminForm, CategoryAdminForm
 from chimere.widgets import TextareaWidget
 
 def get_areas_for_user(user):
@@ -106,9 +108,23 @@ class SubCategoryAdmin(admin.ModelAdmin):
     list_display = ('name', 'category', 'available')
     list_filter = ('category',)
 
+def importing(modeladmin, request, queryset):
+    for importer in queryset:
+        importer.state = unicode(tasks.IMPORT_MESSAGES['import_pending'][0])
+        importer.save()
+        tasks.importing(importer.pk)
+importing.short_description = _(u"Import")
+
+def cancel_import(modeladmin, request, queryset):
+    for importer in queryset:
+        importer.state = tasks.IMPORT_MESSAGES['import_cancel'][0]
+        importer.save()
+cancel_import.short_description = _(u"Cancel import")
+
 class ImporterAdmin(admin.ModelAdmin):
-    list_display = ('importer_type', 'subcategory', 'source_url', 'filtr')
-    list_filter = ('importer_type',)
+    list_display = ('importer_type', 'source_url', 'state', 'filtr')
+    list_filter = ('importer_type', 'source_url')
+    actions = [importing, cancel_import]
 
 class NewsAdmin(admin.ModelAdmin):
     """
diff --git a/chimere/migrations/0009_auto__add_field_importer_state.py b/chimere/migrations/0009_auto__add_field_importer_state.py
new file mode 100644
index 0000000..42d1048
--- /dev/null
+++ b/chimere/migrations/0009_auto__add_field_importer_state.py
@@ -0,0 +1,187 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Importer.state'
+        db.add_column('chimere_importer', 'state', self.gf('django.db.models.fields.CharField')(max_length=200, null=True, blank=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Importer.state'
+        db.delete_column('chimere_importer', 'state')
+
+
+    models = {
+        'chimere.area': {
+            'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'urn': ('django.db.models.fields.SlugField', [], {'db_index': 'True', 'unique': 'True', 'max_length': '50', 'blank': 'True'})
+        },
+        'chimere.category': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Category'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.color': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Color'},
+            'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.colortheme': {
+            'Meta': {'object_name': 'ColorTheme'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.icon': {
+            'Meta': {'object_name': 'Icon'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.importer': {
+            'Meta': {'object_name': 'Importer'},
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}),
+            'source_url': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.marker': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'},
+            'available_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'multimedia_files': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'marker'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['chimere.MultimediaFile']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'pictures': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'marker'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['chimere.PictureFile']"}),
+            'point': ('chimere.widgets.PointField', [], {}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}),
+            'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.multimediafile': {
+            'Meta': {'object_name': 'MultimediaFile'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        'chimere.multimediatype': {
+            'Meta': {'object_name': 'MultimediaType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.news': {
+            'Meta': {'object_name': 'News'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'content': ('django.db.models.fields.TextField', [], {}),
+            'date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.picturefile': {
+            'Meta': {'object_name': 'PictureFile'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.property': {
+            'Meta': {'object_name': 'Property'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}),
+            'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}),
+            'value': ('django.db.models.fields.TextField', [], {})
+        },
+        'chimere.propertymodel': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        'chimere.route': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'},
+            'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}),
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'route': ('chimere.widgets.RouteField', [], {}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'chimere.routefile': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'},
+            'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.subcategory': {
+            'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'},
+            'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'to': "orm['chimere.Area']"}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Category']"}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}),
+            'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.tinyurl': {
+            'Meta': {'object_name': 'TinyUrl'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        }
+    }
+
+    complete_apps = ['chimere']
diff --git a/chimere/models.py b/chimere/models.py
index 21fdd4e..22a51f9 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -209,6 +209,8 @@ class Importer(models.Model):
                              blank=True, null=True)
     categories = SelectMultipleField(SubCategory,
                       verbose_name=_(u"Associated subcategories"))
+    state = models.CharField(_(u"State"), max_length=200,
+                             blank=True, null=True)
 
     class Meta:
         verbose_name = _(u"Importer")
diff --git a/chimere/tasks.py b/chimere/tasks.py
new file mode 100644
index 0000000..ba88304
--- /dev/null
+++ b/chimere/tasks.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+from celery.decorators import task
+
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.translation import ugettext_lazy as _
+
+from chimere.models import Importer
+
+IMPORT_MESSAGES = {'import_pending':[_(u"Import pending")],
+                   'import_process':[_(u"Import processing")],
+                   'import_done':[_(u"Import successfuly done"),
+                    _(u" %(new)d new item(s), %(updated)d updated item(s)")],
+                   'import_failed':[_(u"Import failed"), "%s"],
+                   'import_cancel':[_(u"Import canceled")]
+                   }
+
+@task()
+def importing(importer_pk):
+    try:
+        importer = Importer.objects.get(pk=importer_pk)
+    except ObjectDoesNotExist:
+        # importer doesn't exists anymore: too late!
+        return
+    if importer.state != IMPORT_MESSAGES['import_pending'][0]:
+        # import canceled or done
+        return
+    importer.state = unicode(IMPORT_MESSAGES['import_process'][0])
+    importer.save()
+    new_item, updated_item, error = importer.manager.get()
+    if error:
+        importer.state = unicode(IMPORT_MESSAGES['import_failed'][0]) \
+              + u" - " + unicode(IMPORT_MESSAGES['import_failed'][1]) % error
+        importer.save()
+        return
+    importer.state = unicode(IMPORT_MESSAGES['import_done'][0]) + u" - " \
+         + unicode(IMPORT_MESSAGES['import_done'][1]) % {'new':new_item,
+                                                         'updated':updated_item}
+    importer.save()
+    return True
diff --git a/chimere/utils.py b/chimere/utils.py
index 4e78220..d631fe7 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -57,7 +57,7 @@ class KMLManager(ImportManager):
         Return a tuple with:
         - number of new item ;
         - number of item updated ;
-        - detail
+        - error detail on error
         """
         from models import Marker
         new_item, updated_item, msg = 0, 0, ''
diff --git a/debian/control b/debian/control
index ab3dd24..9fe58d9 100644
--- a/debian/control
+++ b/debian/control
@@ -5,5 +5,5 @@ Depends: python-django (>=1.3), python-gdal, python-psycopg2,
          libjs-jquery-ui, libjs-jquery-ui-theme-base,
          postgresql-9.1, postgresql-9.1-postgis, gettext,
          python-simplejson
-Recommends: tinymce, gpsbabel
+Recommends: tinymce, gpsbabel, python-django-celery, rabbitmq-server
 Suggests: libjs-jquery-ui-theme-south-street
diff --git a/example_project/settings.py.example b/example_project/settings.py.example
index bbe100b..9b3235b 100644
--- a/example_project/settings.py.example
+++ b/example_project/settings.py.example
@@ -88,6 +88,15 @@ DATABASES = {
     },
 }
 
+# celery
+import djcelery
+djcelery.setup_loader()
+BROKER_HOST = "localhost"
+BROKER_PORT = 5672
+BROKER_USER = "guest"
+BROKER_PASSWORD = "guest"
+BROKER_VHOST = "/"
+
 # Local time zone for this installation. Choices can be found here:
 # http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE
 # although not all variations may be possible on all operating systems.
@@ -177,6 +186,7 @@ INSTALLED_APPS = (
     'django.contrib.sites',
     'django.contrib.gis',
     'django.contrib.staticfiles',
+    'djcelery',
     'south',
     'chimere',
     # activate it if you want to use migration scripts
diff --git a/requirements.txt b/requirements.txt
index 02a8290..8ca59a9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ django==1.3
 beautifulsoup
 psycopg2
 pil
-#GDAL
 lxml
 south==0.7.3
 simplejson
+django-celery==2.4.2