Import: import CSV files

2 files changed, 135 insertions, 41 deletions

diff --git a/chimere/models.py b/chimere/models.py
index 6661a30..e6d1636 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -42,7 +42,7 @@ from chimere.widgets import PointField, RouteField, SelectMultipleField, \
                             TextareaWidget, DatePickerWidget
 from chimere.managers import BaseGeoManager
 from chimere.utils import KMLManager, OSMManager, ShapefileManager, \
-                          GeoRSSManager
+                          GeoRSSManager, CSVManager
 
 class News(models.Model):
     """News of the site
@@ -206,13 +206,15 @@ class SubCategory(models.Model):
 IMPORTERS = {'KML':KMLManager,
              'OSM':OSMManager,
              'SHP':ShapefileManager,
-             'RSS':GeoRSSManager
+             'RSS':GeoRSSManager,
+             'CSV':CSVManager
              }
 
 IMPORTER_CHOICES = (('KML', 'KML'),
                     ('OSM', 'OSM'),
                     ('SHP', 'Shapefile'),
-                    ('RSS', 'GeoRSS')
+                    ('RSS', 'GeoRSS'),
+                    ('CSV', 'CSV')
                     )
 
 IMPORTER_CHOICES_DICT = dict(IMPORTER_CHOICES)
@@ -226,22 +228,22 @@ class Importer(models.Model):
     # URL of a KML file or a XAPI service for OSM
     source = models.CharField(_(u"Source"), max_length=200,
                               blank=True, null=True)
+    source_file = models.FileField(_(u"Source file"),
+                        upload_to='import_files', blank=True, null=True)
     filtr = models.CharField(_(u"Filter"), max_length=200,
                              blank=True, null=True)
     default_name = models.CharField(_(u"Name by default"), max_length=200,
                                     blank=True, null=True)
-    categories = SelectMultipleField(SubCategory,
-                      verbose_name=_(u"Associated subcategories"))
-    state = models.CharField(_(u"State"), max_length=200,
-                             blank=True, null=True)
     srid = models.IntegerField(_(u"SRID"), blank=True, null=True)
     zipped = models.BooleanField(_(u"Zipped file"), default=False)
-    source_file = models.FileField(_(u"Source file"),
-                        upload_to='import_files', blank=True, null=True)
     origin = models.CharField(_(u"Origin"), max_length=100,
                               blank=True, null=True)
     license = models.CharField(_(u"License"), max_length=100,
                                blank=True, null=True)
+    categories = SelectMultipleField(SubCategory,
+                      verbose_name=_(u"Associated subcategories"))
+    state = models.CharField(_(u"State"), max_length=200,
+                             blank=True, null=True)
 
     class Meta:
         verbose_name = _(u"Importer")
@@ -331,6 +333,16 @@ class GeographicItem(models.Model):
     def properties(cls):
         return [pm for pm in PropertyModel.objects.filter(available=True)]
 
+def property_setter(cls, propertymodel):
+    def setter(self, value):
+        marker = self
+        if cls == Route:
+            if not self.associated_marker.objects.count():
+                return
+            marker = self.associated_marker.objects.all()[0]
+        marker.setProperty(propertymodel, value)
+    return setter
+
 class Marker(GeographicItem):
     '''Marker for a POI
     '''
@@ -359,6 +371,9 @@ class Marker(GeographicItem):
                 if property:
                     val = property.python_value
                 setattr(self, attr_name, val)
+            if not hasattr(self, attr_name + '_set'):
+                setattr(self, attr_name + '_set',
+                              property_setter(self.__class__, pm))
 
     def get_init_multi(self):
         multis = [forms.model_to_dict(multi)
@@ -432,31 +447,37 @@ class Marker(GeographicItem):
                 properties.append(property)
         return properties
 
+    def setProperty(self, pm, value):
+        u"""
+        Set a property
+        """
+        properties = Property.objects.filter(marker=self,
+                                             propertymodel=pm).all()
+        # in case of multiple edition as the same time delete arbitrary
+        # the others
+        if len(properties) > 1:
+            for property in properties[1:]:
+                property.delete()
+        # new property
+        if not properties:
+            new_property = Property.objects.create(marker=self,
+                                propertymodel=pm,
+                                value=value)
+            new_property.save()
+        else:
+            property = properties[0]
+            property.value = value
+            property.save()
+
     def saveProperties(self, values):
         """
         Save properties
         """
         for propertymodel in PropertyModel.objects.filter(available=True):
-            properties = Property.objects.filter(marker=self,
-                                          propertymodel=propertymodel).all()
-            # in case of multiple edition as the same time delete arbitrary
-            # the others
-            if len(properties) > 1:
-                for property in properties[1:]:
-                    property.delete()
             val = u""
             if unicode(propertymodel.id) in values:
                 val = values[unicode(propertymodel.id)]
-            # new property
-            if not properties:
-                new_property = Property.objects.create(marker=self,
-                                    propertymodel=propertymodel,
-                                    value=val)
-                new_property.save()
-            else:
-                property = properties[0]
-                property.value = val
-                property.save()
+            self.setProperty(propertymodel, val)
 
     def getGeoJSON(self, categories_id=[]):
         '''Return a GeoJSON string
@@ -824,8 +845,10 @@ class Route(GeographicItem):
 
     def __init__(self, *args, **kwargs):
         super(Route, self).__init__(*args, **kwargs)
+        self.description = ''
         try:
             associated_marker = Marker.objects.get(route=self)
+            self.description = associated_marker.description
         except:
             associated_marker = None
         # add read attributes for properties
@@ -838,6 +861,9 @@ class Route(GeographicItem):
                     if property:
                         val = property.python_value
                 setattr(self, attr_name, val)
+            if not hasattr(self, attr_name + '_set'):
+                setattr(self, attr_name + '_set',
+                              property_setter(self.__class__, pm))
 
     @property
     def geometry(self):
diff --git a/chimere/utils.py b/chimere/utils.py
index 53834dc..75c0ad4 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -71,17 +71,20 @@ class ImportManager:
         pass
 
     def create_or_update_item(self, cls, values, import_key, version=None,
-                              key=''):
+                              key='', pk=None):
         updated, created, item = False, False, None
         import_key = unicode(import_key).replace(':', '^')
         if not key:
             key = self.importer_instance.importer_type
-        if import_key:
+        if import_key or pk:
             dct_import = {
                 'import_key__icontains':'%s:%s;' % (key, import_key),
                 'import_source':self.importer_instance.source}
             try:
-                item = cls.objects.get(**dct_import)
+                if pk:
+                    item = cls.objects.get(pk=pk)
+                else:
+                    item = cls.objects.get(**dct_import)
                 if version and item.import_version == int(version):
                     # no update since the last import
                     return item, None, None
@@ -187,7 +190,7 @@ class KMLManager(ImportManager):
 
     def get(self):
         u"""
-        Get data from the source
+        Get data from a KML source
 
         Return a tuple with:
          - number of new item ;
@@ -280,7 +283,7 @@ class ShapefileManager(ImportManager):
     """
     def get(self):
         u"""
-        Get data from the source
+        Get data from a Shapefile source
 
         Return a tuple with:
          - number of new item ;
@@ -443,26 +446,91 @@ class CSVManager(ImportManager):
     u"""
     CSV importer
     """
-    _COLS = [("Id", 'pk'), (_(u"Name"), 'name'),
+    @classmethod
+    def set_categories(value):
+        return
+
+    # (label, getter, setter)
+    COLS = [("Id", 'pk', 'pk'), (_(u"Name"), 'name', 'name'),
             (_(u"Categories"), lambda obj:", ".join(
-                                [c.name for c in obj.categories.all()])),
-            (_(u"State"), 'status')]
-    COLS = {'marker':_COLS+[(_(u"Description"), 'description'),
-                            (_(u"Localisation"), lambda obj: obj.point.wkt)],
-            'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]}
+                                [c.name for c in obj.categories.all()]),
+                                set_categories),
+            (_(u"State"), 'status', lambda x: x),
+            (_(u"Description"), 'description', 'description'),
+            (_(u"Localisation"), 'geometry', 'geometry')]
+
+    def get(self):
+        u"""
+        Get data from a CSV source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        from models import Marker, Route
+        new_item, updated_item, msg = 0, 0, ''
+        source, msg = self.get_source_file(['.csv'])
+        if msg:
+            return (0, 0, msg)
+        reader = csv.reader(source, delimiter=';', quotechar='"')
+        prop_cols = []
+        for pm in Marker.properties():
+            prop_cols.append((pm.name, pm.getAttrName(),
+                              pm.getAttrName()+'_set'))
+        cols = self.COLS + prop_cols
+        datas = []
+        for idx, row in enumerate(reader):
+            if not idx: # first row
+                try:
+                    assert(len(row) >= len(cols))
+                except AssertionError:
+                    return (0, 0, _(u"Invalid CSV format"))
+                continue
+            if len(row) < len(cols):
+                continue
+            pk, name, cats, state = row[0], row[1], row[2], row[3]
+            description, geom = row[4], row[5]
+            COL_INDEX = 6
+            dct = {'description':description,
+                   'name':name,
+                   'origin':self.importer_instance.origin,
+                   'license':self.importer_instance.license}
+            cls = None
+            if 'POINT' in geom:
+                cls = Marker
+                dct['point'] = geom
+            elif 'LINE' in geom:
+                cls = Route
+                dct['route'] = geom
+            else:
+                continue
+            import_key = pk if pk else name
+            item, updated, created = self.create_or_update_item(cls, dct,
+                                                             import_key, pk=pk)
+            if updated:
+                updated_item += 1
+            if created:
+                new_item += 1
+            for idx, col in enumerate(cols[COL_INDEX:]):
+                name, getter, setter_val = col
+                setter = getattr(item, setter_val)
+                val = row[idx+COL_INDEX]
+                setter(item, val)
+        return (new_item, updated_item, msg)
 
     @classmethod
     def export(cls, queryset):
         dct = {'description':unicode(datetime.date.today()), 'data':[]}
         cls_name = queryset.model.__name__.lower()
-        cols = cls.COLS[cls_name][:]
+        cols = cls.COLS
         for pm in queryset.model.properties():
-            cols.append((pm.name, pm.getAttrName()))
-        header = [lbl for lbl, attr in cols]
+            cols.append((pm.name, pm.getAttrName(), pm.getAttrName()+'_set'))
+        header = [col[0] for col in cols]
         dct['data'].append(header)
         for item in queryset.all():
             data = []
-            for lbl, attr in cols:
+            for (lbl, attr, setr) in cols:
                 if callable(attr):
                     data.append(attr(item))
                 else: