Import: import CSV files

1 files changed, 83 insertions, 15 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index 53834dc..75c0ad4 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -71,17 +71,20 @@ class ImportManager:
         pass
 
     def create_or_update_item(self, cls, values, import_key, version=None,
-                              key=''):
+                              key='', pk=None):
         updated, created, item = False, False, None
         import_key = unicode(import_key).replace(':', '^')
         if not key:
             key = self.importer_instance.importer_type
-        if import_key:
+        if import_key or pk:
             dct_import = {
                 'import_key__icontains':'%s:%s;' % (key, import_key),
                 'import_source':self.importer_instance.source}
             try:
-                item = cls.objects.get(**dct_import)
+                if pk:
+                    item = cls.objects.get(pk=pk)
+                else:
+                    item = cls.objects.get(**dct_import)
                 if version and item.import_version == int(version):
                     # no update since the last import
                     return item, None, None
@@ -187,7 +190,7 @@ class KMLManager(ImportManager):
 
     def get(self):
         u"""
-        Get data from the source
+        Get data from a KML source
 
         Return a tuple with:
          - number of new item ;
@@ -280,7 +283,7 @@ class ShapefileManager(ImportManager):
     """
     def get(self):
         u"""
-        Get data from the source
+        Get data from a Shapefile source
 
         Return a tuple with:
          - number of new item ;
@@ -443,26 +446,91 @@ class CSVManager(ImportManager):
     u"""
     CSV importer
     """
-    _COLS = [("Id", 'pk'), (_(u"Name"), 'name'),
+    @classmethod
+    def set_categories(value):
+        return
+
+    # (label, getter, setter)
+    COLS = [("Id", 'pk', 'pk'), (_(u"Name"), 'name', 'name'),
             (_(u"Categories"), lambda obj:", ".join(
-                                [c.name for c in obj.categories.all()])),
-            (_(u"State"), 'status')]
-    COLS = {'marker':_COLS+[(_(u"Description"), 'description'),
-                            (_(u"Localisation"), lambda obj: obj.point.wkt)],
-            'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]}
+                                [c.name for c in obj.categories.all()]),
+                                set_categories),
+            (_(u"State"), 'status', lambda x: x),
+            (_(u"Description"), 'description', 'description'),
+            (_(u"Localisation"), 'geometry', 'geometry')]
+
+    def get(self):
+        u"""
+        Get data from a CSV source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        from models import Marker, Route
+        new_item, updated_item, msg = 0, 0, ''
+        source, msg = self.get_source_file(['.csv'])
+        if msg:
+            return (0, 0, msg)
+        reader = csv.reader(source, delimiter=';', quotechar='"')
+        prop_cols = []
+        for pm in Marker.properties():
+            prop_cols.append((pm.name, pm.getAttrName(),
+                              pm.getAttrName()+'_set'))
+        cols = self.COLS + prop_cols
+        datas = []
+        for idx, row in enumerate(reader):
+            if not idx: # first row
+                try:
+                    assert(len(row) >= len(cols))
+                except AssertionError:
+                    return (0, 0, _(u"Invalid CSV format"))
+                continue
+            if len(row) < len(cols):
+                continue
+            pk, name, cats, state = row[0], row[1], row[2], row[3]
+            description, geom = row[4], row[5]
+            COL_INDEX = 6
+            dct = {'description':description,
+                   'name':name,
+                   'origin':self.importer_instance.origin,
+                   'license':self.importer_instance.license}
+            cls = None
+            if 'POINT' in geom:
+                cls = Marker
+                dct['point'] = geom
+            elif 'LINE' in geom:
+                cls = Route
+                dct['route'] = geom
+            else:
+                continue
+            import_key = pk if pk else name
+            item, updated, created = self.create_or_update_item(cls, dct,
+                                                             import_key, pk=pk)
+            if updated:
+                updated_item += 1
+            if created:
+                new_item += 1
+            for idx, col in enumerate(cols[COL_INDEX:]):
+                name, getter, setter_val = col
+                setter = getattr(item, setter_val)
+                val = row[idx+COL_INDEX]
+                setter(item, val)
+        return (new_item, updated_item, msg)
 
     @classmethod
     def export(cls, queryset):
         dct = {'description':unicode(datetime.date.today()), 'data':[]}
         cls_name = queryset.model.__name__.lower()
-        cols = cls.COLS[cls_name][:]
+        cols = cls.COLS
         for pm in queryset.model.properties():
-            cols.append((pm.name, pm.getAttrName()))
-        header = [lbl for lbl, attr in cols]
+            cols.append((pm.name, pm.getAttrName(), pm.getAttrName()+'_set'))
+        header = [col[0] for col in cols]
         dct['data'].append(header)
         for item in queryset.all():
             data = []
-            for lbl, attr in cols:
+            for (lbl, attr, setr) in cols:
                 if callable(attr):
                     data.append(attr(item))
                 else: