Fix CSV import coding

1 files changed, 33 insertions, 20 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index 79cb68e..707c6bb 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -576,27 +576,10 @@ class CSVManager(ImportManager):
             (_("Description"), 'description', 'description'),
             (_("Localisation"), 'geometry', 'geometry')]
 
-    def get(self):
-        """
-        Get data from a CSV source
-
-        Return a tuple with:
-         - number of new item ;
-         - number of item updated ;
-         - error detail on error
-        """
+    def _get(self, reader, cols):
         from chimere.models import Marker, Route
         new_item, updated_item, msg = 0, 0, ''
-        source, msg = self.get_source_file(['.csv'])
-        if msg:
-            return (0, 0, msg)
-        reader = csv.reader(source, delimiter=';', quotechar='"')
-        prop_cols = []
-        for pm in Marker.all_properties():
-            prop_cols.append((pm.name, pm.getAttrName(),
-                              pm.getAttrName() + '_set'))
-        cols = list(self.COLS) + prop_cols
-        # datas = []
+
         for idx, row in enumerate(reader):
             if not idx:  # first row
                 try:
@@ -626,7 +609,7 @@ class CSVManager(ImportManager):
                 dct['route'] = geom
             else:
                 continue
-            import_key = pk if pk else name.decode('utf-8')
+            import_key = pk if pk else name
             item, updated, created = self.create_or_update_item(
                 cls, dct, import_key, pk=pk)
             if updated:
@@ -640,6 +623,36 @@ class CSVManager(ImportManager):
                 setter(item, val)
         return (new_item, updated_item, msg)
 
+    def get(self):
+        """
+        Get data from a CSV source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        from chimere.models import Marker
+        source, msg = self.get_source_file(['.csv'])
+        if msg:
+            return (0, 0, msg)
+
+        prop_cols = []
+        for pm in Marker.all_properties():
+            prop_cols.append((pm.name, pm.getAttrName(),
+                              pm.getAttrName() + '_set'))
+        cols = list(self.COLS) + prop_cols
+
+        for coding in ['utf-8', 'windows-1252', 'iso-8859-1', 'iso-8859-15']:
+            try:
+                text = open(source.path, 'rt', newline='', encoding=coding)
+                reader = csv.reader(text,
+                                    delimiter=';', quotechar='"')
+                return self._get(reader, cols)
+            except UnicodeDecodeError:
+                pass
+        return (0, 0, _("Invalid CSV format (encoding error)"))
+
     @classmethod
     def export(cls, queryset):
         dct = {'description': str(datetime.date.today()), 'data': []}