CSV import: manage semi-colon as separator

6 files changed, 59 insertions, 11 deletions

diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 00a45ba6a..929cdfeff 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -423,7 +423,8 @@ class ImportFindTest(ImportTest, TestCase):
                      'imported_images': SimpleUploadedFile(mcc_images.name,
                                                            mcc_images.read())}
         post_dict = {'importer_type': MCC.pk, 'skip_lines': 1,
-                     "encoding": 'utf-8', "name": 'init_find_import'}
+                     "encoding": 'utf-8', "name": 'init_find_import',
+                     "csv_sep": u","}
         form = forms_common.NewImportForm(data=post_dict, files=file_dict,
                                           user=self.user)
         form.is_valid()
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 6781cf531..3f659ef3b 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -115,7 +115,8 @@ class ImportTest(object):
         tg.is_set = True
         tg.save()
 
-    def init_ope_import(self, filename='MCC-operations-example.csv'):
+    def init_ope_import(self, filename='MCC-operations-example.csv',
+                        sep=u","):
         mcc_operation = ImporterType.objects.get(name=u"MCC - Opérations")
         mcc_operation_file = open(
             settings.ROOT_PATH +
@@ -126,7 +127,7 @@ class ImportTest(object):
         group, c = TargetKeyGroup.objects.get_or_create(name="My group")
         post_dict = {'importer_type': mcc_operation.pk, 'skip_lines': 1,
                      "encoding": 'utf-8', "name": 'init_ope_import',
-                     "associated_group": group.pk}
+                     "associated_group": group.pk, "csv_sep": sep}
         form = forms_common.NewImportForm(data=post_dict, files=file_dict,
                                           user=self.user)
         form.is_valid()
@@ -222,7 +223,8 @@ class ImportTest(object):
         file_dict = {'imported_file': SimpleUploadedFile(mcc_file.name,
                                                          mcc_file.read())}
         post_dict = {'importer_type': mcc_parcel.pk, 'skip_lines': 1,
-                     "encoding": 'utf-8', "name": 'init_parcel_import'}
+                     "encoding": 'utf-8', "name": 'init_parcel_import',
+                     "csv_sep": u","}
         form = forms_common.NewImportForm(data=post_dict, files=file_dict,
                                           user=self.user)
         form.is_valid()
@@ -244,7 +246,8 @@ class ImportTest(object):
         file_dict = {'imported_file': SimpleUploadedFile(mcc_file.name,
                                                          mcc_file.read())}
         post_dict = {'importer_type': mcc.pk, 'skip_lines': 1,
-                     "encoding": 'utf-8', "name": 'init_context_record_import'}
+                     "encoding": 'utf-8', "name": 'init_context_record_import',
+                     'csv_sep': u","}
         form = forms_common.NewImportForm(data=post_dict, files=file_dict,
                                           user=self.user)
         form.is_valid()
@@ -318,6 +321,19 @@ class ImportOperationTest(ImportTest, TestCase):
     def test_import_bad_encoding(self):
         self.init_ope_import('MCC-operations-example-bad-encoding.csv')
 
+    def test_import_semi_colon_sep(self):
+        first_ope_nb = models.Operation.objects.count()
+        importer, form = self.init_ope_import(
+            'MCC-operations-example-semi-colon.csv', sep=u";")
+        self.assertTrue(form.is_valid())
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        self.init_ope_targetkey(imp=impt)
+        impt.importation()
+        # new operations have now been imported
+        current_ope_nb = models.Operation.objects.count()
+        self.assertEqual(current_ope_nb, first_ope_nb + 2)
+
     def test_keys_limitation(self):
         # each key association associated to the import
         init_ope_number = models.Operation.objects.count()
@@ -537,7 +553,7 @@ class ImportDocumentTest(ImportTest, TestCase):
         group, c = TargetKeyGroup.objects.get_or_create(name="My group")
         post_dict = {'importer_type': doc_import.pk, 'skip_lines': 1,
                      "encoding": 'utf-8', "name": 'init_ope_import',
-                     "associated_group": group.pk}
+                     "associated_group": group.pk, "csv_sep": u","}
         form = forms_common.NewImportForm(data=post_dict, files=file_dict,
                                           user=self.user)
         form.is_valid()
diff --git a/archaeological_operations/tests/MCC-operations-example-semi-colon.csv b/archaeological_operations/tests/MCC-operations-example-semi-colon.csv
new file mode 100644
index 000000000..f94ab5005
--- /dev/null
+++ b/archaeological_operations/tests/MCC-operations-example-semi-colon.csv
@@ -0,0 +1,3 @@
+code OA;region;type operation;intitule operation;operateur;responsable operation;date debut terrain;date fin terrain;chronologie generale;identifiant document georeferencement;notice scientifique
+4201;Bourgogne;Fouille programmée;Oppìdum de Paris 2;L'opérateur;;2000/01/31;2002/12/31;Age du Fer;;
+4200;Bourgogne;Fouille programmée;Oppìdum de Paris;L'opérateur;Jean Sui-Resp'on Sablé;2000/01/22;2002/12/31;Age du Fer & Gallo-Romain & Néolithik & Moderne;;
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index eb15445a4..b5423ca48 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -130,7 +130,7 @@ class NewImportForm(forms.ModelForm):
         model = models.Import
         fields = (
             'name', 'importer_type', 'imported_file', 'encoding',
-            'imported_images', 'imported_images_link',
+            'csv_sep', 'imported_images', 'imported_images_link',
             'associated_group', 'conservative_import', 'skip_lines'
         )
 
diff --git a/ishtar_common/migrations/0089_import_csv_sep.py b/ishtar_common/migrations/0089_import_csv_sep.py
new file mode 100644
index 000000000..c2cd4ac75
--- /dev/null
+++ b/ishtar_common/migrations/0089_import_csv_sep.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2019-03-27 16:16
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0088_auto_20190218_1808'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='import',
+            name='csv_sep',
+            field=models.CharField(choices=[(',', ','), (';', ';')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator'),
+        ),
+    ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 380a582cf..a774cef6d 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -807,6 +807,9 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING),
              (settings.ALT_ENCODING, settings.ALT_ENCODING),
              ('utf-8', 'utf-8')]
 
+CSV_SEPS = ((u",", u","),
+            (u";", u";"),)
+
 delayed_import = None
 delayed_check = None
 
@@ -848,6 +851,11 @@ class Import(models.Model):
     )
     encoding = models.CharField(_(u"Encoding"), choices=ENCODINGS,
                                 default=u'utf-8', max_length=15)
+    csv_sep = models.CharField(
+        _(u"CSV separator"), choices=CSV_SEPS, default=u',', max_length=1,
+        help_text=_(u"Separator for CSV file. Standard is comma but Microsoft "
+                    u"Excel do not follow this standard and use semi-colon.")
+    )
     skip_lines = models.IntegerField(
         _(u"Skip lines"), default=1,
         help_text=_(u"Number of header lines in your file (can be 0)."))
@@ -919,7 +927,7 @@ class Import(models.Model):
         filename = self.imported_file.path
         with open(filename, 'r') as f:
             reader = unicodecsv.reader(
-                f, encoding=self.encoding)
+                f, encoding=self.encoding, delimiter=str(self.csv_sep))
             nb = sum(1 for row in reader) - self.skip_lines
         self.number_of_line = nb
         self.save()
@@ -1040,9 +1048,9 @@ class Import(models.Model):
         for encoding in encodings:
             try:
                 with open(imported_file) as csv_file:
-                    vals = [line
-                            for line in unicodecsv.reader(csv_file,
-                                                          encoding=encoding)]
+                    vals = [line for line in unicodecsv.reader(
+                            csv_file, encoding=encoding,
+                            delimiter=str(self.csv_sep))]
                     if tmpdir:
                         shutil.rmtree(tmpdir)
                     return vals