diff options
| -rw-r--r-- | archaeological_finds/tests.py | 3 | ||||
| -rw-r--r-- | archaeological_operations/tests.py | 26 | ||||
| -rw-r--r-- | archaeological_operations/tests/MCC-operations-example-semi-colon.csv | 3 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 2 | ||||
| -rw-r--r-- | ishtar_common/migrations/0089_import_csv_sep.py | 20 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 16 |
6 files changed, 59 insertions, 11 deletions
diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py index 00a45ba6a..929cdfeff 100644 --- a/archaeological_finds/tests.py +++ b/archaeological_finds/tests.py @@ -423,7 +423,8 @@ class ImportFindTest(ImportTest, TestCase): 'imported_images': SimpleUploadedFile(mcc_images.name, mcc_images.read())} post_dict = {'importer_type': MCC.pk, 'skip_lines': 1, - "encoding": 'utf-8', "name": 'init_find_import'} + "encoding": 'utf-8', "name": 'init_find_import', + "csv_sep": u","} form = forms_common.NewImportForm(data=post_dict, files=file_dict, user=self.user) form.is_valid() diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 6781cf531..3f659ef3b 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -115,7 +115,8 @@ class ImportTest(object): tg.is_set = True tg.save() - def init_ope_import(self, filename='MCC-operations-example.csv'): + def init_ope_import(self, filename='MCC-operations-example.csv', + sep=u","): mcc_operation = ImporterType.objects.get(name=u"MCC - Opérations") mcc_operation_file = open( settings.ROOT_PATH + @@ -126,7 +127,7 @@ class ImportTest(object): group, c = TargetKeyGroup.objects.get_or_create(name="My group") post_dict = {'importer_type': mcc_operation.pk, 'skip_lines': 1, "encoding": 'utf-8', "name": 'init_ope_import', - "associated_group": group.pk} + "associated_group": group.pk, "csv_sep": sep} form = forms_common.NewImportForm(data=post_dict, files=file_dict, user=self.user) form.is_valid() @@ -222,7 +223,8 @@ class ImportTest(object): file_dict = {'imported_file': SimpleUploadedFile(mcc_file.name, mcc_file.read())} post_dict = {'importer_type': mcc_parcel.pk, 'skip_lines': 1, - "encoding": 'utf-8', "name": 'init_parcel_import'} + "encoding": 'utf-8', "name": 'init_parcel_import', + "csv_sep": u","} form = forms_common.NewImportForm(data=post_dict, files=file_dict, user=self.user) form.is_valid() @@ -244,7 +246,8 @@ class ImportTest(object): file_dict = {'imported_file': SimpleUploadedFile(mcc_file.name, mcc_file.read())} post_dict = {'importer_type': mcc.pk, 'skip_lines': 1, - "encoding": 'utf-8', "name": 'init_context_record_import'} + "encoding": 'utf-8', "name": 'init_context_record_import', + 'csv_sep': u","} form = forms_common.NewImportForm(data=post_dict, files=file_dict, user=self.user) form.is_valid() @@ -318,6 +321,19 @@ class ImportOperationTest(ImportTest, TestCase): def test_import_bad_encoding(self): self.init_ope_import('MCC-operations-example-bad-encoding.csv') + def test_import_semi_colon_sep(self): + first_ope_nb = models.Operation.objects.count() + importer, form = self.init_ope_import( + 'MCC-operations-example-semi-colon.csv', sep=u";") + self.assertTrue(form.is_valid()) + impt = form.save(self.ishtar_user) + impt.initialize() + self.init_ope_targetkey(imp=impt) + impt.importation() + # new operations have now been imported + current_ope_nb = models.Operation.objects.count() + self.assertEqual(current_ope_nb, first_ope_nb + 2) + def test_keys_limitation(self): # each key association associated to the import init_ope_number = models.Operation.objects.count() @@ -537,7 +553,7 @@ class ImportDocumentTest(ImportTest, TestCase): group, c = TargetKeyGroup.objects.get_or_create(name="My group") post_dict = {'importer_type': doc_import.pk, 'skip_lines': 1, "encoding": 'utf-8', "name": 'init_ope_import', - "associated_group": group.pk} + "associated_group": group.pk, "csv_sep": u","} form = forms_common.NewImportForm(data=post_dict, files=file_dict, user=self.user) form.is_valid() diff --git a/archaeological_operations/tests/MCC-operations-example-semi-colon.csv b/archaeological_operations/tests/MCC-operations-example-semi-colon.csv new file mode 100644 index 000000000..f94ab5005 --- /dev/null +++ b/archaeological_operations/tests/MCC-operations-example-semi-colon.csv @@ -0,0 +1,3 @@ +code OA;region;type operation;intitule operation;operateur;responsable operation;date debut terrain;date fin terrain;chronologie generale;identifiant document georeferencement;notice scientifique +4201;Bourgogne;Fouille programmée;Oppìdum de Paris 2;L'opérateur;;2000/01/31;2002/12/31;Age du Fer;; +4200;Bourgogne;Fouille programmée;Oppìdum de Paris;L'opérateur;Jean Sui-Resp'on Sablé;2000/01/22;2002/12/31;Age du Fer & Gallo-Romain & Néolithik & Moderne;; diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index eb15445a4..b5423ca48 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -130,7 +130,7 @@ class NewImportForm(forms.ModelForm): model = models.Import fields = ( 'name', 'importer_type', 'imported_file', 'encoding', - 'imported_images', 'imported_images_link', + 'csv_sep', 'imported_images', 'imported_images_link', 'associated_group', 'conservative_import', 'skip_lines' ) diff --git a/ishtar_common/migrations/0089_import_csv_sep.py b/ishtar_common/migrations/0089_import_csv_sep.py new file mode 100644 index 000000000..c2cd4ac75 --- /dev/null +++ b/ishtar_common/migrations/0089_import_csv_sep.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2019-03-27 16:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0088_auto_20190218_1808'), + ] + + operations = [ + migrations.AddField( + model_name='import', + name='csv_sep', + field=models.CharField(choices=[(',', ','), (';', ';')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator'), + ), + ] diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 380a582cf..a774cef6d 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -807,6 +807,9 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING), (settings.ALT_ENCODING, settings.ALT_ENCODING), ('utf-8', 'utf-8')] +CSV_SEPS = ((u",", u","), + (u";", u";"),) + delayed_import = None delayed_check = None @@ -848,6 +851,11 @@ class Import(models.Model): ) encoding = models.CharField(_(u"Encoding"), choices=ENCODINGS, default=u'utf-8', max_length=15) + csv_sep = models.CharField( + _(u"CSV separator"), choices=CSV_SEPS, default=u',', max_length=1, + help_text=_(u"Separator for CSV file. Standard is comma but Microsoft " + u"Excel do not follow this standard and use semi-colon.") + ) skip_lines = models.IntegerField( _(u"Skip lines"), default=1, help_text=_(u"Number of header lines in your file (can be 0).")) @@ -919,7 +927,7 @@ class Import(models.Model): filename = self.imported_file.path with open(filename, 'r') as f: reader = unicodecsv.reader( - f, encoding=self.encoding) + f, encoding=self.encoding, delimiter=str(self.csv_sep)) nb = sum(1 for row in reader) - self.skip_lines self.number_of_line = nb self.save() @@ -1040,9 +1048,9 @@ class Import(models.Model): for encoding in encodings: try: with open(imported_file) as csv_file: - vals = [line - for line in unicodecsv.reader(csv_file, - encoding=encoding)] + vals = [line for line in unicodecsv.reader( + csv_file, encoding=encoding, + delimiter=str(self.csv_sep))] if tmpdir: shutil.rmtree(tmpdir) return vals |