diff options
| -rw-r--r-- | archaeological_operations/tests.py | 4 | ||||
| -rw-r--r-- | archaeological_operations/tests/MCC-operations-example.csv | 2 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 4 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 2 | ||||
| -rw-r--r-- | ishtar_common/migrations/0033_auto__add_field_import_encoding.py | 321 | ||||
| -rw-r--r-- | ishtar_common/models.py | 21 |
6 files changed, 341 insertions, 13 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index c7e75da1b..ccb67cc03 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -86,7 +86,8 @@ class ImportOperationTest(TestCase): # doing manualy connections tg = TargetKey.objects.filter(target__target='operation_type' ).order_by('-pk').all()[0] - tg.value = models.OperationType.objects.get(txt_idx='prog_excavation').pk + tg.value = models.OperationType.objects.get( + txt_idx='prog_excavation').pk tg.is_set = True tg.save() @@ -110,6 +111,7 @@ class ImportOperationTest(TestCase): self.assertTrue(current_ope_nb == (first_ope_nb + 1)) # and well imported last_ope = models.Operation.objects.order_by('-pk').all()[0] + self.assertEqual(last_ope.name, u"Oppìdum de Paris") self.assertTrue(last_ope.code_patriarche == 4200) self.assertTrue(last_ope.operation_type.txt_idx == 'prog_excavation') self.assertEqual(last_ope.periods.count(), 2) diff --git a/archaeological_operations/tests/MCC-operations-example.csv b/archaeological_operations/tests/MCC-operations-example.csv index 4962c0969..432ceffca 100644 --- a/archaeological_operations/tests/MCC-operations-example.csv +++ b/archaeological_operations/tests/MCC-operations-example.csv @@ -1,2 +1,2 @@ code OA,region,type operation,intitule operation,operateur,responsable operation,date debut terrain,date fin terrain,chronologie generale,identifiant document georeferencement,notice scientifique -4200,Bourgogne,Fouille programmée,Oppidum de Paris,L'opérateur,Jean Sui-Resp'on Sablé,2000/01/22,2002/12/31,Age du Fer & Gallo-Romain,, +4200,Bourgogne,Fouille programmée,Oppìdum de Paris,L'opérateur,Jean Sui-Resp'on Sablé,2000/01/22,2002/12/31,Age du Fer & Gallo-Romain,, diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 1954559cc..ea1fbf76f 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -107,7 +107,9 @@ class UnicodeFormater(Formater): def format(self, value): try: - value = unicode(value.strip()) + value = value.strip() + if type(value) != unicode: + value = unicode(value.strip()) if self.re_filter: m = self.re_filter.match(value) if m: diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 9665bcf18..ac08b6c4e 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -92,7 +92,7 @@ class NewItemForm(forms.Form): class NewImportForm(forms.ModelForm): class Meta: model = models.Import - fields = ('importer_type', 'imported_file', 'skip_lines') + fields = ('importer_type', 'imported_file', 'encoding', 'skip_lines') def save(self, user, commit=True): self.instance.user = user diff --git a/ishtar_common/migrations/0033_auto__add_field_import_encoding.py b/ishtar_common/migrations/0033_auto__add_field_import_encoding.py new file mode 100644 index 000000000..e790a928a --- /dev/null +++ b/ishtar_common/migrations/0033_auto__add_field_import_encoding.py @@ -0,0 +1,321 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Import.encoding' + db.add_column('ishtar_common_import', 'encoding', + self.gf('django.db.models.fields.CharField')(default='utf-8', max_length=15), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Import.encoding' + db.delete_column('ishtar_common_import', 'encoding') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'ishtar_common.arrondissement': { + 'Meta': {'object_name': 'Arrondissement'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.author': { + 'Meta': {'object_name': 'Author'}, + 'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"}) + }, + 'ishtar_common.authortype': { + 'Meta': {'object_name': 'AuthorType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.canton': { + 'Meta': {'object_name': 'Canton'}, + 'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.department': { + 'Meta': {'ordering': "['number']", 'object_name': 'Department'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}) + }, + 'ishtar_common.documenttemplate': { + 'Meta': {'ordering': "['associated_object_name', 'name']", 'object_name': 'DocumentTemplate'}, + 'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}) + }, + 'ishtar_common.format': { + 'Meta': {'object_name': 'Format'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.formatertype': { + 'Meta': {'ordering': "('formater_type', 'options')", 'unique_together': "(('formater_type', 'options', 'many_split'),)", 'object_name': 'FormaterType'}, + 'formater_type': ('django.db.models.fields.CharField', [], {'max_length': '20'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'many_split': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'options': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.globalvar': { + 'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.historicalorganization': { + 'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.import': { + 'Meta': {'object_name': 'Import'}, + 'creation_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}), + 'encoding': ('django.db.models.fields.CharField', [], {'default': "'utf-8'", 'max_length': '15'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'error_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imported_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.ImporterType']"}), + 'result_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'seconds_remaining': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'skip_lines': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'state': ('django.db.models.fields.CharField', [], {'default': "'C'", 'max_length': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']"}) + }, + 'ishtar_common.importercolumn': { + 'Meta': {'object_name': 'ImporterColumn'}, + 'col_number': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'columns'", 'to': "orm['ishtar_common.ImporterType']"}), + 'regexp_pre_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'ishtar_common.importerdefault': { + 'Meta': {'object_name': 'ImporterDefault'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'defaults'", 'to': "orm['ishtar_common.ImporterType']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerdefaultvalues': { + 'Meta': {'object_name': 'ImporterDefaultValues'}, + 'default_target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'default_values'", 'to': "orm['ishtar_common.ImporterDefault']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerduplicatefield': { + 'Meta': {'object_name': 'ImporterDuplicateField'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_fields'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'field_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'ishtar_common.importertype': { + 'Meta': {'object_name': 'ImporterType'}, + 'associated_models': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_template': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.importtarget': { + 'Meta': {'object_name': 'ImportTarget'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'targets'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'formater_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.FormaterType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'regexp_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.ishtaruser': { + 'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']}, + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}), + 'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'ishtar_common.itemkey': { + 'Meta': {'object_name': 'ItemKey'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}) + }, + 'ishtar_common.organization': { + 'Meta': {'object_name': 'Organization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_organization'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.organizationtype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.person': { + 'Meta': {'object_name': 'Person'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_person'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '2', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.persontype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.regexp': { + 'Meta': {'object_name': 'Regexp'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'regexp': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.sourcetype': { + 'Meta': {'object_name': 'SourceType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.supporttype': { + 'Meta': {'object_name': 'SupportType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.targetkey': { + 'Meta': {'unique_together': "(('target', 'key', 'associated_user', 'associated_import'),)", 'object_name': 'TargetKey'}, + 'associated_import': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}), + 'associated_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_set': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'key': ('django.db.models.fields.TextField', [], {}), + 'target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'keys'", 'to': "orm['ishtar_common.ImportTarget']"}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.town': { + 'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'}, + 'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}), + 'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}), + 'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}), + 'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['ishtar_common']
\ No newline at end of file diff --git a/ishtar_common/models.py b/ishtar_common/models.py index a23808c73..1f654fdf4 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1352,12 +1352,17 @@ IMPORT_STATE = (("C", _(u"Created")), ) IMPORT_STATE_DCT = dict(IMPORT_STATE) +ENCODINGS = [(settings.ENCODING, settings.ENCODING), + (settings.ALT_ENCODING, settings.ALT_ENCODING), + ('utf-8', 'utf-8')] class Import(models.Model): user = models.ForeignKey('IshtarUser') importer_type = models.ForeignKey(ImporterType) imported_file = models.FileField(_(u"Imported file"), upload_to="upload/imports/") + encoding = models.CharField(_(u"Encoding"), choices=ENCODINGS, + default='utf-8', max_length=15) skip_lines = models.IntegerField(_(u"Skip lines"), default=1) error_file = models.FileField(_(u"Error file"), upload_to="upload/imports/", @@ -1420,16 +1425,14 @@ class Import(models.Model): @property def data_table(self): - encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] with open(self.imported_file.path) as csv_file: - for encoding in encodings: - try: - return [line for line in unicodecsv.reader(csv_file, - encoding=encoding)] - except UnicodeDecodeError: - if encoding != encodings[-1]: - csv_file.seek(0) - continue + try: + return [line for line in unicodecsv.reader(csv_file, + encoding=self.encoding)] + except UnicodeDecodeError: + if encoding != encodings[-1]: + csv_file.seek(0) + return [] return [] def initialize(self): |