Work on SRA importation

7 files changed, 1208 insertions, 176 deletions

diff --git a/archaeological_files/data_importer.py b/archaeological_files/data_importer.py
index b5f63fb67..23e9c6a32 100644
--- a/archaeological_files/data_importer.py
+++ b/archaeological_files/data_importer.py
@@ -17,11 +17,13 @@
 
 # See the file COPYING for details.
 
-import re, copy
+import copy, datetime, re
 import unicodecsv
 
 from django.conf import settings
+from django.db import IntegrityError
 from django.template.defaultfilters import slugify
+from django.utils.translation import ugettext_lazy as _
 
 from ishtar_common.data_importer import *
 from ishtar_common.models import Town, Person, OrganizationType
@@ -29,7 +31,95 @@ from ishtar_common.unicode_csv import unicode_csv_reader
 
 from archaeological_files import models
 
+from archaeological_operations.models import Parcel
+from archaeological_operations.utils import parse_parcels
+
 RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
+RE_PERMIT_REFERENCE = re.compile('[A-Za-z]*(.*)')
+
+class StrToBoolean(Formater):
+    def __init__(self, choices={}, cli=False, strict=False):
+        self.dct = copy.copy(choices)
+        self.cli = cli
+        self.strict= strict
+        self.missings = set()
+
+    def prepare(self, value):
+        value = unicode(value).strip()
+        if not self.strict:
+            value = slugify(value)
+        return value
+
+    def check(self, values):
+        msgstr = unicode(_(u"Choice for \"%s\" is not available. "\
+                           u"Which one is relevant?\n"))
+        msgstr += u"1. True\n"
+        msgstr += u"2. False\n"
+        msgstr += u"3. Empty\n"
+        for value in values:
+            value = self.prepare(value)
+            if value in self.dct:
+                continue
+            if not self.cli:
+                self.missings.add(value)
+                continue
+            res = None
+            while res not in range(1, 4):
+                sys.stdout.write(msgstr % value)
+                res = raw_input(">>> ")
+                try:
+                    res = int(res)
+                except ValueError:
+                    pass
+            if res == 1:
+                self.dct[value] = True
+            elif res == 2:
+                self.dct[value] = False
+            else:
+                self.dct[value] = None
+
+    def format(self, value):
+        value = self.prepare(value)
+        if value in self.dct:
+            return self.dct[value]
+
+class ImportClosingFormater(ImportFormater):
+    def post_process(self, obj, context, value, owner=None):
+        value = self.formater.format(value)
+        if not value:
+            return
+        open_date = obj.reception_date or obj.creation_date
+        if not open_date:
+            return
+        obj.end_date = open_date + datetime.timedelta(30)
+        obj.save()
+
+class ImportParcelFormater(ImportFormater):
+    NEED = ['town',]
+    PARCEL_OWNER_KEY = 'associated_file'
+
+    def post_process(self, obj, context, value, owner=None):
+        value = value.strip()
+        base_dct = {self.PARCEL_OWNER_KEY:obj, 'history_modifier':owner}
+        if 'parcels' in context:
+            for key in context['parcels']:
+                if context['parcels'][key]:
+                    base_dct[key] = context['parcels'][key]
+        for parcel_dct in parse_parcels(value, owner=owner):
+            parcel_dct.update(base_dct)
+            try:
+                Parcel.objects.get_or_create(**parcel_dct)
+            except IntegrityError:
+                raise ImporterError("Erreur d'import parcelle, contexte : %s" \
+                                                    % unicode(parcel_dct))
+
+class ImportYearFormater(ImportFormater):
+    def post_process(self, obj, context, value, owner=None):
+        value = self.formater.format(value)
+        if not value:
+            return
+        obj.year = value.year
+        obj.save()
 
 class TownFormater(Formater):
     def __init__(self, town_full_dct={}, town_dct={}):
@@ -69,15 +159,56 @@ class TownFormater(Formater):
         if key in self._town_dct:
             return self._town_dct[key]
 
+class TownINSEEFormater(Formater):
+    def __init__(self):
+        self._town_dct = {}
+
+    def format(self, value, extra=None):
+        value = value.strip()
+        if not value:
+            return None
+        if value in self._town_dct:
+            return self._town_dct[value]
+        q = Town.objects.filter(insee_code=value)
+        if not q.count():
+            return
+        self._town_dct[value] = q.all()[0]
+        return self._town_dct[value]
+
+class SurfaceFormater(Formater):
+    def test(self):
+        assert self.format(u"352 123") == 352123
+        assert self.format(u"456 789 m²") == 456789
+        assert self.format(u"78ha") == 780000
+
+    def format(self, value, extra=None):
+        value = value.strip()
+        if not value:
+            return None
+        factor = 1
+        if value.endswith(u"m2") or value.endswith(u"m²"):
+            value = value[:-2]
+        if value.endswith(u"ha"):
+            value = value[:-2]
+            factor = 10000
+        try:
+            return int(value.replace(' ', '')) * factor
+        except ValueError:
+            raise ImporterError("Erreur import surface : %s" \
+                                                    % unicode(value))
+
 #RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)[, ](\d{5}) (.*?) *(?: "\
 #                                   "*CEDEX|cedex|Cedex *\d*)*")
 
-RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]+(\d{5})[, ]+(.+)")
+RE_NAME_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*" + NEW_LINE_BREAK \
+                                      + "(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)")
 
 RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)")
 
 RE_CD_POSTAL_FILTER = re.compile("(\d*) (\d*)")
 
+RE_ORGA = re.compile("([^,]*)")
+
 class FileImporterSraPdL(Importer):
     LINE_FORMAT = []
     OBJECT_CLS = models.File
@@ -89,30 +220,29 @@ class FileImporterSraPdL(Importer):
                                              txt_idx="general_contractor")},
                 tuple():{
                     'file_type': models.FileType.objects.get(
-                                             txt_idx='undefined'),}
+                                             txt_idx='undefined'),
+                },
+                ('in_charge',):{'attached_to':None}, # initialized in __init__
                }
 
     def _init_line_format(self):
         tf = TownFormater()
         tf.town_dct_init()
         self.line_format = [
-        ImportFormater('responsible_town_planning_service__name',
-                       UnicodeFormater(300),
-                       comment=u"Service instructeur - nom",
-                       required=False),
-        ImportFormater(['address', 'postal_code', 'towns'],
+        None, # A, 1
+        ImportFormater(['address', 'postal_code', ['towns', 'parcels__town']], # B, 2
                     [UnicodeFormater(500, clean=True),
                      UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
                      tf],
                     regexp=RE_ADD_CD_POSTAL_TOWN,
                     regexp_formater_args=[[0], [1], [2, 1]], required=False,
                     comment="Dossier - adresse"),
-        ImportFormater('general_contractor__name',
+        ImportFormater('general_contractor__raw_name', # C, 3 TODO - extraire nom_prenom_titre
                        UnicodeFormater(200),
-                       comment=u"Aménageur - nom",
+                       comment=u"Aménageur - nom brut",
                        duplicate_field='general_contractor__attached_to__name',
                        required=False),
-        ImportFormater(['general_contractor__attached_to__address',
+        ImportFormater(['general_contractor__attached_to__address', # D, 4
                         'general_contractor__attached_to__postal_code',
                         'general_contractor__attached_to__town'],
                     [UnicodeFormater(500, clean=True),
@@ -122,13 +252,174 @@ class FileImporterSraPdL(Importer):
                     regexp=RE_ADD_CD_POSTAL_TOWN,
                     regexp_formater_args=[[0], [1], [2, 1]], required=False,
                     comment="Aménageur - adresse"),
-        ImportFormater("general_contractor__title",
+        ImportFormater("general_contractor__title", # E, 5
                         StrChoiceFormater(Person.TYPE, cli=True),
+                        required=False,
+                        comment="Aménageur - titre"),
+        None, # F, 6
+        None, # G, 7
+        None, # H, 8
+        ImportFormater("parcels__year", # I, 9
+                        YearNoFuturFormater(),
                         required=False),
+        ImportParcelFormater('', required=False, post_processing=True), # J, 10
+        None, # K, 11
+        ImportFormater([['towns', 'parcels__town']], # L, 12
+                       tf,
+                       required=False,
+                       comment="Commune (si non définie avant)"),
+        ImportFormater([['towns', 'parcels__town']], # M, 13
+                       tf,
+                       required=False,
+                       comment="Commune (si non définie avant)"),
+        ImportFormater('saisine_type', # N, 14
+                        StrChoiceFormater(models.SaisineType.get_types(),
+                                          model=models.SaisineType, cli=True),
+                        required=False,
+                        comment="Type de saisine"),
+        None, # O, 15
+        ImportFormater('comment', # P, 16
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        None, # Q, 17
+        ImportFormater([
+                    'responsible_town_planning_service__raw_name', # R, 18 service instructeur
+                    'responsible_town_planning_service__attached_to__address',
+                    'responsible_town_planning_service__attached_to__postal_code',
+                    'responsible_town_planning_service__attached_to__town',],
+                    [UnicodeFormater(300, clean=True),
+                     UnicodeFormater(300, clean=True),
+                     UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
+                     TownFormater(town_full_dct=tf._town_full_dct,
+                                  town_dct=tf._town_dct)],
+                    regexp=RE_NAME_ADD_CD_POSTAL_TOWN,
+                    regexp_formater_args=[[0], [1], [2], [3, 2]],
+                    comment="Aménageur - adresse",
+                    required=False),
+        ImportFormater('comment', # S, 19
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        ImportYearFormater('reception_date', # T, 20
+                       DateFormater(),
+                       comment=u"Date de création",
+                       required=False,
+                       duplicate_field='creation_date'),
+        None, # U, 21
+        None, # V, 22
+        None, # W, 23
+        None, # X, 24
+        None, # Y, 25
+        None, # Z, 26
+        None, # AA, 27
+        None, # AB, 28
+        None, # AC, 29
+        None, # AD, 30
+        None, # AE, 31
+        None, # AF, 32
+        None, # AG, 33
+        None, # AH, 34
+        ImportFormater('creation_date', # AI, 35
+                       DateFormater(),
+                       force_value=True,
+                       comment=u"Date de création",
+                       required=False,),
+        None, # AJ, 36
+        ImportFormater('comment', # AK, 37
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        None, # AL, 38
+        None, # AM, 39
+        None, # AN, 40
+        None, # AO, 41
+        ImportFormater('comment', # AP, 42
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        None, # AQ, 43
+        None, # AR, 44
+        None, # AS, 45
+        None, # AT, 46
+        ImportFormater('comment', # AU, 47
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        None, # AV, 48
+        ImportFormater('permit_reference', # AW, 49
+                       UnicodeFormater(300, clean=True),
+                       regexp=RE_PERMIT_REFERENCE,
+                       comment="Réf. du permis de construire",
+                       required=False),
+        None, # AX, 50
+        None, # AY, 51
+        None, # AZ, 52
+        None, # BA, 53
+        None, # BB, 54
+        None, # BC, 55
+        None, # BD, 56
+        ImportFormater([['towns', 'parcels__town']], # BE, 57
+                       TownINSEEFormater(),
+                       required=False,
+                       comment="Commune (si non définie avant)"),
+        ImportFormater('comment', # BF, 58
+                       UnicodeFormater(2000),
+                       comment=u"Commentaire",
+                       concat=True, required=False),
+        None, # BG, 59
+        None, # BH, 60
+        None, # BI, 61
+        None, # BJ, 62
+        None, # BK, 63
+        None, # BL, 64
+        None, # BM, 65
+        None, # BN, 66
+        None, # BO, 67
+        None, # BP, 68
+        None, # BQ, 69
+        None, # BR, 70
+        None, # BS, 71
+        ImportFormater(
+                    'responsible_town_planning_service__attached_to__name', # BT, 72 service instructeur
+                    UnicodeFormater(300, clean=True),
+                    regexp=RE_ORGA,
+                    comment="Service instructeur - nom",
+                    required=False),
+        None, # BU, 73
+        ImportClosingFormater('', StrToBoolean(cli=True),
+                              post_processing=True, required=False), # BV, 74, end date
+        ImportClosingFormater('in_charge__raw_name', # BW, 75 responsable
+                       UnicodeFormater(200),
+                       comment=u"Responsable - nom brut",
+                       required=False),
+        ImportFormater('total_surface', # BX, 76 surface totale
+                       SurfaceFormater(),
+                       comment=u"Surface totale",
+                       required=False),
+        ImportFormater('total_developed_surface', # BY, 77 surface totale aménagée
+                       SurfaceFormater(),
+                       comment=u"Surface totale aménagée",
+                       required=False),
+        None, # BZ, 78
+        None, # CA, 79
+        None, # CB, 80
+        None, # CC, 81
+        None, # CD, 82
+        None, # CE, 83
+        None, # CF, 84
+        ImportFormater('permit_type',
+                        StrChoiceFormater(models.PermitType.get_types(),
+                                          model=models.PermitType, cli=True),
+                        required=False,
+                        comment="Type de permis"), # CG, 85
+        None, # CH, 85
         ]
 
     def __init__(self, *args, **kwargs):
         super(FileImporterSraPdL, self).__init__(*args, **kwargs)
+        self.DEFAULTS[('in_charge',)]['attached_to'] = \
+                    models.Organization.objects.get(name='SRA Pays de la Loire')
         self._init_line_format()
         if tuple() not in self._defaults:
             self._defaults[tuple()] = {}
diff --git a/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py b/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py
new file mode 100644
index 000000000..4555145a3
--- /dev/null
+++ b/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py
@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'File.imported_line'
+        db.add_column('archaeological_files_file', 'imported_line',
+                      self.gf('django.db.models.fields.TextField')(null=True, blank=True),
+                      keep_default=False)
+
+
+        # Changing field 'File.responsible_town_planning_service'
+        db.alter_column('archaeological_files_file', 'responsible_town_planning_service_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.general_contractor'
+        db.alter_column('archaeological_files_file', 'general_contractor_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.in_charge'
+        db.alter_column('archaeological_files_file', 'in_charge_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.scientist'
+        db.alter_column('archaeological_files_file', 'scientist_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.organization'
+        db.alter_column('archaeological_files_file', 'organization_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Organization']))
+        # Adding field 'HistoricalFile.imported_line'
+        db.add_column('archaeological_files_historicalfile', 'imported_line',
+                      self.gf('django.db.models.fields.TextField')(null=True, blank=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'File.imported_line'
+        db.delete_column('archaeological_files_file', 'imported_line')
+
+
+        # Changing field 'File.responsible_town_planning_service'
+        db.alter_column('archaeological_files_file', 'responsible_town_planning_service_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.general_contractor'
+        db.alter_column('archaeological_files_file', 'general_contractor_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.in_charge'
+        db.alter_column('archaeological_files_file', 'in_charge_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.scientist'
+        db.alter_column('archaeological_files_file', 'scientist_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person']))
+
+        # Changing field 'File.organization'
+        db.alter_column('archaeological_files_file', 'organization_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['ishtar_common.Organization'], null=True))
+        # Deleting field 'HistoricalFile.imported_line'
+        db.delete_column('archaeological_files_historicalfile', 'imported_line')
+
+
+    models = {
+        'archaeological_files.file': {
+            'Meta': {'ordering': "('cached_label',)", 'object_name': 'File'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'cached_label': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'cira_advised': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'classified_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'creation_date': ('django.db.models.fields.DateField', [], {'default': 'datetime.date.today'}),
+            'departments': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'file_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.FileType']"}),
+            'general_contractor': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'general_contractor'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imported_line': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'in_charge': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'file_responsability'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}),
+            'internal_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}),
+            'mh_listing': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'mh_register': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'numeric_reference': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'organization': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'files'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}),
+            'permit_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}),
+            'permit_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.PermitType']", 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'protected_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'reception_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'reference_number': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'related_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.File']", 'null': 'True', 'blank': 'True'}),
+            'requested_operation_type': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['archaeological_operations.OperationType']"}),
+            'research_comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'responsible_town_planning_service': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'responsible_town_planning_service'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}),
+            'saisine_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.SaisineType']", 'null': 'True', 'blank': 'True'}),
+            'scientist': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'scientist'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}),
+            'total_developed_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'total_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'towns': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'file'", 'symmetrical': 'False', 'to': "orm['ishtar_common.Town']"}),
+            'year': ('django.db.models.fields.IntegerField', [], {'default': '2014'})
+        },
+        'archaeological_files.filebydepartment': {
+            'Meta': {'object_name': 'FileByDepartment', 'db_table': "'file_department'", 'managed': 'False'},
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.File']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'archaeological_files.filetype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'FileType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'archaeological_files.historicalfile': {
+            'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalFile'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'cached_label': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'cira_advised': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'classified_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'creation_date': ('django.db.models.fields.DateField', [], {'default': 'datetime.date.today'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'file_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'general_contractor_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}),
+            'imported_line': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'in_charge_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'internal_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}),
+            'mh_listing': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'mh_register': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'numeric_reference': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'organization_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'permit_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}),
+            'permit_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'protected_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'reception_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'reference_number': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'related_file_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'requested_operation_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'research_comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'responsible_town_planning_service_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'saisine_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'scientist_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'total_developed_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'total_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'year': ('django.db.models.fields.IntegerField', [], {'default': '2014'})
+        },
+        'archaeological_files.permittype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'PermitType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'archaeological_files.saisinetype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'SaisineType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'delay': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'archaeological_operations.operationtype': {
+            'Meta': {'ordering': "['-preventive', 'order', 'label']", 'object_name': 'OperationType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'preventive': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'ishtar_common.arrondissement': {
+            'Meta': {'object_name': 'Arrondissement'},
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.canton': {
+            'Meta': {'object_name': 'Canton'},
+            'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.department': {
+            'Meta': {'ordering': "['number']", 'object_name': 'Department'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'})
+        },
+        'ishtar_common.organization': {
+            'Meta': {'object_name': 'Organization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.organizationtype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.person': {
+            'Meta': {'object_name': 'Person'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.persontype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.town': {
+            'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'},
+            'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}),
+            'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}),
+            'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}),
+            'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['archaeological_files']
\ No newline at end of file
diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index 0c18af090..a1b42f722 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -147,6 +147,7 @@ class File(BaseHistorizedItem, OwnPerms, ValueGetter, ShortMenuItem,
     # <-- research archaeology
     cached_label = models.CharField(_(u"Cached name"), max_length=500,
                                     null=True, blank=True)
+    imported_line = models.TextField(_(u"Imported line"), null=True, blank=True)
     history = HistoricalRecords()
 
     class Meta:
diff --git a/archaeological_files/tests.py b/archaeological_files/tests.py
index b43967401..cf73b8726 100644
--- a/archaeological_files/tests.py
+++ b/archaeological_files/tests.py
@@ -27,7 +27,7 @@ from django.contrib.auth.models import User
 from django.test import TestCase
 
 from ishtar_common.models import PersonType
-import models
+from archaeological_files import models, data_importer
 
 class FileTest(TestCase):
     fixtures = [settings.ROOT_PATH + \
@@ -152,3 +152,7 @@ class FileTest(TestCase):
         self.assertTrue(data['records'] == 1)
         self.assertEqual(data['rows'][0]['internal_reference'], initial_ref)
 
+class ImporterTest(TestCase):
+    def testFormaters(self):
+        for formater in [data_importer.SurfaceFormater]:
+            formater().test()
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 86285e33e..1d768c6b0 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -17,139 +17,24 @@
 
 # See the file COPYING for details.
 
-"""
-# Usage exemple (extracted from simulabio application)
-
-class HarvestPlotImporter(Importer):
-    LINE_FORMAT = [
-               ImportFormater('name', Importer.get_unicode_formater(100)),
-               ImportFormater('plot_group_number',
-                              Importer.get_unicode_formater(3), required=False),
-               ImportFormater('geographical_area', unicode, required=False),
-               ImportFormater('soil_type', Importer.choices_check(SOIL_TYPE),
-                                                            required=False),
-               ImportFormater('cow_access', Importer.boolean_formater),
-               ImportFormater('area', Importer.float_formater),
-               ImportFormater('remark', unicode, required=False),
-               ImportFormater('diagnostic', Importer.boolean_formater),
-               ImportFormater('project', Importer.boolean_formater),
-               ImportFormater('harvest_n2', 'harvest_formater', required=False),
-               ImportFormater('harvest_n1', 'harvest_formater', required=False),
-               ImportFormater('harvest', 'harvest_formater'),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':1},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':2},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':3},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':4},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':5},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':6},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ]
-    OBJECT_CLS = HarvestPlots
-    UNICITY_KEYS = []
-
-    def __init__(self, study, skip_first_line=None):
-        # get the reference header
-        dct = {'separator':settings.CSV_DELIMITER}
-        dct['data'] = Harvest.objects.filter(available=True).all()
-        reference_file = render_to_string('simulabio/files/parcelles_ref.csv',
-                                          dct)
-        reference_header = unicode_csv_reader(
-                                [reference_file.split('\n')[0]]).next()
-        super(HarvestPlotImporter, self).__init__(
-                                           skip_first_line=skip_first_line,
-                                           reference_header=reference_header)
-        self.study = study
-        self.default_vals = {'study':self.study}
-
-    def harvest_formater(self, value):
-        value = value.strip()
-        if not value:
-            return
-        try:
-            harvest = Harvest.objects.get(name__iexact=value)
-        except ObjectDoesNotExist:
-            raise ValueError(_(u"\"%(value)s\" not in %(values)s") % {
-                'value':value,
-                'values':u", ".join([val.name
-                        for val in Harvest.objects.filter(available=True)])
-                })
-        hs, created = HarvestSettings.objects.get_or_create(study=self.study,
-                                                            harvest=harvest)
-        if created:
-            self.message = _(u"\"%(harvest)s\" has been added in your settings "
-                             u"don't forget to fill yields for this harvest.") \
-                             % {'harvest':harvest.name}
-        return hs
-
-class HarvestPlotsImportForm(forms.Form):
-    csv_file = forms.FileField(label=_(u"Plot list file (CSV)"))
-
-    def save(self, study):
-        csv_file = self.cleaned_data['csv_file']
-        importer = models.HarvestPlotImporter(study, skip_first_line=True)
-        # some softwares (at least Gnumeric) convert CSV file to utf-8 no matter
-        # what the CSV source encoding is
-        encodings = [settings.ENCODING, 'utf-8']
-        for encoding in encodings:
-            try:
-                importer.importation(unicode_csv_reader(
-                                [line.decode(encoding)
-                                 for line in csv_file.readlines()]))
-            except ImporterError, e:
-                if e.type == ImporterError.HEADER and encoding != encodings[-1]:
-                    csv_file.seek(0)
-                    continue
-                return 0, [[0, 0, e.msg]], []
-            except UnicodeDecodeError, e:
-                return 0, [[0, 0, Importer.ERRORS['header_check']]], []
-            break
-        return importer.number_imported, importer.errors, importer.messages
-"""
-
 import copy, csv, datetime, logging, sys
 from tempfile import NamedTemporaryFile
 
 from django.contrib.auth.models import User
-from django.db import DatabaseError
+from django.db import DatabaseError, IntegrityError
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
 from ishtar_common.unicode_csv import UnicodeWriter
 
+NEW_LINE_BREAK = '#####@@@#####'
+
 class ImportFormater(object):
     def __init__(self, field_name, formater=None, required=True, through=None,
                 through_key=None, through_dict=None, through_unicity_keys=None,
                 duplicate_field=None, regexp=None, regexp_formater_args=[],
-                reverse_for_test=None, comment=""):
+                reverse_for_test=None, force_value=None, post_processing=False,
+                concat=False, comment=""):
         self.field_name = field_name
         self.formater = formater
         self.required = required
@@ -161,6 +46,12 @@ class ImportFormater(object):
         self.regexp = regexp
         self.regexp_formater_args = regexp_formater_args
         self.reverse_for_test = reverse_for_test
+        # write this value even if a value exists
+        self.force_value = force_value
+        # post process after import
+        self.post_processing = post_processing
+        # concatenate with existing value
+        self.concat = concat
         self.comment = comment
 
     def __unicode__(self):
@@ -178,7 +69,11 @@ class ImportFormater(object):
         except TypeError:
             lst = [self.formater]
         for formater in lst:
-            formater.check(vals)
+            if formater:
+                formater.check(vals)
+
+    def post_process(self, obj, context, value, owner=None):
+        raise NotImplemented()
 
 class ImporterError(Exception):
     STANDARD = 'S'
@@ -186,6 +81,7 @@ class ImporterError(Exception):
     def __init__(self, message, type='S'):
         self.msg = message
         self.type = type
+
     def __str__(self):
         return self.msg
 
@@ -214,6 +110,7 @@ class UnicodeFormater(Formater):
                     value = value[1:]
                 if value.endswith(","):
                     value = value[:-1]
+                value = value.replace(", , ", ", ")
         except UnicodeDecodeError:
             return
         if len(value) > self.max_length:
@@ -243,19 +140,59 @@ class FloatFormater(Formater):
             raise ValueError(_(u"\"%(value)s\" is not a float") % {
                                                                  'value':value})
 
+class YearFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            value = int(value)
+            assert value > 0 and value < (datetime.date.today().year + 30)
+        except (ValueError, AssertionError):
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                                 'value':value})
+
+class YearNoFuturFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            value = int(value)
+            assert value > 0 and value < (datetime.date.today().year)
+        except (ValueError, AssertionError):
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                                 'value':value})
+
+class IntegerFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            return int(value)
+        except ValueError:
+            raise ValueError(_(u"\"%(value)s\" is not an integer") % {
+                                                                 'value':value})
+
 class StrChoiceFormater(Formater):
-    def __init__(self, choices, strict=False, equiv_dict={}, cli=False):
-        self.choices = choices
+    def __init__(self, choices, strict=False, equiv_dict={}, model=None,
+                 cli=False):
+        self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
         self.cli = cli
+        self.model = model
         self.missings = set()
         for key, value in self.choices:
             value = unicode(value)
             if not self.strict:
                 value = slugify(value)
             if value not in self.equiv_dict:
-                self.equiv_dict[value] = key
+                v = key
+                if model and v:
+                    v = model.objects.get(pk=v)
+                self.equiv_dict[value] = v
 
     def prepare(self, value):
         return unicode(value).strip()
@@ -283,7 +220,10 @@ class StrChoiceFormater(Formater):
                     pass
             res -= 1
             if res < len(self.choices):
-                self.equiv_dict[value] = self.choices[res]
+                v = self.choices[res][0]
+                if self.model and v:
+                    v = self.model.objects.get(pk=v)
+                self.equiv_dict[value] = v
             else:
                 self.equiv_dict[value] = None
 
@@ -294,11 +234,24 @@ class StrChoiceFormater(Formater):
         if value in self.equiv_dict:
             return self.equiv_dict[value]
 
+class DateFormater(Formater):
+    def __init__(self, date_format="%d/%m/%Y"):
+        self.date_format = date_format
+
+    def format(self, value):
+        value = value.strip()
+        try:
+            return datetime.datetime.strptime(value, self.date_format).date()
+        except:
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                           'value':value})
+
 logger = logging.getLogger(__name__)
 
 class Importer(object):
     LINE_FORMAT = []
     OBJECT_CLS = None
+    IMPORTED_LINE_FIELD = None
     UNICITY_KEYS = []
     DEFAULTS = {}
     ERRORS = {
@@ -356,23 +309,264 @@ class Importer(object):
                     vals.append([])
                 vals[idx_col].append(val)
         for idx, formater in enumerate(self.line_format):
-            formater.init(vals[idx])
+            if formater:
+                formater.init(vals[idx])
         self._initialized = True
 
     def importation(self, table):
+        self.validity_file = None
         if not self._initialized:
             self.initialize(table)
         if self.check_validity:
             with NamedTemporaryFile(delete=False) as validity_file:
-                print(validity_file.name)
-                validity_file_writer = UnicodeWriter(validity_file,
+                self.validity_file = UnicodeWriter(validity_file,
                                 delimiter=',', quotechar='"',
                                 quoting=csv.QUOTE_MINIMAL)
-                self._importation(table, validity_file_writer)
+                self._importation(table)
         else:
             self._importation(table)
 
-    def _importation(self, table, validity_file=None):
+    @classmethod
+    def _field_name_to_data_dict(cls, field_name, value, data,
+                                 force_value=False, concat=False):
+        field_names = field_name
+        if type(field_names) not in (list, tuple):
+            field_names = [field_name]
+        for field_name in field_names:
+            keys = field_name.split('__')
+            current_data = data
+            for idx, key in enumerate(keys):
+                if idx == (len(keys) - 1): # last
+                    if concat:
+                        if not value:
+                            value = ""
+                        current_data[key] = (current_data[key] + u"\n") or u""\
+                                            + value
+                    elif force_value and value:
+                        current_data[key] = value
+                    elif key not in current_data or not current_data[key]:
+                        current_data[key] = value
+                elif key not in current_data:
+                    current_data[key] = {}
+                current_data = current_data[key]
+        return data
+
+    def _importation(self, table):
+        table = list(table)
+        if not table or not table[0]:
+            raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
+        if self.check_col_num and len(table[0]) > len(self.line_format):
+            raise ImporterError(self.ERRORS['too_many_cols'] % {
+                     'user_col':len(table[0]), 'ref_col':len(self.line_format)})
+        self.errors = []
+        self.messages = []
+        self.number_imported = 0
+        # index of the last required column
+        for idx_last_col, formater in enumerate(reversed(self.line_format)):
+            if formater and formater.required:
+                break
+        else:
+            idx_last_col += 1
+        # min col number to be filled
+        self.min_col_number = len(self.line_format) - idx_last_col
+        # check the conformity with the reference header
+        if self.reference_header and \
+           self.skip_first_line and \
+           self.reference_header != table[0]:
+            raise ImporterError(self.ERRORS['header_check'],
+                                type=ImporterError.HEADER)
+        self.now = datetime.datetime.now()
+        for idx_line, line in enumerate(table):
+            self._line_processing(idx_line, line)
+
+    def _line_processing(self, idx_line, line):
+        if (self.skip_first_line and not idx_line):
+            if self.validity_file:
+                self.validity_file.writerow(line)
+            return
+        if not line:
+            if self.validity_file:
+                self.validity_file.writerow([])
+            return
+        self._throughs = [] # list of (formater, value)
+        self._post_processing = [] # list of (formater, value)
+        data = {}
+
+        # keep in database the raw line for testing purpose
+        if self.IMPORTED_LINE_FIELD:
+            output = io.StringIO()
+            writer = csv.writer(output)
+            writer.writerow(line)
+            data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+
+        n = datetime.datetime.now()
+        logger.debug('%s - Processing line %d' % (unicode(n-self.now), idx_line))
+        self.now = n
+        n2 = n
+        self.c_errors = False
+        c_row = []
+        for idx_col, val in enumerate(line):
+            try:
+                self._row_processing(c_row, idx_col, idx_line, val, data)
+            except:
+                pass
+
+        if self.validity_file:
+            self.validity_file.writerow(c_row)
+        if not self.c_errors and (idx_col + 1) < self.min_col_number:
+            self.c_errors = True
+            self.errors.append((idx_line+1, idx_col+1,
+                          self.ERRORS['not_enough_cols'] % self.min_col_number))
+        if self.c_errors:
+            return
+        n = datetime.datetime.now()
+        logger.debug('* %s - Cols read' % (unicode(n-n2)))
+        n2 = n
+        if self.test:
+            return
+        # manage unicity of items (mainly for updates)
+        self.number_imported += 1
+        if self.UNICITY_KEYS:
+            data['defaults'] = {}
+            for k in data.keys():
+                if k not in self.UNICITY_KEYS \
+                   and k != 'defaults':
+                    data['defaults'][k] = data.pop(k)
+
+        data['history_modifier'] = self.history_modifier
+        obj, created = self.get_object(self.OBJECT_CLS, data)
+
+        if not created and 'defaults' in data:
+            for k in data['defaults']:
+                setattr(obj, k, data['defaults'][k])
+            obj.save()
+        n = datetime.datetime.now()
+        logger.debug('* %s - Item saved' % (unicode(n-n2)))
+        n2 = n
+        for formater, value in self._throughs:
+            n = datetime.datetime.now()
+            logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
+                                                    formater.field_name))
+            n2 = n
+            data = {}
+            if formater.through_dict:
+                data = formater.through_dict.copy()
+            if formater.through_key:
+                data[formater.through_key] = obj
+            data[formater.field_name] = value
+            through_cls = formater.through
+            if formater.through_unicity_keys:
+                data['defaults'] = {}
+                for k in data.keys():
+                    if k not in formater.through_unicity_keys \
+                       and k != 'defaults':
+                        data['defaults'][k] = data.pop(k)
+            t_obj, created = through_cls.objects.get_or_create(**data)
+            if not created and 'defaults' in data:
+                for k in data['defaults']:
+                    setattr(t_obj, k, data['defaults'][k])
+                t_obj.save()
+
+        for formater, val in self._post_processing:
+            formater.post_process(obj, data, val, owner=self.history_modifier)
+
+    def _row_processing(self, c_row, idx_col, idx_line, val, data):
+        if idx_col >= len(self.line_format):
+            return
+
+        formater = self.line_format[idx_col]
+
+        if formater and formater.post_processing:
+            self._post_processing.append((formater, val))
+
+        if not formater or not formater.field_name:
+            if self.validity_file:
+                c_row.append(val)
+            return
+
+        # regex management
+        if formater.regexp:
+            # multiline regexp is a mess...
+            val = val.replace('\n', NEW_LINE_BREAK)
+            match = formater.regexp.match(val)
+            if not match:
+                if formater.required:
+                    self.errors.append((idx_line+1, idx_col+1,
+                                       self.ERRORS['value_required']))
+                elif not val.strip():
+                    c_row.append("")
+                    return
+                self.c_errors = True
+                val = val.replace(NEW_LINE_BREAK, '\n')
+                self.errors.append((idx_line+1, idx_col+1,
+                         unicode(self.ERRORS['regex_not_match']) + val))
+                c_row.append("")
+                return
+            val_group = [v.replace(NEW_LINE_BREAK, '\n')
+                         for v in match.groups()]
+        else:
+            val_group = [val]
+
+        c_values = []
+        for idx_v, v in enumerate(val_group):
+            self.message = ''
+            func = formater.formater
+            if type(func) in (list, tuple):
+                func = func[idx_v]
+            if not callable(func) and type(func) in (unicode, str):
+                func = getattr(self, func)
+            value = None
+
+            try:
+                if formater.regexp_formater_args:
+                    args = []
+                    for idx in formater.regexp_formater_args[idx_v]:
+                        args.append(val_group[idx])
+                    value = func.format(*args)
+                else:
+                    value = func.format(v)
+            except ValueError, e:
+                if formater.required:
+                    self.c_errors = True
+                self.errors.append((idx_line+1, idx_col+1, e.message))
+                c_values.append(None)
+                return
+
+            if self.message:
+                self.messages.append(self.message)
+            c_values.append(value)
+
+            if value == None:
+                if formater.required:
+                    self.c_errors = True
+                    self.errors.append((idx_line+1, idx_col+1,
+                                       self.ERRORS['value_required']))
+                return
+
+            field_name = formater.field_name
+            if type(field_name) in (list, tuple):
+                field_name = field_name[idx_v]
+            field_names = [field_name]
+            if formater.duplicate_field:
+                duplicate_field = formater.duplicate_field
+                if type(duplicate_field) in (list, tuple):
+                    duplicate_field = duplicate_field[idx_v]
+                field_names += [duplicate_field]
+
+            if formater.through:
+                self._throughs.append((formater, value))
+            else:
+                for field_name in field_names:
+                    self._field_name_to_data_dict(field_name,
+                                              value, data, formater.force_value)
+        if formater.reverse_for_test:
+            c_row.append(formater.reverse_for_test(**c_values))
+        else:
+            c_row.append(unicode(c_values))
+
+
+    """
+    def _importation(self, table):
         table = list(table)
         if not table or not table[0]:
             raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
@@ -398,33 +592,46 @@ class Importer(object):
                                 type=ImporterError.HEADER)
         now = datetime.datetime.now()
         for idx_line, line in enumerate(table):
+            #self._line_processing()
+
             if (self.skip_first_line and not idx_line):
-                if validity_file:
-                    validity_file.writerow(line)
+                if self.validity_file:
+                    self.validity_file.writerow(line)
                 continue
             if not line:
-                if validity_file:
-                    validity_file.writerow([])
+                if self.validity_file:
+                    self.validity_file.writerow([])
                 continue
-            throughs = [] # list of (formater, value)
+            self.throughs = [] # list of (formater, value)
+            self.post_processing = [] # list of (formater, value)
             data = {}
+
+            # keep in database the raw line for testing purpose
+            if self.IMPORTED_LINE_FIELD:
+                output = io.StringIO()
+                writer = csv.writer(output)
+                writer.writerow(line)
+                data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+
             n = datetime.datetime.now()
             logger.debug('%s - Processing line %d' % (unicode(n-now), idx_line))
             now = n
             n2 = n
-            c_errors = False
+            self.c_errors = False
             c_row = []
             for idx_col, val in enumerate(line):
+                #self._row_processing(self, c_row, idx_col, val):
+
                 if idx_col >= len(self.line_format):
                     break
                 formater = self.line_format[idx_col]
                 if not formater.field_name:
-                    if validity_file:
+                    if self.validity_file:
                         c_row.append(val)
                     continue
                 if formater.regexp:
                     # multiline regexp is a mess...
-                    val = val.replace('\n', '######???#####')
+                    val = val.replace('\n', NEW_LINE_BREAK)
                     match = formater.regexp.match(val)
                     if not match:
                         if formater.required:
@@ -434,12 +641,12 @@ class Importer(object):
                             c_row.append("")
                             continue
                         c_errors = True
-                        val = val.replace('######???#####', '\n')
+                        val = val.replace(NEW_LINE_BREAK, '\n')
                         self.errors.append((idx_line+1, idx_col+1,
                                  unicode(self.ERRORS['regex_not_match']) + val))
                         c_row.append("")
                         continue
-                    val_group = [v.replace('######???#####', '\n')
+                    val_group = [v.replace(NEW_LINE_BREAK, '\n')
                                  for v in match.groups()]
                 else:
                     val_group = [val]
@@ -483,29 +690,26 @@ class Importer(object):
                         if type(duplicate_field) in (list, tuple):
                             duplicate_field = duplicate_field[idx_v]
                         field_names += [duplicate_field]
-                    if not formater.through:
-                        for field_name in field_names:
-                            keys = field_name.split('__')
-                            current_data = data
-                            for idx, key in enumerate(keys):
-                                if idx == (len(keys) - 1): # last
-                                    current_data[key] = value
-                                elif key not in current_data:
-                                    current_data[key] = {}
-                                current_data = current_data[key]
-                    else:
+
+
+                    if formater.through:
                         throughs.append((formater, value))
+                    else:
+                        for field_name in field_names:
+                            self._field_name_to_data_dict(field_name,
+                                                          value, data)
                 if formater.reverse_for_test:
                     c_row.append(formater.reverse_for_test(**c_values))
                 else:
                     c_row.append(unicode(c_values))
-            if validity_file:
-                validity_file.writerow(c_row)
-            if not c_errors and (idx_col + 1) < min_col_number:
-                c_errors = True
+
+            if self.validity_file:
+                self.validity_file.writerow(c_row)
+            if not self.c_errors and (idx_col + 1) < min_col_number:
+                self.c_errors = True
                 self.errors.append((idx_line+1, idx_col+1,
                                self.ERRORS['not_enough_cols'] % min_col_number))
-            if c_errors:
+            if self.c_errors:
                 continue
             n = datetime.datetime.now()
             logger.debug('* %s - Cols read' % (unicode(n-n2)))
@@ -530,7 +734,7 @@ class Importer(object):
             n = datetime.datetime.now()
             logger.debug('* %s - Item saved' % (unicode(n-n2)))
             n2 = n
-            for formater, value in throughs:
+            for formater, value in self.throughs:
                 n = datetime.datetime.now()
                 logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
                                                         formater.field_name))
@@ -553,6 +757,7 @@ class Importer(object):
                     for k in data['defaults']:
                         setattr(t_obj, k, data['defaults'][k])
                     t_obj.save()
+    """
 
     def get_object(self, cls, data, path=[]):
         m2ms = []
@@ -563,7 +768,8 @@ class Importer(object):
                     continue
                 field_object, model, direct, m2m = \
                                     cls._meta.get_field_by_name(attribute)
-                if field_object.rel and type(data[attribute]) == dict:
+                if hasattr(field_object, 'rel') and field_object.rel and \
+                   type(data[attribute]) == dict:
                     c_path.append(attribute)
                     # put history_modifier for every created item
                     data[attribute]['history_modifier'] = \
@@ -577,15 +783,26 @@ class Importer(object):
                 for k in self._defaults[path]:
                     if k not in data or not data[k]:
                         data[k] = self._defaults[path][k]
-            obj, created = cls.objects.get_or_create(**data)
-            for attr, value in m2ms:
-                getattr(obj, attr).add(value)
+
+            # filter default values
+            create_dict = copy.deepcopy(data)
+            for k in create_dict.keys():
+                if type(create_dict[k]) == dict:
+                    create_dict.pop(k)
+
+            try:
+                obj, created = cls.objects.get_or_create(**create_dict)
+                for attr, value in m2ms:
+                    getattr(obj, attr).add(value)
+            except IntegrityError:
+                raise ImporterError("Erreur d'import %s, contexte : %s" \
+                                % (unicode(cls), unicode(data)))
             return obj, created
         return data
 
     def get_csv_errors(self):
         for line, col, error in self.errors:
-            print '"%d","%d","%s"' % (line, col, unicode(error))
+            print('"%d","%d","%s"' % (line, col, unicode(error)))
 
     @classmethod
     def choices_check(cls, choices):
diff --git a/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py b/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py
new file mode 100644
index 000000000..015956610
--- /dev/null
+++ b/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py
@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Person.raw_name'
+        db.add_column('ishtar_common_person', 'raw_name',
+                      self.gf('django.db.models.fields.CharField')(max_length=300, null=True, blank=True),
+                      keep_default=False)
+
+
+        # Changing field 'Person.name'
+        db.alter_column('ishtar_common_person', 'name', self.gf('django.db.models.fields.CharField')(max_length=200, null=True))
+
+    def backwards(self, orm):
+        # Deleting field 'Person.raw_name'
+        db.delete_column('ishtar_common_person', 'raw_name')
+
+
+        # User chose to not deal with backwards NULL issues for 'Person.name'
+        raise RuntimeError("Cannot reverse this migration. 'Person.name' and its values cannot be restored.")
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'ishtar_common.arrondissement': {
+            'Meta': {'object_name': 'Arrondissement'},
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.author': {
+            'Meta': {'object_name': 'Author'},
+            'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"})
+        },
+        'ishtar_common.authortype': {
+            'Meta': {'object_name': 'AuthorType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.canton': {
+            'Meta': {'object_name': 'Canton'},
+            'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.department': {
+            'Meta': {'ordering': "['number']", 'object_name': 'Department'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'})
+        },
+        'ishtar_common.documenttemplate': {
+            'Meta': {'ordering': "['associated_object_name']", 'object_name': 'DocumentTemplate'},
+            'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'})
+        },
+        'ishtar_common.globalvar': {
+            'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'},
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}),
+            'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.historicalorganization': {
+            'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.ishtaruser': {
+            'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']},
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'ishtar_common.organization': {
+            'Meta': {'object_name': 'Organization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.organizationtype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.person': {
+            'Meta': {'object_name': 'Person'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.persontype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.sourcetype': {
+            'Meta': {'object_name': 'SourceType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.town': {
+            'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'},
+            'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}),
+            'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}),
+            'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}),
+            'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['ishtar_common']
\ No newline at end of file
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index f77c4f980..a96b24840 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -875,7 +875,10 @@ class Person(Address, OwnPerms, ValueGetter) :
     title = models.CharField(_(u"Title"), max_length=2, choices=TYPE)
     surname = models.CharField(_(u"Surname"), max_length=50, blank=True,
                                null=True)
-    name = models.CharField(_(u"Name"), max_length=200)
+    name = models.CharField(_(u"Name"), max_length=200, blank=True,
+                               null=True)
+    raw_name = models.CharField(_(u"Raw name"), max_length=300, blank=True,
+                               null=True)
     person_types = models.ManyToManyField(PersonType, verbose_name=_(u"Types"))
     attached_to = models.ForeignKey('Organization', related_name='members',
                        on_delete=models.SET_NULL,
@@ -896,6 +899,8 @@ class Person(Address, OwnPerms, ValueGetter) :
         values = [unicode(getattr(self, attr))
                    for attr in ('surname', 'name')
                                if getattr(self, attr)]
+        if not values:
+            values = [self.raw_name]
         if self.attached_to:
             values.append(u"- " + unicode(self.attached_to))
         return u" ".join(values)