Import: add management of link between user data and db data

4 files changed, 489 insertions, 32 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c2924a541..194a9a5fa 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -88,6 +88,9 @@ class ImporterError(Exception):
         return self.msg
 
 class Formater(object):
+    def __init__(self, *args, **kwargs):
+        self.db_target = kwargs.get('db_target', None)
+
     def format(self, value):
         return value
 
@@ -95,8 +98,10 @@ class Formater(object):
         return
 
 class UnicodeFormater(Formater):
-    def __init__(self, max_length, clean=False, re_filter=None, notnull=False):
+    def __init__(self, max_length, clean=False, re_filter=None, notnull=False,
+                 db_target=None):
         self.max_length = max_length
+        self.db_target = db_target
         self.clean = clean
         self.re_filter = re_filter
         self.notnull = notnull
@@ -182,12 +187,13 @@ class IntegerFormater(Formater):
 
 class StrChoiceFormater(Formater):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
-                 cli=False, many_split=''):
+                 cli=False, many_split='', db_target=None):
         self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
         self.cli = cli
         self.model = model
+        self.db_target = db_target
         self.create = False
         self.missings = set()
         self.many_split = many_split
@@ -200,6 +206,17 @@ class StrChoiceFormater(Formater):
                 if model and v:
                     v = model.objects.get(pk=v)
                 self.equiv_dict[value] = v
+        if self.db_target:
+            for target_key in self.db_target.keys.filter(is_set=True).all():
+                value = target_key.value
+                if not self.strict:
+                    value = slugify(value)
+                if value in self.equiv_dict:
+                    continue
+                v = target_key.key
+                if model and v:
+                    v = model.objects.get(pk=v)
+                self.equiv_dict[value] = v
 
     def prepare(self, value):
         return unicode(value).strip()
@@ -257,6 +274,13 @@ class StrChoiceFormater(Formater):
                                      unicode(self.equiv_dict[value])))
             else:
                 self.equiv_dict[value] = None
+        if output == 'db' and self.db_target:
+            for missing in missings:
+                try:
+                    q = {'target':self.db_target, 'value':missing}
+                    models.TargetKey.objects.create(**q)
+                except IntegrityError:
+                    pass
 
     def new(self, value):
         return
@@ -272,12 +296,14 @@ class StrChoiceFormater(Formater):
             return self.equiv_dict[value]
 
 class TypeFormater(StrChoiceFormater):
-    def __init__(self, model, cli=False, defaults={}, many_split=False):
+    def __init__(self, model, cli=False, defaults={}, many_split=False,
+                 db_target=None):
         self.create = True
         self.strict = False
         self.model = model
         self.defaults = defaults
         self.many_split = many_split
+        self.db_target = db_target
         self.missings = set()
         self.equiv_dict, self.choices = {}, []
         for item in model.objects.all():
@@ -304,8 +330,9 @@ class TypeFormater(StrChoiceFormater):
         return self.model.objects.create(**values)
 
 class DateFormater(Formater):
-    def __init__(self, date_format="%d/%m/%Y"):
+    def __init__(self, date_format="%d/%m/%Y", db_target=None):
         self.date_format = date_format
+        self.db_target = db_target
 
     def format(self, value):
         value = value.strip()
@@ -318,11 +345,26 @@ class DateFormater(Formater):
                                                            'value':value})
 
 class StrToBoolean(Formater):
-    def __init__(self, choices={}, cli=False, strict=False):
+    def __init__(self, choices={}, cli=False, strict=False, db_target=None):
         self.dct = copy.copy(choices)
         self.cli = cli
         self.strict= strict
+        self.db_target = db_target
         self.missings = set()
+        if self.db_target:
+            for target_key in self.db_target.keys.filter(is_set=True).all():
+                value = target_key.value
+                value = self.prepare(value)
+                if value in self.dct:
+                    continue
+                v = target_key.key
+                if v in ('False', '0'):
+                    v = False
+                elif v:
+                    v = True
+                else:
+                    v = None
+                self.dct[value] = v
 
     def prepare(self, value):
         value = unicode(value).strip()
@@ -359,6 +401,13 @@ class StrToBoolean(Formater):
                 self.dct[value] = False
             else:
                 self.dct[value] = None
+        if output == 'db' and self.db_target:
+            for missing in missings:
+                try:
+                    q = {'target':self.db_target, 'value':missing}
+                    models.TargetKey.objects.create(**q)
+                except IntegrityError:
+                    pass
 
     def format(self, value):
         value = self.prepare(value)
@@ -388,8 +437,9 @@ class Importer(object):
         }
 
     def __init__(self, skip_lines=0, reference_header=None,
-                 check_col_num=False, test=False, check_validity=True,
-                 history_modifier=None, output='silent'):
+             check_col_num=False, test=False, check_validity=True,
+             history_modifier=None, output='silent',
+             importer_instance=None):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
@@ -408,14 +458,18 @@ class Importer(object):
         self.check_col_num = check_col_num
         self.check_validity = check_validity
         self.line_format = copy.copy(self.LINE_FORMAT)
+        self.importer_instance = importer_instance
         self._initialized = False
         self._defaults = self.DEFAULTS.copy()
         self.history_modifier = history_modifier
         self.output = output
         if not self.history_modifier:
-            # get the first admin
-            self.history_modifier = User.objects.filter(is_superuser=True
-                                                ).order_by('pk')[0]
+            if self.importer_instance:
+                self.history_modifier = self.importer_instance.user
+            else:
+                # import made by the CLI: get the first admin
+                self.history_modifier = User.objects.filter(
+                                is_superuser=True).order_by('pk')[0]
 
     def initialize(self, table, output='silent'):
         """
diff --git a/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py b/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py
new file mode 100644
index 000000000..b4752a48e
--- /dev/null
+++ b/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py
@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'TargetKey'
+        db.create_table('ishtar_common_targetkey', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('target', self.gf('django.db.models.fields.related.ForeignKey')(related_name='keys', to=orm['ishtar_common.ImporterColumn'])),
+            ('key', self.gf('django.db.models.fields.TextField')(null=True, blank=True)),
+            ('value', self.gf('django.db.models.fields.TextField')()),
+            ('is_set', self.gf('django.db.models.fields.BooleanField')(default=False)),
+        ))
+        db.send_create_signal('ishtar_common', ['TargetKey'])
+
+        # Adding unique constraint on 'TargetKey', fields ['target', 'value']
+        db.create_unique('ishtar_common_targetkey', ['target_id', 'value'])
+
+        # Adding field 'Import.skip_lines'
+        db.add_column('ishtar_common_import', 'skip_lines',
+                      self.gf('django.db.models.fields.IntegerField')(default=1),
+                      keep_default=False)
+
+
+        # Changing field 'Import.creation_date'
+        db.alter_column('ishtar_common_import', 'creation_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, null=True))
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'TargetKey', fields ['target', 'value']
+        db.delete_unique('ishtar_common_targetkey', ['target_id', 'value'])
+
+        # Deleting model 'TargetKey'
+        db.delete_table('ishtar_common_targetkey')
+
+        # Deleting field 'Import.skip_lines'
+        db.delete_column('ishtar_common_import', 'skip_lines')
+
+
+        # Changing field 'Import.creation_date'
+        db.alter_column('ishtar_common_import', 'creation_date', self.gf('django.db.models.fields.DateTimeField')(null=True))
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'ishtar_common.arrondissement': {
+            'Meta': {'object_name': 'Arrondissement'},
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.author': {
+            'Meta': {'object_name': 'Author'},
+            'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"})
+        },
+        'ishtar_common.authortype': {
+            'Meta': {'object_name': 'AuthorType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.canton': {
+            'Meta': {'object_name': 'Canton'},
+            'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.department': {
+            'Meta': {'ordering': "['number']", 'object_name': 'Department'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'})
+        },
+        'ishtar_common.documenttemplate': {
+            'Meta': {'ordering': "['associated_object_name', 'name']", 'object_name': 'DocumentTemplate'},
+            'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'})
+        },
+        'ishtar_common.format': {
+            'Meta': {'object_name': 'Format'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.formatertype': {
+            'Meta': {'unique_together': "(('formater_type', 'options', 'many_split'),)", 'object_name': 'FormaterType'},
+            'formater_type': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'many_split': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'options': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.globalvar': {
+            'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'},
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}),
+            'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.historicalorganization': {
+            'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.import': {
+            'Meta': {'object_name': 'Import'},
+            'creation_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}),
+            'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'error_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imported_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.ImporterType']"}),
+            'result_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'seconds_remaining': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'skip_lines': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'state': ('django.db.models.fields.CharField', [], {'default': "'C'", 'max_length': '2'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']"})
+        },
+        'ishtar_common.importercolumn': {
+            'Meta': {'object_name': 'ImporterColumn'},
+            'col_number': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'columns'", 'to': "orm['ishtar_common.ImporterType']"}),
+            'regexp_pre_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}),
+            'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'ishtar_common.importerdefault': {
+            'Meta': {'object_name': 'ImporterDefault'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'defaults'", 'to': "orm['ishtar_common.ImporterType']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.importerdefaultvalues': {
+            'Meta': {'object_name': 'ImporterDefaultValues'},
+            'default_target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'default_values'", 'to': "orm['ishtar_common.ImporterDefault']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'value': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.importerduplicatefield': {
+            'Meta': {'object_name': 'ImporterDuplicateField'},
+            'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_fields'", 'to': "orm['ishtar_common.ImporterColumn']"}),
+            'field_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'ishtar_common.importertype': {
+            'Meta': {'object_name': 'ImporterType'},
+            'associated_models': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_template': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.importtarget': {
+            'Meta': {'object_name': 'ImportTarget'},
+            'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'targets'", 'to': "orm['ishtar_common.ImporterColumn']"}),
+            'formater_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.FormaterType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'regexp_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.ishtaruser': {
+            'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']},
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'ishtar_common.itemkey': {
+            'Meta': {'object_name': 'ItemKey'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {})
+        },
+        'ishtar_common.organization': {
+            'Meta': {'object_name': 'Organization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}),
+            'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.organizationtype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.person': {
+            'Meta': {'object_name': 'Person'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '2', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.persontype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.regexp': {
+            'Meta': {'object_name': 'Regexp'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'regexp': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.sourcetype': {
+            'Meta': {'object_name': 'SourceType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.supporttype': {
+            'Meta': {'object_name': 'SupportType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'ishtar_common.targetkey': {
+            'Meta': {'unique_together': "(('target', 'value'),)", 'object_name': 'TargetKey'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_set': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'key': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'keys'", 'to': "orm['ishtar_common.ImporterColumn']"}),
+            'value': ('django.db.models.fields.TextField', [], {})
+        },
+        'ishtar_common.town': {
+            'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'},
+            'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}),
+            'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}),
+            'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}),
+            'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['ishtar_common']
\ No newline at end of file
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index f2193eae2..1f059f2f5 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -28,6 +28,7 @@ from importlib import import_module
 import os
 import re
 import tempfile
+import unicodecsv
 
 from django.conf import settings
 from django.core.cache import cache
@@ -955,13 +956,6 @@ class OrganizationType(GeneralType):
         verbose_name_plural = _(u"Organization types")
         ordering = ('label',)
 
-IMPORT_STATE = (("C", _(u"Created")),
-                ("AP", _(u"Analyse in progress")),
-                ("A", _(u"Analysed")),
-                ("P", _(u"Import pending")),
-                ("IP", _(u"Import in progress")),
-                ("F", _(u"Finished")))
-
 MODELS = [
           ('archaeological_operations.models.Operation', _(u"Operation")),
           ('archaeological_operations.models.Parcel', _(u"Parcels")),
@@ -985,6 +979,9 @@ def get_model_fields(model):
     return fields
 
 class ImporterType(models.Model):
+    """
+    Description of a table to be mapped with ishtar database
+    """
     name = models.CharField(_(u"Name"), blank=True, null=True,
                             max_length=100)
     description = models.CharField(_(u"Description"), blank=True, null=True,
@@ -1022,7 +1019,7 @@ class ImporterType(models.Model):
                 LINE_FORMAT.append(None)
                 continue
             for target in column.targets.all():
-                ft = target.formater_type.get_formater_type()
+                ft = target.formater_type.get_formater_type(target)
                 if not ft:
                     continue
                 formater_types.append(ft)
@@ -1032,8 +1029,9 @@ class ImporterType(models.Model):
                 formater_kwargs['regexp'] = re.compile(
                                                 column.regexp_pre_filter.regexp)
             formater_kwargs['duplicate_fields'] = [field.field_name
-                                        for field in column.duplicate_fields.all()]
-            formater = ImportFormater(targets, formater_types, **formater_kwargs)
+                                     for field in column.duplicate_fields.all()]
+            formater = ImportFormater(targets, formater_types,
+                                      **formater_kwargs)
             LINE_FORMAT.append(formater)
         args = {'OBJECT_CLS':OBJECT_CLS, 'DESC':self.description,
                 'DEFAULTS':DEFAULTS, 'LINE_FORMAT':LINE_FORMAT}
@@ -1041,6 +1039,9 @@ class ImporterType(models.Model):
         return newclass
 
 class ImporterDefault(models.Model):
+    """
+    Targets of default values in an import
+    """
     importer_type = models.ForeignKey(ImporterType, related_name='defaults')
     target = models.CharField(u"Target", max_length=500)
     class Meta:
@@ -1071,6 +1072,9 @@ class ImporterDefault(models.Model):
         return values
 
 class ImporterDefaultValues(models.Model):
+    """
+    Default values in an import
+    """
     default_target = models.ForeignKey(ImporterDefault,
                                        related_name='default_values')
     target = models.CharField(u"Target", max_length=500)
@@ -1096,6 +1100,9 @@ class ImporterDefaultValues(models.Model):
         return ""
 
 class ImporterColumn(models.Model):
+    """
+    Import file column description
+    """
     importer_type = models.ForeignKey(ImporterType, related_name='columns')
     col_number = models.IntegerField(_(u"Column number"), default=1)
     regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True)
@@ -1105,6 +1112,9 @@ class ImporterColumn(models.Model):
         verbose_name_plural = _(u"Importer - Columns")
 
 class ImporterDuplicateField(models.Model):
+    """
+    Direct copy of result in other fields
+    """
     column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields')
     field_name = models.CharField(_(u"Field name"), blank=True, null=True,
                                   max_length=200)
@@ -1124,6 +1134,9 @@ class Regexp(models.Model):
 IMPORTER_TYPES = []
 
 class ImportTarget(models.Model):
+    """
+    Ishtar database target for a column
+    """
     column = models.ForeignKey(ImporterColumn, related_name='targets')
     target = models.CharField(u"Target", max_length=500)
     regexp_filter = models.ForeignKey("Regexp", blank=True, null=True)
@@ -1132,6 +1145,26 @@ class ImportTarget(models.Model):
         verbose_name = _(u"Importer - Target")
         verbose_name_plural = _(u"Importer - Targets")
 
+    def __unicode__(self):
+        return u" - ".join([unicode(self.column), self.target[:50]])
+
+class TargetKey(models.Model):
+    """
+    User's link between import source and ishtar database.
+    Also temporary used for GeneralType to point missing link before adding
+    them in ItemKey table
+    """
+    target = models.ForeignKey(ImporterColumn, related_name='keys')
+    key = models.TextField(_(u"Key"), blank=True, null=True)
+    value = models.TextField(_(u"Value"))
+    is_set = models.BooleanField(_(u"Is set"), default=False)
+
+    class Meta:
+        unique_together = ('target', 'value')
+
+    def __unicode__(self):
+        return u" - ".join([unicode(self.target), self.key[:50]])
+
 TARGET_MODELS = [
     ('OrganizationType', _(u"Organization type")),
     ('SourceType', _(u"Source type")),
@@ -1190,10 +1223,10 @@ class FormaterType(models.Model):
         if self.format_type in IMPORTER_TYPES_CHOICES:
             return IMPORTER_TYPES_CHOICES[self.format_type]
 
-    def get_formater_type(self):
+    def get_formater_type(self, target):
         if self.formater_type not in IMPORTER_TYPES_DCT.keys():
             return
-        kwargs = {}
+        kwargs = {'target':target}
         if self.many_split:
             kwargs['many_split'] = self.many_split
         if self.formater_type == 'TypeFormater':
@@ -1206,35 +1239,44 @@ class FormaterType(models.Model):
                 model = import_module(self.options)
             return TypeFormater(model, **kwargs)
         elif self.formater_type == 'IntegerFormater':
-            return IntegerFormater()
+            return IntegerFormater(**kwargs)
         elif self.formater_type == 'FloatFormater':
-            return FloatFormater()
+            return FloatFormater(**kwargs)
         elif self.format_type == 'UnicodeFormater':
             try:
-                return UnicodeFormater(int(self.options.strip()))
+                return UnicodeFormater(int(self.options.strip()), **kwargs)
             except ValueError:
                 return
         elif self.format_type == 'DateFormater':
-            return DateFormater(self.options)
+            return DateFormater(self.options, **kwargs)
+
+IMPORT_STATE = (("C", _(u"Created")),
+                ("AP", _(u"Analyse in progress")),
+                ("A", _(u"Analysed")),
+                ("P", _(u"Import pending")),
+                ("IP", _(u"Import in progress")),
+                ("F", _(u"Finished")))
 
 class Import(models.Model):
     user = models.ForeignKey('IshtarUser')
     importer_type = models.ForeignKey(ImporterType)
     imported_file = models.FileField(_(u"Imported file"),
                                      upload_to="upload/imports/")
+    skip_lines = models.IntegerField(default=1)
     error_file = models.FileField(_(u"Error file"),
                                      upload_to="upload/imports/",
                                      blank=True, null=True)
     result_file = models.FileField(_(u"Result file"),
                                      upload_to="upload/imports/",
                                      blank=True, null=True)
-    state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE)
-    creation_date = models.DateTimeField(_(u"Creation date"), blank=True,
-                                         null=True)
+    state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE,
+                             default='C')
+    creation_date = models.DateTimeField(_(u"Creation date"), auto_now_add=True,
+                                         blank=True, null=True)
     end_date = models.DateTimeField(_(u"End date"), blank=True,
-                                    null=True)
+                                    null=True, editable=False)
     seconds_remaining = models.IntegerField(_(u"Seconds remaining"), blank=True,
-                                            null=True)
+                                            null=True, editable=False)
     class Meta:
         verbose_name = _(u"Import")
         verbose_name_plural = _(u"Imports")
@@ -1243,6 +1285,28 @@ class Import(models.Model):
         return u"%s - %s" % (unicode(self.importer_type),
                              unicode(self.user))
 
+    @property
+    def importer_instance(self):
+        return self.importer_type.importer_class(skip_lines=self.skip_lines,
+                                                 import_instance=self)
+
+    @property
+    def data_table(self):
+        encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
+        with open(self.imported_file.filename) as csv_file:
+            for encoding in encodings:
+                try:
+                    return [line for line in unicodecsv.reader(csv_file,
+                                                       encoding=encoding)]
+                except UnicodeDecodeError:
+                    if encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+        return []
+
+    def initialize(self):
+        self.importer_instance.initialize(self.data_table, output='db')
+
 class Organization(Address, Merge, OwnPerms, ValueGetter):
     TABLE_COLS = ('name', 'organization_type',)
     name = models.CharField(_(u"Name"), max_length=300)
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 4b4f38e4a..8745b079d 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -103,7 +103,6 @@ class MergeTest(TestCase):
         self.assertTrue(self.person_types[1] in self.person_3.person_types.all())
 
 class ImportKeyTest(TestCase):
-
     def testKeys(self):
         content_type = ContentType.objects.get_for_model(models.OrganizationType)