Imports: manage soft import (update) with unicity keys - script to initialize SRA file import

5 files changed, 456 insertions, 45 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 6239bf4c0..3d623e7a6 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -177,7 +177,8 @@ class UnicodeFormater(Formater):
                                      'length': self.max_length})
         if self.notnull and not value:
             return
-        value = self.prefix + value
+        if value:
+            value = self.prefix + value
         return value
 
 
@@ -603,7 +604,6 @@ class Importer(object):
     UNICITY_KEYS = []
     EXTRA_DEFAULTS = {}
     DEFAULTS = {}
-    STR_CUT = {}
     ERRORS = {
         'header_check': _(
             u"The given file is not correct. Check the file "
@@ -630,9 +630,13 @@ class Importer(object):
         model_name = self.OBJECT_CLS.__module__ + '.' + \
             self.OBJECT_CLS.__name__
 
+        unicity_keys = ''
+        if self.UNICITY_KEYS:
+            unicity_keys = ";".join(self.UNICITY_KEYS)
+
         importer = models.ImporterType.objects.create(
             slug=self.SLUG, name=name, description=self.DESC,
-            associated_models=model_name)
+            associated_models=model_name, unicity_keys=unicity_keys)
 
         for default in self.DEFAULTS:
             values = self.DEFAULTS[default]
@@ -691,13 +695,15 @@ class Importer(object):
                 models.ImportTarget.objects.get_or_create(
                     column=column, target=target, formater_type=formater_model,
                     force_new=getattr(formater, 'force_new', False),
+                    concat=getattr(formater, 'concat', False),
                     regexp_filter=regexp_filter,
                     comment=line.comment)
         return True
 
     def __init__(self, skip_lines=0, reference_header=None,
                  check_col_num=False, test=False, history_modifier=None,
-                 output='silent', import_instance=None):
+                 output='silent', import_instance=None,
+                 conservative_import=False):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
@@ -715,8 +721,12 @@ class Importer(object):
         self.line_format = copy.copy(self.LINE_FORMAT)
         self.import_instance = import_instance
         self.archive = None
+        self.conservative_import = conservative_import
+        # for a conservative_import UNICITY_KEYS should be defined
+        assert not self.conservative_import or bool(self.UNICITY_KEYS)
         self.DB_TARGETS = {}
         self.match_table = {}
+        self.concats = set()
         if import_instance and import_instance.imported_images:
             self.archive = import_instance.imported_images
         self._defaults = self.DEFAULTS.copy()
@@ -883,6 +893,7 @@ class Importer(object):
                 self.errors.append((idx_line, None, msg))
 
     def _line_processing(self, idx_line, line):
+        self.idx_line = idx_line
         if self.skip_lines > idx_line:
             self.validity.append(line)
             return
@@ -927,17 +938,11 @@ class Importer(object):
         if self.test:
             return
         # manage unicity of items (mainly for updates)
-        if self.UNICITY_KEYS:
-            data['defaults'] = {}
-            for k in data.keys():
-                if k not in self.UNICITY_KEYS \
-                   and k != 'defaults':
-                    data['defaults'][k] = data.pop(k)
-
         if 'history_modifier' in \
                 self.OBJECT_CLS._meta.get_all_field_names():
             data['history_modifier'] = self.history_modifier
 
+        self.plouf = 0
         obj, created = self.get_object(self.OBJECT_CLS, data)
         if self.import_instance and hasattr(obj, 'imports') \
            and created:
@@ -1014,10 +1019,10 @@ class Importer(object):
                     self.errors.append(
                         (idx_line + 1, idx_col + 1,
                          self.ERRORS['value_required']))
+                    self.c_errors = True
                 elif not val.strip():
                     c_row.append("")
                     return
-                self.c_errors = True
                 val = val.replace(NEW_LINE_BREAK, '\n')
                 self.errors.append(
                     (idx_line + 1, idx_col + 1,
@@ -1049,6 +1054,8 @@ class Importer(object):
                 field_name = field_name[idx_v]
             if type(force_new) in (list, tuple):
                 force_new = force_new[idx_v]
+            if formater.concat:
+                self.concats.add(field_name)
 
             if self.DB_TARGETS:
                 formater.reinit_db_target(
@@ -1192,14 +1199,18 @@ class Importer(object):
                         m2ms.append((attribute, v))
         elif hasattr(field_object, 'rel') and field_object.rel:
             if type(data[attribute]) == dict:
-                c_path.append(attribute)
                 # put history_modifier for every created item
                 if 'history_modifier' in \
                    field_object.rel.to._meta.get_all_field_names():
                     data[attribute]['history_modifier'] = \
                         self.history_modifier
-                data[attribute], created = self.get_object(
-                    field_object.rel.to, data[attribute], c_path)
+                try:
+                    c_path.append(attribute)
+                    data[attribute], created = self.get_object(
+                        field_object.rel.to, data[attribute].copy(), c_path)
+                except ImporterError, msg:
+                    self.errors.append((self.idx_line, None, msg))
+                    data[attribute] = None
             elif type(data[attribute]) == list:
                 data[attribute] = data[attribute][0]
 
@@ -1207,37 +1218,37 @@ class Importer(object):
         m2ms = []
         if data and type(data) == dict:
             c_path = path[:]
-            for attribute in data.keys():
+
+            # get all related fields
+            for attribute in list(data.keys()):
+                c_c_path = c_path[:]
                 if not attribute:
                     data.pop(attribute)
                     continue
                 if not data[attribute]:
                     continue
                 if attribute != '__force_new':
-                    self.get_field(cls, attribute, data, m2ms, c_path)
-            # default values
-            path = tuple(path)
-            if path in self._defaults:
-                for k in self._defaults[path]:
-                    if k not in data or not data[k]:
-                        data[k] = self._defaults[path][k]
-
-            # pre treatment
-            if path in self.STR_CUT:
-                for k in self.STR_CUT[path]:
-                    if k in data and data[k]:
-                        data[k] = unicode(data[k])[:self.STR_CUT[path][k]]
+                    self.get_field(cls, attribute, data, m2ms, c_c_path)
 
-            # filter default values
+            # filter uncessary default values
             create_dict = copy.deepcopy(data)
             for k in create_dict.keys():
                 if type(create_dict[k]) == dict:
                     create_dict.pop(k)
+
+            # default values
+            path = tuple(path)
             defaults = {}
+            if path in self._defaults:
+                for k in self._defaults[path]:
+                    if (k not in data or not data[k]):
+                        defaults[k] = self._defaults[path][k]
+
             if 'history_modifier' in create_dict:
-                defaults = {
+                defaults.update({
                     'history_modifier': create_dict.pop('history_modifier')
-                }
+                })
+
             try:
                 try:
                     dct = create_dict.copy()
@@ -1248,10 +1259,44 @@ class Importer(object):
                         created = dct.pop('__force_new')
                         if not [k for k in dct if dct[k] is not None]:
                             return None, created
-                        obj = cls.objects.create(**dct)
+                        new_dct = defaults.copy()
+                        new_dct.update(dct)
+                        obj = cls.objects.create(**new_dct)
                     else:
-                        dct['defaults'] = defaults
+                        # manage UNICITY_KEYS - only level 1
+                        if not path and self.UNICITY_KEYS:
+                            for k in dct.keys():
+                                if k not in self.UNICITY_KEYS \
+                                   and k != 'defaults':
+                                    defaults[k] = dct.pop(k)
+
+                        dct['defaults'] = defaults.copy()
                         obj, created = cls.objects.get_or_create(**dct)
+
+                        if not created and not path and self.UNICITY_KEYS:
+                            changed = False
+                            if self.conservative_import:
+                                for k in dct['defaults']:
+                                    new_val = dct['defaults'][k]
+                                    if new_val is None or new_val == '':
+                                        continue
+                                    val = getattr(obj, k)
+                                    if val is None or val == '':
+                                        changed = True
+                                        setattr(obj, k, new_val)
+                                    elif k in self.concats \
+                                            and type(val) == unicode \
+                                            and type(new_val) == unicode:
+                                        setattr(obj, k, val + u" - " + new_val)
+                            else:
+                                for k in dct['defaults']:
+                                    new_val = dct['defaults'][k]
+                                    if new_val is None or new_val == '':
+                                        continue
+                                    changed = True
+                                    setattr(obj, k, new_val)
+                            if changed:
+                                obj.save()
                     if self.import_instance and hasattr(obj, 'imports') \
                        and created:
                         obj.imports.add(self.import_instance)
@@ -1286,24 +1331,25 @@ class Importer(object):
                 except UnicodeDecodeError:
                     data = ''
                 raise ImporterError(
-                    "Erreur d'import %s, contexte : %s, erreur : %s"
-                    % (unicode(cls), unicode(data), message))
+                    "Erreur d'import %s %s, contexte : %s, erreur : %s"
+                    % (unicode(cls), unicode("__".join(path)),
+                       unicode(data), message))
             return obj, created
         return data
 
-    def _format_csv_line(self, values):
+    def _format_csv_line(self, values, empty=u"-"):
         return u'"' + u'","'.join(
-            [(v and unicode(v).replace('"', '""')) or u'-'
+            [(v and unicode(v).replace('"', '""')) or empty
              for v in values]) + u'"'
 
-    def _get_csv(self, rows, header=[]):
+    def _get_csv(self, rows, header=[], empty=u"-"):
         if not rows:
             return ""
         csv_v = []
         if header:
-            csv_v.append(self._format_csv_line(header))
+            csv_v.append(self._format_csv_line(header, empty=empty))
         for values in rows:
-            csv_v.append(self._format_csv_line(values))
+            csv_v.append(self._format_csv_line(values, empty=empty))
         return u"\n".join(csv_v)
 
     def get_csv_errors(self):
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index c3b70d42b..0e6a34b74 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -104,7 +104,7 @@ class NewImportForm(forms.ModelForm):
     class Meta:
         model = models.Import
         fields = ('importer_type', 'imported_file', 'imported_images',
-                  'encoding', 'skip_lines')
+                  'conservative_import', 'encoding', 'skip_lines')
 
     def save(self, user, commit=True):
         self.instance.user = user
diff --git a/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py b/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py
new file mode 100644
index 000000000..990f35efc
--- /dev/null
+++ b/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py
@@ -0,0 +1,353 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'ImporterType.unicity_keys'
+        db.add_column('ishtar_common_importertype', 'unicity_keys',
+                      self.gf('django.db.models.fields.CharField')(max_length=500, null=True, blank=True),
+                      keep_default=False)
+
+        # Adding field 'ImportTarget.concat'
+        db.add_column('ishtar_common_importtarget', 'concat',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+        # Adding field 'Import.conservative_import'
+        db.add_column('ishtar_common_import', 'conservative_import',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'ImporterType.unicity_keys'
+        db.delete_column('ishtar_common_importertype', 'unicity_keys')
+
+        # Deleting field 'ImportTarget.concat'
+        db.delete_column('ishtar_common_importtarget', 'concat')
+
+        # Deleting field 'Import.conservative_import'
+        db.delete_column('ishtar_common_import', 'conservative_import')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'ishtar_common.arrondissement': {
+            'Meta': {'object_name': 'Arrondissement'},
+            'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.author': {
+            'Meta': {'object_name': 'Author'},
+            'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"})
+        },
+        'ishtar_common.authortype': {
+            'Meta': {'object_name': 'AuthorType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.canton': {
+            'Meta': {'object_name': 'Canton'},
+            'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'ishtar_common.department': {
+            'Meta': {'ordering': "['number']", 'object_name': 'Department'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}),
+            'state': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.State']", 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.documenttemplate': {
+            'Meta': {'ordering': "['associated_object_name', 'name']", 'object_name': 'DocumentTemplate'},
+            'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'})
+        },
+        'ishtar_common.format': {
+            'Meta': {'object_name': 'Format'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.formatertype': {
+            'Meta': {'ordering': "('formater_type', 'options')", 'unique_together': "(('formater_type', 'options', 'many_split'),)", 'object_name': 'FormaterType'},
+            'formater_type': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'many_split': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'options': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.globalvar': {
+            'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'},
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}),
+            'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.historicalorganization': {
+            'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}),
+            'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.import': {
+            'Meta': {'object_name': 'Import'},
+            'conservative_import': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'creation_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}),
+            'encoding': ('django.db.models.fields.CharField', [], {'default': "'utf-8'", 'max_length': '15'}),
+            'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'error_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imported_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'imported_images': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.ImporterType']"}),
+            'match_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'result_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'seconds_remaining': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'skip_lines': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'state': ('django.db.models.fields.CharField', [], {'default': "'C'", 'max_length': '2'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']"})
+        },
+        'ishtar_common.importercolumn': {
+            'Meta': {'object_name': 'ImporterColumn'},
+            'col_number': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'columns'", 'to': "orm['ishtar_common.ImporterType']"}),
+            'regexp_pre_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}),
+            'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'ishtar_common.importerdefault': {
+            'Meta': {'object_name': 'ImporterDefault'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'defaults'", 'to': "orm['ishtar_common.ImporterType']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.importerdefaultvalues': {
+            'Meta': {'object_name': 'ImporterDefaultValues'},
+            'default_target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'default_values'", 'to': "orm['ishtar_common.ImporterDefault']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'value': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.importerduplicatefield': {
+            'Meta': {'object_name': 'ImporterDuplicateField'},
+            'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_fields'", 'to': "orm['ishtar_common.ImporterColumn']"}),
+            'field_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'force_new': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'ishtar_common.importertype': {
+            'Meta': {'object_name': 'ImporterType'},
+            'associated_models': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_template': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '100', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'unicity_keys': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.importtarget': {
+            'Meta': {'object_name': 'ImportTarget'},
+            'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'targets'", 'to': "orm['ishtar_common.ImporterColumn']"}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'concat': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'force_new': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'formater_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.FormaterType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'regexp_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.ishtaruser': {
+            'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']},
+            'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'ishtar_common.itemkey': {
+            'Meta': {'object_name': 'ItemKey'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {})
+        },
+        'ishtar_common.organization': {
+            'Meta': {'object_name': 'Organization'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_organization'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}),
+            'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}),
+            'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.organizationtype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.person': {
+            'Meta': {'object_name': 'Person'},
+            'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_person'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}),
+            'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}),
+            'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}),
+            'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}),
+            'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.persontype': {
+            'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.regexp': {
+            'Meta': {'object_name': 'Regexp'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'regexp': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        },
+        'ishtar_common.sourcetype': {
+            'Meta': {'object_name': 'SourceType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.state': {
+            'Meta': {'ordering': "['number']", 'object_name': 'State'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'})
+        },
+        'ishtar_common.supporttype': {
+            'Meta': {'object_name': 'SupportType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'ishtar_common.targetkey': {
+            'Meta': {'unique_together': "(('target', 'key', 'associated_user', 'associated_import'),)", 'object_name': 'TargetKey'},
+            'associated_import': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}),
+            'associated_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_set': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'key': ('django.db.models.fields.TextField', [], {}),
+            'target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'keys'", 'to': "orm['ishtar_common.ImportTarget']"}),
+            'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'ishtar_common.town': {
+            'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'},
+            'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}),
+            'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}),
+            'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}),
+            'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['ishtar_common']
\ No newline at end of file
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 63ea7d2a8..fd0de385e 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -1149,6 +1149,8 @@ class ImporterType(models.Model):
     associated_models = models.CharField(_(u"Associated model"),
                                          max_length=200, choices=MODELS)
     is_template = models.BooleanField(_(u"Is template"), default=False)
+    unicity_keys = models.CharField(_(u"Unicity keys (separator \";\")"),
+                                    blank=True, null=True, max_length=500)
 
     class Meta:
         verbose_name = _(u"Importer - Type")
@@ -1197,8 +1199,12 @@ class ImporterType(models.Model):
             formater = ImportFormater(targets, formater_types,
                                       **formater_kwargs)
             LINE_FORMAT.append(formater)
+        UNICITY_KEYS = []
+        if self.unicity_keys:
+            UNICITY_KEYS = [un.strip() for un in self.unicity_keys.split(';')]
         args = {'OBJECT_CLS': OBJECT_CLS, 'DESC': self.description,
-                'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT}
+                'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT,
+                'UNICITY_KEYS': UNICITY_KEYS}
         name = str(''.join(
             x for x in slugify(self.name).replace('-', ' ').title()
             if not x.isspace()))
@@ -1355,6 +1361,8 @@ class ImportTarget(models.Model):
     formater_type = models.ForeignKey("FormaterType")
     force_new = models.BooleanField(_(u"Force creation of new item"),
                                     default=False)
+    concat = models.BooleanField(_(u"Concatenate with existing"),
+                                 default=False)
     comment = models.TextField(_(u"Comment"), blank=True, null=True)
 
     class Meta:
@@ -1604,6 +1612,9 @@ class Import(models.Model):
                                   blank=True, null=True)
     state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE,
                              default='C')
+    conservative_import = models.BooleanField(
+        _(u"Conservative import"), default=False,
+        help_text='If set to true, do not overload existing values')
     creation_date = models.DateTimeField(_(u"Creation date"),
                                          auto_now_add=True, blank=True,
                                          null=True)
@@ -1656,7 +1667,8 @@ class Import(models.Model):
 
     def get_importer_instance(self):
         return self.importer_type.get_importer_class()(
-            skip_lines=self.skip_lines, import_instance=self)
+            skip_lines=self.skip_lines, import_instance=self,
+            conservative_import=self.conservative_import)
 
     @property
     def data_table(self):
diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py
index abbdda711..c8b017293 100644
--- a/ishtar_common/wizards.py
+++ b/ishtar_common/wizards.py
@@ -677,7 +677,7 @@ class Wizard(NamedUrlWizardView):
         if not multi:
             key = key.startswith(form_key) and key or form_key + '-' + key
             val = request.session[storage.prefix]['step_data'][form_key][key]
-            if type(val) in (list, tuple):
+            if type(val) in (list, tuple) and val:
                 val = val[0]
             return val
         vals = []