Merge branch 'v0.9' into wheezy

13 files changed, 355 insertions, 223 deletions

diff --git a/CHANGES.md b/CHANGES.md
index df5bc087b..b0b04ae8c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,6 +1,21 @@
 Ishtar changelog
 ================
 
+v0.99.13 (2017-02-17)
+--------------------
+### Features ###
+- Accounts: initialize new account with default username extract from raw name
+- Imports:
+  - Prevent creation of new items when data is empty
+  - Item keys are now related to the current importer
+  - Report bad configuration on importers in a cleaner way
+
+### Bug fixes ###
+- Fix password initialization on user creation
+- Fix MCC operation import
+- Makefile: graph model generation fix
+- Finds table: fix columns
+
 v0.99.12.2 (2017-02-10)
 -----------------------
 ### Features ###
diff --git a/Makefile.example b/Makefile.example
index 89cae31a1..e7e604706 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -145,14 +145,15 @@ schemamigrations_initial:
 
 generate_doc:
 	cd $(project);\
-	$(PYTHON) manage.py graph_models --pydot -I "ImporterType,ImporterDefault,ImporterDefaultValues,ImporterColumn,Regexp,ImportTarget,FormaterType,Import" ishtar_common > /tmp/ishtar.dot ;\
-	dot -Tpng /tmp/ishtar.dot -o ../docs/source/_static/db-imports.png
+	$(PYTHON) manage.py graph_models --pydot -g -I "ImporterModel,ImporterType,ImporterDefault,ImporterDefaultValues,ImporterColumn,Regexp,ImportTarget,FormaterType,Import" ishtar_common > /tmp/ishtar-imports.dot ;\
+	dot -Tpng /tmp/ishtar-imports.dot -o ../docs/source/_static/db-imports.png
+	rm /tmp/ishtar-imports.dot
 	cd $(project);\
 	for APP in $(apps); do \
-		$(PYTHON) manage.py graph_models --pydot $$APP > /tmp/ishtar.dot; \
-		dot -Tpng /tmp/ishtar.dot -o ../docs/source/_static/db-$$APP.png; \
+		$(PYTHON) manage.py graph_models -g --pydot $$APP > /tmp/ishtar-$$APP.dot; \
+		dot -Tpng /tmp/ishtar-$$APP.dot -o ../docs/source/_static/db-$$APP.png; \
+		rm /tmp/ishtar-$$APP.dot ; \
 	done
-	rm /tmp/ishtar.dot
 
 fixtures: fixtures_auth fixtures_common fixtures_operations fixtures_context_records fixtures_finds fixtures_warehouse fixtures_files
 
diff --git a/archaeological_context_records/tests.py b/archaeological_context_records/tests.py
index f1e6581d7..1c900184c 100644
--- a/archaeological_context_records/tests.py
+++ b/archaeological_context_records/tests.py
@@ -88,8 +88,9 @@ class ImportContextRecordTest(ImportTest, TestCase):
         self.init_cr_targetkey(impt)
         # Dating is not in models that can be created but force new is
         # set for a column that references Dating
-        with self.assertRaises(ImproperlyConfigured):
-            impt.importation()
+        impt.importation()
+        self.assertEqual(len(impt.errors), 4)
+        self.assertIn("doesn't exist in the database.", impt.errors[0]['error'])
 
         # retry with only Dating (no context record)
         for cr in models.ContextRecord.objects.all():
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index c8903fc10..8f0270236 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -325,7 +325,7 @@ class Find(BaseHistorizedItem, ImageModel, OwnPerms, ShortMenuItem):
     TABLE_COLS_FOR_OPE = [
         'base_finds__cache_short_id',
         'base_finds__cache_complete_id',
-        'previous_id', 'label', 'material_types',
+        'previous_id', 'label', 'material_types__label',
         'datings__period__label', 'find_number', 'object_types',
         'container__cached_label',
         'description',
@@ -334,6 +334,7 @@ class Find(BaseHistorizedItem, ImageModel, OwnPerms, ShortMenuItem):
     COL_LABELS = {
         'datings__period__label': _(u"Periods"),
         'container__cached_label': _(u"Container"),
+        'material_types__label': _(u"Material types"),
     }
 
     EXTRA_FULL_FIELDS = [
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 040c7c3d8..364cc4c8e 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -34,8 +34,9 @@ import models
 
 from archaeological_operations import views
 
-from ishtar_common.models import OrganizationType, Organization, \
-    ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, Town
+from ishtar_common.models import OrganizationType, Organization, ItemKey, \
+    ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, \
+    Town, ImporterColumn, Person
 from archaeological_context_records.models import Unit
 
 from ishtar_common import forms_common
@@ -73,26 +74,30 @@ class ImportTest(object):
 
     def init_ope_targetkey(self, imp):
         # doing manually connections
-        tg = TargetKey.objects.filter(target__target='operation_type'
-                                      ).order_by('-pk').all()[0]
-        tg.value = models.OperationType.objects.get(
+        target = TargetKey.objects.filter(
+            target__target='operation_type').order_by('-pk').all()[0]
+        target.value = models.OperationType.objects.get(
             txt_idx='prog_excavation').pk
-        tg.is_set = True
-        tg.save()
-
-        target = TargetKey.objects.get(key='gallo-romain',
-                                       associated_import=imp)
-        gallo = models.Period.objects.get(txt_idx='gallo-roman')
-        target.value = gallo.pk
         target.is_set = True
+        target.associated_import = imp
         target.save()
 
-        target = TargetKey.objects.get(key='age-du-fer',
-                                       associated_import=imp)
+        target2 = TargetKey.objects.get(key='gallo-romain',
+                                        associated_import=imp)
+        gallo = models.Period.objects.get(txt_idx='gallo-roman')
+        target2.value = gallo.pk
+        target2.is_set = True
+        target2.associated_import = imp
+        target2.save()
+
+        target3 = TargetKey.objects.get(key='age-du-fer',
+                                        associated_import=imp)
         iron = models.Period.objects.get(txt_idx='iron_age')
-        target.value = iron.pk
-        target.is_set = True
-        target.save()
+        target3.value = iron.pk
+        target3.is_set = True
+        target3.associated_import = imp
+        target3.save()
+        return [target, target2, target3]
 
     def init_ope(self):
         importer, form = self.init_ope_import()
@@ -166,6 +171,7 @@ class ImportOperationTest(ImportTest, TestCase):
 
     def test_mcc_import_operation(self):
         first_ope_nb = models.Operation.objects.count()
+        first_person_nb = Person.objects.count()
         importer, form = self.init_ope_import()
         self.assertTrue(form.is_valid())
         impt = form.save(self.ishtar_user)
@@ -179,18 +185,20 @@ class ImportOperationTest(ImportTest, TestCase):
         current_ope_nb = models.Operation.objects.count()
         # no new operation imported because of a missing connection for
         # operation_type value
-        self.assertTrue(current_ope_nb == first_ope_nb)
+        self.assertEqual(current_ope_nb, first_ope_nb)
         self.init_ope_targetkey(imp=impt)
 
         impt.importation()
-        # a new operation has now been imported
+        # new operations have now been imported
         current_ope_nb = models.Operation.objects.count()
-        self.assertTrue(current_ope_nb == (first_ope_nb + 1))
+        self.assertEqual(current_ope_nb, first_ope_nb + 2)
+        current_person_nb = Person.objects.count()
+        self.assertEqual(current_person_nb, first_person_nb + 1)
         # and well imported
         last_ope = models.Operation.objects.order_by('-pk').all()[0]
         self.assertEqual(last_ope.name, u"Oppìdum de Paris")
-        self.assertTrue(last_ope.code_patriarche == 4200)
-        self.assertTrue(last_ope.operation_type.txt_idx == 'prog_excavation')
+        self.assertEqual(last_ope.code_patriarche, 4200)
+        self.assertEqual(last_ope.operation_type.txt_idx, 'prog_excavation')
         self.assertEqual(last_ope.periods.count(), 2)
         periods = [period.txt_idx for period in last_ope.periods.all()]
         self.assertIn('iron_age', periods)
@@ -199,9 +207,43 @@ class ImportOperationTest(ImportTest, TestCase):
         # a second importation will be not possible: no two same patriarche
         # code
         impt.importation()
-        models.Operation.objects.count()
-        self.assertTrue(last_ope ==
-                        models.Operation.objects.order_by('-pk').all()[0])
+        self.assertEqual(last_ope,
+                         models.Operation.objects.order_by('-pk').all()[0])
+
+    def test_keys_limitation(self):
+        # each key association is associated to the import
+        init_ope_number = models.Operation.objects.count()
+        importer, form = self.init_ope_import()
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        self.init_ope_targetkey(imp=impt)
+
+        importer, form = self.init_ope_import()
+        other_imp = form.save(self.ishtar_user)
+        # associate with another import
+        for ik in ItemKey.objects.filter(importer=impt).all():
+            ik.importer = other_imp
+            ik.save()
+
+        impt.importation()
+        current_ope_nb = models.Operation.objects.count()
+        # no new operation
+        self.assertEqual(current_ope_nb, init_ope_number)
+
+    def test_bad_configuration(self):
+        importer, form = self.init_ope_import()
+        col = ImporterColumn.objects.get(importer_type=importer, col_number=1)
+        target = col.targets.all()[0]
+        target.target = "cody"  # random and not appropriate string
+        target.save()
+        # self.init_ope()
+        # importer, form = self.init_ope_import()
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        self.init_ope_targetkey(imp=impt)
+        impt.importation()
+        self.assertEqual(len(impt.errors), 2)
+        self.assertIn("Importer configuration error", impt.errors[0]['error'])
 
     def test_model_limitation(self):
         importer, form = self.init_ope_import()
@@ -214,10 +256,10 @@ class ImportOperationTest(ImportTest, TestCase):
         init_ope_number = models.Operation.objects.count()
         impt.importation()
         current_ope_nb = models.Operation.objects.count()
-        self.assertEqual(current_ope_nb, init_ope_number + 1)
+        self.assertEqual(current_ope_nb, init_ope_number + 2)
 
-        last_ope = models.Operation.objects.order_by('-pk').all()[0]
-        last_ope.delete()
+        for ope in models.Operation.objects.order_by('-pk').all()[:2]:
+            ope.delete()
 
         importer, form = self.init_ope_import()
         # add an inadequate model to make created_models non empty
@@ -247,7 +289,7 @@ class ImportOperationTest(ImportTest, TestCase):
         # import of operations
         impt.importation()
         current_ope_nb = models.Operation.objects.count()
-        self.assertEqual(current_ope_nb, init_ope_number + 1)
+        self.assertEqual(current_ope_nb, init_ope_number + 2)
 
     def test_mcc_import_parcels(self):
         old_nb = models.Parcel.objects.count()
@@ -793,7 +835,6 @@ class OperationSearchTest(TestCase, OperationInitTest):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(json.loads(response.content)['total'], 1)
 
-
     def testOwnSearch(self):
         c = Client()
         response = c.get(reverse('get-operation'), {'year': '2010'})
@@ -895,12 +936,10 @@ class OperationWizardCreationTest(WizardTest, OperationInitTest, TestCase):
             return super(OperationWizardCreationTest, self).pre_wizard()
         town = self.create_towns()[0]
         town_data = {'town': town.pk}
-        self.form_datas[0].form_datas['townsgeneral-operation_creation'].append(
-            town_data
-        )
-        self.form_datas[1].form_datas['townsgeneral-operation_creation'].append(
-            town_data
-        )
+        self.form_datas[0].form_datas[
+            'townsgeneral-operation_creation'].append(town_data)
+        self.form_datas[1].form_datas[
+            'townsgeneral-operation_creation'].append(town_data)
         parcel_data = {
             'town': town.pk, 'year': 2017, 'section': 'S',
             'parcel_number': '42'}
@@ -981,7 +1020,8 @@ class OperationWizardClosingTest(OperationWizardCreationTest):
         self.assertFalse(ope.is_active())
         self.assertEqual(
             ope.closing()['date'].strftime('%Y-%d-%m'),
-            self.form_datas[0].form_datas['date-operation_closing']['end_date'])
+            self.form_datas[0].form_datas['date-operation_closing']['end_date']
+        )
 
 
 class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest,
@@ -1001,12 +1041,11 @@ class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest,
         FormData(
             "Admin act creation",
             form_datas={
-                    'selec-operation_administrativeactop': {
-                    },
-                    'administrativeact-operation_administrativeactop': {
-                        'signature_date': str(datetime.date.today())
-                    }
-                },
+                'selec-operation_administrativeactop': {},
+                'administrativeact-operation_administrativeactop': {
+                    'signature_date': str(datetime.date.today())
+                }
+            },
         )
     ]
 
diff --git a/archaeological_operations/tests/MCC-operations-example.csv b/archaeological_operations/tests/MCC-operations-example.csv
index 432ceffca..3b9801c33 100644
--- a/archaeological_operations/tests/MCC-operations-example.csv
+++ b/archaeological_operations/tests/MCC-operations-example.csv
@@ -1,2 +1,3 @@
 code OA,region,type operation,intitule operation,operateur,responsable operation,date debut terrain,date fin terrain,chronologie generale,identifiant document georeferencement,notice scientifique
+4201,Bourgogne,Fouille programmée,Oppìdum de Paris 2,L'opérateur,,2000/01/31,2002/12/31,Age du Fer,,
 4200,Bourgogne,Fouille programmée,Oppìdum de Paris,L'opérateur,Jean Sui-Resp'on Sablé,2000/01/22,2002/12/31,Age du Fer & Gallo-Romain,,
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index a03f4de34..426d32a7a 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -29,9 +29,10 @@ import zipfile
 
 from django.conf import settings
 from django.contrib.auth.models import User
-from django.core.exceptions import ImproperlyConfigured
+from django.db.models.fields import FieldDoesNotExist
 from django.core.files import File
 from django.db import IntegrityError, DatabaseError, transaction
+from django.db.models import Q
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
@@ -148,13 +149,15 @@ class ChoiceChecker(object):
 
 class UnicodeFormater(Formater):
     def __init__(self, max_length=None, clean=False, re_filter=None,
-                 notnull=False, prefix=u'', db_target=None):
+                 notnull=False, prefix=u'', db_target=None,
+                 import_instance=None):
         self.max_length = max_length
         self.db_target = db_target
         self.clean = clean
         self.re_filter = re_filter
         self.notnull = notnull
         self.prefix = prefix
+        self.import_instance = import_instance
 
     def format(self, value):
         try:
@@ -255,7 +258,8 @@ class IntegerFormater(Formater):
 
 class StrChoiceFormater(Formater, ChoiceChecker):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
-                 cli=False, many_split='', db_target=None):
+                 cli=False, many_split='', db_target=None,
+                 import_instance=None):
         self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
@@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         self.new_keys = {}
         self.match_table = {}
         self.many_split = many_split
+        self.import_instance = None
         for key, value in self.choices:
             value = unicode(value)
             if not self.strict:
@@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker):
     def init_db_target(self):
         if not self.db_target:
             return
-        for target_key in self.db_target.keys.filter(is_set=True).all():
+        q = self.db_target.keys.filter(is_set=True)
+        if self.import_instance:
+            q = q.filter(Q(associated_import=self.import_instance) |
+                         Q(associated_import__isnull=True))
+        for target_key in q.all():
             key = target_key.key
             if not self.strict:
                 key = slugify(key)
@@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
 
 class TypeFormater(StrChoiceFormater):
     def __init__(self, model, cli=False, defaults={}, many_split=False,
-                 db_target=None):
+                 db_target=None, import_instance=None):
         self.create = True
         self.strict = False
         self.model = model
@@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater):
         self.equiv_dict, self.choices = {}, []
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
         for item in model.objects.all():
             self.choices.append((item.pk, unicode(item)))
-            for key in item.get_keys():
+            for key in item.get_keys(importer_id=import_instance.pk):
                 self.equiv_dict[key] = item
 
     def prepare(self, value):
@@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater):
 
 
 class DateFormater(Formater):
-    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None):
+    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None,
+                 import_instance=None):
         self.date_formats = date_formats
         if type(date_formats) not in (list, tuple):
             self.date_formats = [self.date_formats]
         self.db_target = db_target
+        self.import_instance = import_instance
 
     def format(self, value):
         value = value.strip()
@@ -511,7 +523,8 @@ class FileFormater(Formater):
 
 
 class StrToBoolean(Formater, ChoiceChecker):
-    def __init__(self, choices={}, cli=False, strict=False, db_target=None):
+    def __init__(self, choices={}, cli=False, strict=False, db_target=None,
+                 import_instance=None):
         self.dct = copy.copy(choices)
         self.cli = cli
         self.strict = strict
@@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker):
         self.init_db_target()
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
 
     def init_db_target(self):
         if not self.db_target:
@@ -733,7 +747,7 @@ class Importer(object):
         q = ImporterModel.objects.filter(klass=cls_name)
         if q.count():
             cls_name = q.all()[0].name
-        return ImproperlyConfigured(
+        return ImporterError(
             unicode(self.ERRORS['improperly_configured']).format(cls_name))
 
     def _get_does_not_exist_in_db_error(self, model, data):
@@ -827,7 +841,7 @@ class Importer(object):
                 vals[idx_col].append(val)
         for idx, formater in enumerate(self.line_format):
             if formater and idx < len(vals):
-
+                formater.import_instance = self.import_instance
                 if self.DB_TARGETS:
                     field_names = formater.field_name
                     if type(field_names) not in (list, tuple):
@@ -1143,6 +1157,7 @@ class Importer(object):
                 self.concat_str[field_name] = concat_str
 
             if self.DB_TARGETS:
+                formater.import_instance = self.import_instance
                 formater.reinit_db_target(
                     self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)],
                     idx_v)
@@ -1217,8 +1232,13 @@ class Importer(object):
         c_row.append(u" ; ".join([v for v in c_values]))
 
     def get_field(self, cls, attribute, data, m2ms, c_path, new_created):
-        field_object, model, direct, m2m = \
-            cls._meta.get_field_by_name(attribute)
+        try:
+            field_object, model, direct, m2m = \
+                cls._meta.get_field_by_name(attribute)
+        except FieldDoesNotExist:
+            raise ImporterError(unicode(
+                _(u"Importer configuration error: field \"{}\" does not exist "
+                  u"for {}.")).format(attribute, cls._meta.verbose_name))
         if m2m:
             many_values = data.pop(attribute)
             if hasattr(field_object, 'rel'):
@@ -1361,146 +1381,152 @@ class Importer(object):
 
     def get_object(self, cls, data, path=[]):
         m2ms = []
-        if data and type(data) == dict:
-            c_path = path[:]
-
-            # get all related fields
-            new_created = {}
-            for attribute in list(data.keys()):
-                c_c_path = c_path[:]
-                if not attribute:
-                    data.pop(attribute)
-                    continue
-                if not data[attribute]:
-                    continue
-                if attribute != '__force_new':
-                    self.get_field(cls, attribute, data, m2ms, c_c_path,
-                                   new_created)
-
-            create_dict = copy.deepcopy(data)
-            for k in create_dict.keys():
-                # filter unnecessary default values
-                if type(create_dict[k]) == dict:
-                    create_dict.pop(k)
-                # File doesn't like deepcopy
-                if type(create_dict[k]) == File:
-                    create_dict[k] = copy.copy(data[k])
-
-            # default values
-            path = tuple(path)
-            defaults = {}
-            if path in self._defaults:
-                for k in self._defaults[path]:
-                    if (k not in data or not data[k]):
-                        defaults[k] = self._defaults[path][k]
-
-            if 'history_modifier' in create_dict:
-                defaults.update({
-                    'history_modifier': create_dict.pop('history_modifier')
-                })
-
-            created = False
+        if type(data) != dict:
+            return data, False
+        is_empty = not bool(
+            [k for k in data if k not in ('history_modifier', 'defaults')
+                and data[k]])
+        if is_empty:
+            return None, False
+
+        c_path = path[:]
+
+        # get all related fields
+        new_created = {}
+        for attribute in list(data.keys()):
+            c_c_path = c_path[:]
+            if not attribute:
+                data.pop(attribute)
+                continue
+            if not data[attribute]:
+                continue
+            if attribute != '__force_new':
+                self.get_field(cls, attribute, data, m2ms, c_c_path,
+                               new_created)
+
+        create_dict = copy.deepcopy(data)
+        for k in create_dict.keys():
+            # filter unnecessary default values
+            if type(create_dict[k]) == dict:
+                create_dict.pop(k)
+            # File doesn't like deepcopy
+            if type(create_dict[k]) == File:
+                create_dict[k] = copy.copy(data[k])
+
+        # default values
+        path = tuple(path)
+        defaults = {}
+        if path in self._defaults:
+            for k in self._defaults[path]:
+                if k not in data or not data[k]:
+                    defaults[k] = self._defaults[path][k]
+
+        if 'history_modifier' in create_dict:
+            defaults.update({
+                'history_modifier': create_dict.pop('history_modifier')
+            })
+
+        created = False
+        try:
             try:
-                try:
-                    dct = create_dict.copy()
-                    for key in dct:
-                        if callable(dct[key]):
-                            dct[key] = dct[key]()
-                    if '__force_new' in dct:
-                        created = dct.pop('__force_new')
-                        if not [k for k in dct if dct[k] is not None]:
-                            return None, created
-                        new_dct = defaults.copy()
-                        new_dct.update(dct)
-                        if self.MODEL_CREATION_LIMIT and \
-                                cls not in self.MODEL_CREATION_LIMIT:
-                            raise self._get_improperly_conf_error(cls)
-                        obj = cls.objects.create(**new_dct)
+                dct = create_dict.copy()
+                for key in dct:
+                    if callable(dct[key]):
+                        dct[key] = dct[key]()
+                if '__force_new' in dct:
+                    created = dct.pop('__force_new')
+                    if not [k for k in dct if dct[k] is not None]:
+                        return None, created
+                    new_dct = defaults.copy()
+                    new_dct.update(dct)
+                    if self.MODEL_CREATION_LIMIT and \
+                            cls not in self.MODEL_CREATION_LIMIT:
+                        raise self._get_improperly_conf_error(cls)
+                    obj = cls.objects.create(**new_dct)
+                else:
+                    # manage UNICITY_KEYS - only level 1
+                    if not path and self.UNICITY_KEYS:
+                        for k in dct.keys():
+                            if k not in self.UNICITY_KEYS \
+                               and k != 'defaults':
+                                defaults[k] = dct.pop(k)
+                    if not self.MODEL_CREATION_LIMIT or \
+                            cls in self.MODEL_CREATION_LIMIT:
+                        dct['defaults'] = defaults.copy()
+                        obj, created = cls.objects.get_or_create(**dct)
                     else:
-                        # manage UNICITY_KEYS - only level 1
-                        if not path and self.UNICITY_KEYS:
-                            for k in dct.keys():
-                                if k not in self.UNICITY_KEYS \
-                                   and k != 'defaults':
-                                    defaults[k] = dct.pop(k)
-                        if not self.MODEL_CREATION_LIMIT or \
-                                cls in self.MODEL_CREATION_LIMIT:
+                        try:
+                            obj = cls.objects.get(**dct)
                             dct['defaults'] = defaults.copy()
-                            obj, created = cls.objects.get_or_create(**dct)
-                        else:
-                            try:
-                                obj = cls.objects.get(**dct)
-                                dct['defaults'] = defaults.copy()
-                            except cls.DoesNotExist:
-                                raise self._get_does_not_exist_in_db_error(
-                                    cls, dct)
-
-                        if not created and not path and self.UNICITY_KEYS:
-                            changed = False
-                            if self.conservative_import:
-                                for k in dct['defaults']:
-                                    new_val = dct['defaults'][k]
-                                    if new_val is None or new_val == '':
-                                        continue
-                                    val = getattr(obj, k)
-                                    if val is None or val == '':
-                                        changed = True
-                                        setattr(obj, k, new_val)
-                                    elif k in self.concats \
-                                            and type(val) == unicode \
-                                            and type(new_val) == unicode:
-                                        setattr(obj, k, val + u"\n" + new_val)
-                            else:
-                                for k in dct['defaults']:
-                                    new_val = dct['defaults'][k]
-                                    if new_val is None or new_val == '':
-                                        continue
+                        except cls.DoesNotExist:
+                            raise self._get_does_not_exist_in_db_error(
+                                cls, dct)
+
+                    if not created and not path and self.UNICITY_KEYS:
+                        changed = False
+                        if self.conservative_import:
+                            for k in dct['defaults']:
+                                new_val = dct['defaults'][k]
+                                if new_val is None or new_val == '':
+                                    continue
+                                val = getattr(obj, k)
+                                if val is None or val == '':
                                     changed = True
                                     setattr(obj, k, new_val)
-                            if changed:
-                                obj.save()
-                    if self.import_instance and hasattr(obj, 'imports') \
-                       and created:
-                        obj.imports.add(self.import_instance)
-                except ValueError as e:
-                    raise IntegrityError(e.message)
-                except IntegrityError as e:
-                    raise IntegrityError(e.message)
-                except DatabaseError as e:
-                    raise IntegrityError(e.message)
-                except cls.MultipleObjectsReturned as e:
-                    created = False
-                    if 'defaults' in dct:
-                        dct.pop('defaults')
-                    raise IntegrityError(e.message)
-                    # obj = cls.objects.filter(**dct).all()[0]
-                for attr, value in m2ms:
-                    values = [value]
-                    if type(value) in (list, tuple):
-                        values = value
-                    for v in values:
-                        getattr(obj, attr).add(v)
-                        # force post save script
-                        v.save()
-                if m2ms:
-                    # force post save script
-                    obj.save()
+                                elif k in self.concats \
+                                        and type(val) == unicode \
+                                        and type(new_val) == unicode:
+                                    setattr(obj, k, val + u"\n" + new_val)
+                        else:
+                            for k in dct['defaults']:
+                                new_val = dct['defaults'][k]
+                                if new_val is None or new_val == '':
+                                    continue
+                                changed = True
+                                setattr(obj, k, new_val)
+                        if changed:
+                            obj.save()
+                if self.import_instance and hasattr(obj, 'imports') \
+                   and created:
+                    obj.imports.add(self.import_instance)
+            except ValueError as e:
+                raise IntegrityError(e.message)
             except IntegrityError as e:
-                message = e.message
-                try:
-                    message = e.message.decode('utf-8')
-                except (UnicodeDecodeError, UnicodeDecodeError):
-                    message = ''
-                try:
-                    data = unicode(data)
-                except UnicodeDecodeError:
-                    data = ''
-                raise ImporterError(
-                    "Erreur d'import %s %s, contexte : %s, erreur : %s"
-                    % (unicode(cls), unicode("__".join(path)),
-                       unicode(data), message))
-            return obj, created
-        return data
+                raise IntegrityError(e.message)
+            except DatabaseError as e:
+                raise IntegrityError(e.message)
+            except cls.MultipleObjectsReturned as e:
+                created = False
+                if 'defaults' in dct:
+                    dct.pop('defaults')
+                raise IntegrityError(e.message)
+                # obj = cls.objects.filter(**dct).all()[0]
+            for attr, value in m2ms:
+                values = [value]
+                if type(value) in (list, tuple):
+                    values = value
+                for v in values:
+                    getattr(obj, attr).add(v)
+                    # force post save script
+                    v.save()
+            if m2ms:
+                # force post save script
+                obj.save()
+        except IntegrityError as e:
+            message = e.message
+            try:
+                message = e.message.decode('utf-8')
+            except (UnicodeDecodeError, UnicodeDecodeError):
+                message = ''
+            try:
+                data = unicode(data)
+            except UnicodeDecodeError:
+                data = ''
+            raise ImporterError(
+                "Erreur d'import %s %s, contexte : %s, erreur : %s"
+                % (unicode(cls), unicode("__".join(path)),
+                   unicode(data), message))
+        return obj, created
 
     def _format_csv_line(self, values, empty=u"-"):
         return u'"' + u'","'.join(
diff --git a/ishtar_common/fixtures/initial_importtypes-fr.json b/ishtar_common/fixtures/initial_importtypes-fr.json
index 1b155ddf6..9ee6710c4 100644
--- a/ishtar_common/fixtures/initial_importtypes-fr.json
+++ b/ishtar_common/fixtures/initial_importtypes-fr.json
@@ -100,7 +100,7 @@
             "description": "", 
             "created_models": [], 
             "is_template": true, 
-            "unicity_keys": "external_id", 
+            "unicity_keys": "code_patriarche", 
             "users": [], 
             "slug": "mcc-operations", 
             "associated_models": 6, 
@@ -5306,4 +5306,4 @@
             "force_new": false
         }
     }
-]
\ No newline at end of file
+]
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 7ab09f9f7..e2c0e5db5 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super(TargetKeyForm, self).__init__(*args, **kwargs)
         instance = getattr(self, 'instance', None)
+        self.associated_import = None
         if instance and instance.pk:
+            self.associated_import = instance.associated_import
             self.fields['target'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['title'] = unicode(instance)
@@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm):
         super(TargetKeyForm, self).save(commit)
         if self.cleaned_data.get('value'):
             self.instance.is_set = True
-            self.associated_import = None
+            self.instance.associated_import = self.associated_import
             self.instance.save()
 
 
@@ -579,6 +581,7 @@ class AccountForm(forms.Form):
         widget=forms.PasswordInput, required=False)
 
     def __init__(self, *args, **kwargs):
+        person = None
         if 'initial' in kwargs and 'pk' in kwargs['initial']:
             try:
                 person = models.Person.objects.get(pk=kwargs['initial']['pk'])
@@ -589,7 +592,12 @@ class AccountForm(forms.Form):
                     kwargs['initial']['email'] = account.email
             except ObjectDoesNotExist:
                 pass
-        return super(AccountForm, self).__init__(*args, **kwargs)
+        if 'person' in kwargs:
+            person = kwargs.pop('person')
+        super(AccountForm, self).__init__(*args, **kwargs)
+        if person:
+            self.fields['username'].initial = \
+                person.raw_name.lower().replace(' ', '.')
 
     def clean(self):
         cleaned_data = self.cleaned_data
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 90e2bd6f6..2496e4372 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -22,6 +22,7 @@ Models description
 """
 from cStringIO import StringIO
 import copy
+import csv
 import datetime
 from PIL import Image
 from importlib import import_module
@@ -72,17 +73,18 @@ logger = logging.getLogger(__name__)
 
 def post_save_user(sender, **kwargs):
     user = kwargs['instance']
+
     try:
         q = IshtarUser.objects.filter(username=user.username)
         if not q.count():
             ishtaruser = IshtarUser.create_from_user(user)
         else:
             ishtaruser = q.all()[0]
-        ADMINISTRATOR, created = PersonType.objects.get_or_create(
+        administrator, created = PersonType.objects.get_or_create(
             txt_idx='administrator')
         if ishtaruser.is_superuser \
            and not ishtaruser.has_right('administrator'):
-            ishtaruser.person.person_types.add(ADMINISTRATOR)
+            ishtaruser.person.person_types.add(administrator)
     except DatabaseError:  # manage when db is not synced
         pass
 post_save.connect(post_save_user, sender=User)
@@ -648,27 +650,34 @@ class GeneralType(Cached, models.Model):
         self.generate_key(force=True)
         return obj
 
-    def add_key(self, key, force=False):
+    def add_key(self, key, force=False, importer=None):
         content_type = ContentType.objects.get_for_model(self.__class__)
-        if not force and ItemKey.objects.filter(
+        if not importer and not force and ItemKey.objects.filter(
                 key=key, content_type=content_type).count():
             return
         if force:
-            ItemKey.objects.filter(key=key, content_type=content_type)\
+            ItemKey.objects.filter(key=key, content_type=content_type,
+                                   importer=importer)\
                            .exclude(object_id=self.pk).delete()
-        ItemKey.objects.get_or_create(object_id=self.pk, key=key,
-                                      content_type=content_type)
+        ItemKey.objects.get_or_create(
+            object_id=self.pk, key=key, content_type=content_type,
+            importer=importer
+        )
 
     def generate_key(self, force=False):
         for key in (slugify(self.label), self.txt_idx):
             self.add_key(key)
 
-    def get_keys(self):
+    def get_keys(self, importer_id=None):
         keys = [self.txt_idx]
         content_type = ContentType.objects.get_for_model(self.__class__)
-        for ik in ItemKey.objects.filter(
-                content_type=content_type, object_id=self.pk).exclude(
-                key=self.txt_idx).all():
+        query = Q(content_type=content_type, object_id=self.pk,
+                  importer__isnull=True)
+        if importer_id:
+            query |= Q(content_type=content_type, object_id=self.pk,
+                       importer__pk=importer_id)
+        q = ItemKey.objects.filter(query)
+        for ik in q.exclude(key=self.txt_idx).all():
             keys.append(ik.key)
         return keys
 
@@ -1782,7 +1791,7 @@ class ImporterType(models.Model):
     def __unicode__(self):
         return self.name
 
-    def get_importer_class(self):
+    def get_importer_class(self, import_instance=None):
         if self.slug and self.slug in IMPORTER_CLASSES:
             cls = import_class(IMPORTER_CLASSES[self.slug])
             return cls
@@ -1805,7 +1814,8 @@ class ImporterType(models.Model):
             force_news = []
             concat_str = []
             for target in column.targets.all():
-                ft = target.formater_type.get_formater_type(target)
+                ft = target.formater_type.get_formater_type(
+                    target, import_instance=import_instance)
                 if not ft:
                     continue
                 formater_types.append(ft)
@@ -2106,10 +2116,10 @@ class TargetKey(models.Model):
                 try:
                     v = self.target.associated_model.objects.get(
                         txt_idx=unicode(self.value))
-                except (self.target.associated_model.DoesNotExist):
+                except self.target.associated_model.DoesNotExist:
                     pass
             if v:
-                v.add_key(self.key)
+                v.add_key(self.key, importer=self.associated_import)
         return obj
 
 TARGET_MODELS = [
@@ -2203,10 +2213,10 @@ class FormaterType(models.Model):
         if self.format_type in IMPORTER_TYPES_CHOICES:
             return IMPORTER_TYPES_CHOICES[self.format_type]
 
-    def get_formater_type(self, target):
+    def get_formater_type(self, target, import_instance=None):
         if self.formater_type not in IMPORTER_TYPES_DCT.keys():
             return
-        kwargs = {'db_target': target}
+        kwargs = {'db_target': target, 'import_instance': import_instance}
         if self.many_split:
             kwargs['many_split'] = self.many_split
         if self.formater_type == 'TypeFormater':
@@ -2302,6 +2312,19 @@ class Import(models.Model):
         return bool(TargetKey.objects.filter(associated_import=self,
                                              is_set=False).count())
 
+    @property
+    def errors(self):
+        if not self.error_file:
+            return []
+        errors = []
+        with open(self.error_file.path, 'rb') as csvfile:
+            reader = csv.DictReader(csvfile, fieldnames=['line', 'column',
+                                                         'error'])
+            reader.next()  # pass the header
+            for row in reader:
+                errors.append(row)
+        return errors
+
     def get_actions(self):
         """
         Get available action relevant with the current status
@@ -2332,7 +2355,7 @@ class Import(models.Model):
         return IMPORT_STATE_DCT[self.state]
 
     def get_importer_instance(self):
-        return self.importer_type.get_importer_class()(
+        return self.importer_type.get_importer_class(import_instance=self)(
             skip_lines=self.skip_lines, import_instance=self,
             conservative_import=self.conservative_import)
 
@@ -2783,8 +2806,10 @@ class IshtarUser(User):
                                        name=name, email=email,
                                        history_modifier=user)
         person.person_types.add(person_type)
-        return IshtarUser.objects.create(user_ptr=user, username=default,
-                                         person=person)
+        password = user.password
+        isht_user = IshtarUser.objects.create(
+            user_ptr=user, username=default, person=person, password=password)
+        return isht_user
 
     def has_right(self, right_name, session=None):
         return self.person.has_right(right_name, session=session)
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 82acb1904..42bb1860e 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -319,6 +319,15 @@ class AdminGenTypeTest(TestCase):
         for model in self.models_with_data:
             self.assertTrue(str(model.objects.all()[0]))
 
+    def test_user_creation(self):
+        url = '/admin/auth/user/add/'
+        password = 'ishtar is the queen'
+        response = self.client.post(
+            url, {'username': 'test', 'password1': password,
+                  'password2': password})
+        self.assertEqual(response.status_code, 302)
+        self.assertTrue(self.client.login(username='test', password=password))
+
 
 class MergeTest(TestCase):
     def setUp(self):
@@ -798,7 +807,7 @@ class ImportTest(TestCase):
         # town should be deleted
         self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0)
 
-    def testKeys(self):
+    def test_keys(self):
         content_type = ContentType.objects.get_for_model(
             models.OrganizationType)
 
diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py
index 5105418a5..c2cd54d03 100644
--- a/ishtar_common/wizards.py
+++ b/ishtar_common/wizards.py
@@ -1343,6 +1343,12 @@ class AccountWizard(Wizard):
                                  context_instance=RequestContext(self.request))
         return res
 
+    def get_form_kwargs(self, step=None):
+        kwargs = super(AccountWizard, self).get_form_kwargs(step)
+        if step == 'account-account_management':
+            kwargs['person'] = self.get_current_object()
+        return kwargs
+
     def get_form(self, step=None, data=None, files=None):
         """
         Display the "Send email" field if necessary
diff --git a/version.py b/version.py
index 8fdddac48..71bcbeeac 100644
--- a/version.py
+++ b/version.py
@@ -1,4 +1,4 @@
-VERSION = (0, 99, 12, 2)
+VERSION = (0, 99, 13)
 
 
 def get_version():