Item keys are now related to the current importer (refs #3451)

5 files changed, 105 insertions, 60 deletions

diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 7c141af93..296236029 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -34,7 +34,7 @@ import models
 
 from archaeological_operations import views
 
-from ishtar_common.models import OrganizationType, Organization, \
+from ishtar_common.models import OrganizationType, Organization, ItemKey, \
     ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, \
     Town, ImporterColumn
 from archaeological_context_records.models import Unit
@@ -74,26 +74,30 @@ class ImportTest(object):
 
     def init_ope_targetkey(self, imp):
         # doing manually connections
-        tg = TargetKey.objects.filter(target__target='operation_type'
-                                      ).order_by('-pk').all()[0]
-        tg.value = models.OperationType.objects.get(
+        target = TargetKey.objects.filter(
+            target__target='operation_type').order_by('-pk').all()[0]
+        target.value = models.OperationType.objects.get(
             txt_idx='prog_excavation').pk
-        tg.is_set = True
-        tg.save()
-
-        target = TargetKey.objects.get(key='gallo-romain',
-                                       associated_import=imp)
-        gallo = models.Period.objects.get(txt_idx='gallo-roman')
-        target.value = gallo.pk
         target.is_set = True
+        target.associated_import = imp
         target.save()
 
-        target = TargetKey.objects.get(key='age-du-fer',
-                                       associated_import=imp)
+        target2 = TargetKey.objects.get(key='gallo-romain',
+                                        associated_import=imp)
+        gallo = models.Period.objects.get(txt_idx='gallo-roman')
+        target2.value = gallo.pk
+        target2.is_set = True
+        target2.associated_import = imp
+        target2.save()
+
+        target3 = TargetKey.objects.get(key='age-du-fer',
+                                        associated_import=imp)
         iron = models.Period.objects.get(txt_idx='iron_age')
-        target.value = iron.pk
-        target.is_set = True
-        target.save()
+        target3.value = iron.pk
+        target3.is_set = True
+        target3.associated_import = imp
+        target3.save()
+        return [target, target2, target3]
 
     def init_ope(self):
         importer, form = self.init_ope_import()
@@ -186,7 +190,7 @@ class ImportOperationTest(ImportTest, TestCase):
         impt.importation()
         # a new operation has now been imported
         current_ope_nb = models.Operation.objects.count()
-        self.assertTrue(current_ope_nb == (first_ope_nb + 1))
+        self.assertEqual(current_ope_nb, first_ope_nb + 1)
         # and well imported
         last_ope = models.Operation.objects.order_by('-pk').all()[0]
         self.assertEqual(last_ope.name, u"Oppìdum de Paris")
@@ -200,9 +204,28 @@ class ImportOperationTest(ImportTest, TestCase):
         # a second importation will be not possible: no two same patriarche
         # code
         impt.importation()
-        models.Operation.objects.count()
-        self.assertTrue(last_ope ==
-                        models.Operation.objects.order_by('-pk').all()[0])
+        self.assertEqual(last_ope,
+                         models.Operation.objects.order_by('-pk').all()[0])
+
+    def test_keys_limitation(self):
+        # each key association is associated to the import
+        init_ope_number = models.Operation.objects.count()
+        importer, form = self.init_ope_import()
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        self.init_ope_targetkey(imp=impt)
+
+        importer, form = self.init_ope_import()
+        other_imp = form.save(self.ishtar_user)
+        # associate with another import
+        for ik in ItemKey.objects.filter(importer=impt).all():
+            ik.importer = other_imp
+            ik.save()
+
+        impt.importation()
+        current_ope_nb = models.Operation.objects.count()
+        # no new operation
+        self.assertEqual(current_ope_nb, init_ope_number)
 
     def test_bad_configuration(self):
         importer, form = self.init_ope_import()
@@ -809,7 +832,6 @@ class OperationSearchTest(TestCase, OperationInitTest):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(json.loads(response.content)['total'], 1)
 
-
     def testOwnSearch(self):
         c = Client()
         response = c.get(reverse('get-operation'), {'year': '2010'})
@@ -911,12 +933,10 @@ class OperationWizardCreationTest(WizardTest, OperationInitTest, TestCase):
             return super(OperationWizardCreationTest, self).pre_wizard()
         town = self.create_towns()[0]
         town_data = {'town': town.pk}
-        self.form_datas[0].form_datas['townsgeneral-operation_creation'].append(
-            town_data
-        )
-        self.form_datas[1].form_datas['townsgeneral-operation_creation'].append(
-            town_data
-        )
+        self.form_datas[0].form_datas[
+            'townsgeneral-operation_creation'].append(town_data)
+        self.form_datas[1].form_datas[
+            'townsgeneral-operation_creation'].append(town_data)
         parcel_data = {
             'town': town.pk, 'year': 2017, 'section': 'S',
             'parcel_number': '42'}
@@ -997,7 +1017,8 @@ class OperationWizardClosingTest(OperationWizardCreationTest):
         self.assertFalse(ope.is_active())
         self.assertEqual(
             ope.closing()['date'].strftime('%Y-%d-%m'),
-            self.form_datas[0].form_datas['date-operation_closing']['end_date'])
+            self.form_datas[0].form_datas['date-operation_closing']['end_date']
+        )
 
 
 class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest,
@@ -1017,12 +1038,11 @@ class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest,
         FormData(
             "Admin act creation",
             form_datas={
-                    'selec-operation_administrativeactop': {
-                    },
-                    'administrativeact-operation_administrativeactop': {
-                        'signature_date': str(datetime.date.today())
-                    }
-                },
+                'selec-operation_administrativeactop': {},
+                'administrativeact-operation_administrativeactop': {
+                    'signature_date': str(datetime.date.today())
+                }
+            },
         )
     ]
 
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 0b6662e1c..fbf19def2 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -32,6 +32,7 @@ from django.contrib.auth.models import User
 from django.db.models.fields import FieldDoesNotExist
 from django.core.files import File
 from django.db import IntegrityError, DatabaseError, transaction
+from django.db.models import Q
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
@@ -148,13 +149,15 @@ class ChoiceChecker(object):
 
 class UnicodeFormater(Formater):
     def __init__(self, max_length=None, clean=False, re_filter=None,
-                 notnull=False, prefix=u'', db_target=None):
+                 notnull=False, prefix=u'', db_target=None,
+                 import_instance=None):
         self.max_length = max_length
         self.db_target = db_target
         self.clean = clean
         self.re_filter = re_filter
         self.notnull = notnull
         self.prefix = prefix
+        self.import_instance = import_instance
 
     def format(self, value):
         try:
@@ -255,7 +258,8 @@ class IntegerFormater(Formater):
 
 class StrChoiceFormater(Formater, ChoiceChecker):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
-                 cli=False, many_split='', db_target=None):
+                 cli=False, many_split='', db_target=None,
+                 import_instance=None):
         self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
@@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         self.new_keys = {}
         self.match_table = {}
         self.many_split = many_split
+        self.import_instance = None
         for key, value in self.choices:
             value = unicode(value)
             if not self.strict:
@@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker):
     def init_db_target(self):
         if not self.db_target:
             return
-        for target_key in self.db_target.keys.filter(is_set=True).all():
+        q = self.db_target.keys.filter(is_set=True)
+        if self.import_instance:
+            q = q.filter(Q(associated_import=self.import_instance) |
+                         Q(associated_import__isnull=True))
+        for target_key in q.all():
             key = target_key.key
             if not self.strict:
                 key = slugify(key)
@@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
 
 class TypeFormater(StrChoiceFormater):
     def __init__(self, model, cli=False, defaults={}, many_split=False,
-                 db_target=None):
+                 db_target=None, import_instance=None):
         self.create = True
         self.strict = False
         self.model = model
@@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater):
         self.equiv_dict, self.choices = {}, []
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
         for item in model.objects.all():
             self.choices.append((item.pk, unicode(item)))
-            for key in item.get_keys():
+            for key in item.get_keys(importer_id=import_instance.pk):
                 self.equiv_dict[key] = item
 
     def prepare(self, value):
@@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater):
 
 
 class DateFormater(Formater):
-    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None):
+    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None,
+                 import_instance=None):
         self.date_formats = date_formats
         if type(date_formats) not in (list, tuple):
             self.date_formats = [self.date_formats]
         self.db_target = db_target
+        self.import_instance = import_instance
 
     def format(self, value):
         value = value.strip()
@@ -511,7 +523,8 @@ class FileFormater(Formater):
 
 
 class StrToBoolean(Formater, ChoiceChecker):
-    def __init__(self, choices={}, cli=False, strict=False, db_target=None):
+    def __init__(self, choices={}, cli=False, strict=False, db_target=None,
+                 import_instance=None):
         self.dct = copy.copy(choices)
         self.cli = cli
         self.strict = strict
@@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker):
         self.init_db_target()
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
 
     def init_db_target(self):
         if not self.db_target:
@@ -827,7 +841,7 @@ class Importer(object):
                 vals[idx_col].append(val)
         for idx, formater in enumerate(self.line_format):
             if formater and idx < len(vals):
-
+                formater.import_instance = self.import_instance
                 if self.DB_TARGETS:
                     field_names = formater.field_name
                     if type(field_names) not in (list, tuple):
@@ -1143,6 +1157,7 @@ class Importer(object):
                 self.concat_str[field_name] = concat_str
 
             if self.DB_TARGETS:
+                formater.import_instance = self.import_instance
                 formater.reinit_db_target(
                     self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)],
                     idx_v)
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 7ab09f9f7..67299fbac 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super(TargetKeyForm, self).__init__(*args, **kwargs)
         instance = getattr(self, 'instance', None)
+        self.associated_import = None
         if instance and instance.pk:
+            self.associated_import = instance.associated_import
             self.fields['target'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['title'] = unicode(instance)
@@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm):
         super(TargetKeyForm, self).save(commit)
         if self.cleaned_data.get('value'):
             self.instance.is_set = True
-            self.associated_import = None
+            self.instance.associated_import = self.associated_import
             self.instance.save()
 
 
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index c1edc13c6..fa2b348bf 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -649,27 +649,34 @@ class GeneralType(Cached, models.Model):
         self.generate_key(force=True)
         return obj
 
-    def add_key(self, key, force=False):
+    def add_key(self, key, force=False, importer=None):
         content_type = ContentType.objects.get_for_model(self.__class__)
-        if not force and ItemKey.objects.filter(
+        if not importer and not force and ItemKey.objects.filter(
                 key=key, content_type=content_type).count():
             return
         if force:
-            ItemKey.objects.filter(key=key, content_type=content_type)\
+            ItemKey.objects.filter(key=key, content_type=content_type,
+                                   importer=importer)\
                            .exclude(object_id=self.pk).delete()
-        ItemKey.objects.get_or_create(object_id=self.pk, key=key,
-                                      content_type=content_type)
+        ItemKey.objects.get_or_create(
+            object_id=self.pk, key=key, content_type=content_type,
+            importer=importer
+        )
 
     def generate_key(self, force=False):
         for key in (slugify(self.label), self.txt_idx):
             self.add_key(key)
 
-    def get_keys(self):
+    def get_keys(self, importer_id=None):
         keys = [self.txt_idx]
         content_type = ContentType.objects.get_for_model(self.__class__)
-        for ik in ItemKey.objects.filter(
-                content_type=content_type, object_id=self.pk).exclude(
-                key=self.txt_idx).all():
+        query = Q(content_type=content_type, object_id=self.pk,
+                  importer__isnull=True)
+        if importer_id:
+            query |= Q(content_type=content_type, object_id=self.pk,
+                       importer__pk=importer_id)
+        q = ItemKey.objects.filter(query)
+        for ik in q.exclude(key=self.txt_idx).all():
             keys.append(ik.key)
         return keys
 
@@ -1783,7 +1790,7 @@ class ImporterType(models.Model):
     def __unicode__(self):
         return self.name
 
-    def get_importer_class(self):
+    def get_importer_class(self, import_instance=None):
         if self.slug and self.slug in IMPORTER_CLASSES:
             cls = import_class(IMPORTER_CLASSES[self.slug])
             return cls
@@ -1806,7 +1813,8 @@ class ImporterType(models.Model):
             force_news = []
             concat_str = []
             for target in column.targets.all():
-                ft = target.formater_type.get_formater_type(target)
+                ft = target.formater_type.get_formater_type(
+                    target, import_instance=import_instance)
                 if not ft:
                     continue
                 formater_types.append(ft)
@@ -2107,10 +2115,10 @@ class TargetKey(models.Model):
                 try:
                     v = self.target.associated_model.objects.get(
                         txt_idx=unicode(self.value))
-                except (self.target.associated_model.DoesNotExist):
+                except self.target.associated_model.DoesNotExist:
                     pass
             if v:
-                v.add_key(self.key)
+                v.add_key(self.key, importer=self.associated_import)
         return obj
 
 TARGET_MODELS = [
@@ -2204,10 +2212,10 @@ class FormaterType(models.Model):
         if self.format_type in IMPORTER_TYPES_CHOICES:
             return IMPORTER_TYPES_CHOICES[self.format_type]
 
-    def get_formater_type(self, target):
+    def get_formater_type(self, target, import_instance=None):
         if self.formater_type not in IMPORTER_TYPES_DCT.keys():
             return
-        kwargs = {'db_target': target}
+        kwargs = {'db_target': target, 'import_instance': import_instance}
         if self.many_split:
             kwargs['many_split'] = self.many_split
         if self.formater_type == 'TypeFormater':
@@ -2346,7 +2354,7 @@ class Import(models.Model):
         return IMPORT_STATE_DCT[self.state]
 
     def get_importer_instance(self):
-        return self.importer_type.get_importer_class()(
+        return self.importer_type.get_importer_class(import_instance=self)(
             skip_lines=self.skip_lines, import_instance=self,
             conservative_import=self.conservative_import)
 
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 82acb1904..8953e3c1b 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -798,7 +798,7 @@ class ImportTest(TestCase):
         # town should be deleted
         self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0)
 
-    def testKeys(self):
+    def test_keys(self):
         content_type = ContentType.objects.get_for_model(
             models.OrganizationType)