Item keys are now related to the current importer (refs #3451)

4 files changed, 51 insertions, 26 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 0b6662e1c..fbf19def2 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -32,6 +32,7 @@ from django.contrib.auth.models import User
 from django.db.models.fields import FieldDoesNotExist
 from django.core.files import File
 from django.db import IntegrityError, DatabaseError, transaction
+from django.db.models import Q
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
@@ -148,13 +149,15 @@ class ChoiceChecker(object):
 
 class UnicodeFormater(Formater):
     def __init__(self, max_length=None, clean=False, re_filter=None,
-                 notnull=False, prefix=u'', db_target=None):
+                 notnull=False, prefix=u'', db_target=None,
+                 import_instance=None):
         self.max_length = max_length
         self.db_target = db_target
         self.clean = clean
         self.re_filter = re_filter
         self.notnull = notnull
         self.prefix = prefix
+        self.import_instance = import_instance
 
     def format(self, value):
         try:
@@ -255,7 +258,8 @@ class IntegerFormater(Formater):
 
 class StrChoiceFormater(Formater, ChoiceChecker):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
-                 cli=False, many_split='', db_target=None):
+                 cli=False, many_split='', db_target=None,
+                 import_instance=None):
         self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
@@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         self.new_keys = {}
         self.match_table = {}
         self.many_split = many_split
+        self.import_instance = None
         for key, value in self.choices:
             value = unicode(value)
             if not self.strict:
@@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker):
     def init_db_target(self):
         if not self.db_target:
             return
-        for target_key in self.db_target.keys.filter(is_set=True).all():
+        q = self.db_target.keys.filter(is_set=True)
+        if self.import_instance:
+            q = q.filter(Q(associated_import=self.import_instance) |
+                         Q(associated_import__isnull=True))
+        for target_key in q.all():
             key = target_key.key
             if not self.strict:
                 key = slugify(key)
@@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
 
 class TypeFormater(StrChoiceFormater):
     def __init__(self, model, cli=False, defaults={}, many_split=False,
-                 db_target=None):
+                 db_target=None, import_instance=None):
         self.create = True
         self.strict = False
         self.model = model
@@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater):
         self.equiv_dict, self.choices = {}, []
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
         for item in model.objects.all():
             self.choices.append((item.pk, unicode(item)))
-            for key in item.get_keys():
+            for key in item.get_keys(importer_id=import_instance.pk):
                 self.equiv_dict[key] = item
 
     def prepare(self, value):
@@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater):
 
 
 class DateFormater(Formater):
-    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None):
+    def __init__(self, date_formats=["%d/%m/%Y"], db_target=None,
+                 import_instance=None):
         self.date_formats = date_formats
         if type(date_formats) not in (list, tuple):
             self.date_formats = [self.date_formats]
         self.db_target = db_target
+        self.import_instance = import_instance
 
     def format(self, value):
         value = value.strip()
@@ -511,7 +523,8 @@ class FileFormater(Formater):
 
 
 class StrToBoolean(Formater, ChoiceChecker):
-    def __init__(self, choices={}, cli=False, strict=False, db_target=None):
+    def __init__(self, choices={}, cli=False, strict=False, db_target=None,
+                 import_instance=None):
         self.dct = copy.copy(choices)
         self.cli = cli
         self.strict = strict
@@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker):
         self.init_db_target()
         self.match_table = {}
         self.new_keys = {}
+        self.import_instance = import_instance
 
     def init_db_target(self):
         if not self.db_target:
@@ -827,7 +841,7 @@ class Importer(object):
                 vals[idx_col].append(val)
         for idx, formater in enumerate(self.line_format):
             if formater and idx < len(vals):
-
+                formater.import_instance = self.import_instance
                 if self.DB_TARGETS:
                     field_names = formater.field_name
                     if type(field_names) not in (list, tuple):
@@ -1143,6 +1157,7 @@ class Importer(object):
                 self.concat_str[field_name] = concat_str
 
             if self.DB_TARGETS:
+                formater.import_instance = self.import_instance
                 formater.reinit_db_target(
                     self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)],
                     idx_v)
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 7ab09f9f7..67299fbac 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super(TargetKeyForm, self).__init__(*args, **kwargs)
         instance = getattr(self, 'instance', None)
+        self.associated_import = None
         if instance and instance.pk:
+            self.associated_import = instance.associated_import
             self.fields['target'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['readonly'] = True
             self.fields['key'].widget.attrs['title'] = unicode(instance)
@@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm):
         super(TargetKeyForm, self).save(commit)
         if self.cleaned_data.get('value'):
             self.instance.is_set = True
-            self.associated_import = None
+            self.instance.associated_import = self.associated_import
             self.instance.save()
 
 
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index c1edc13c6..fa2b348bf 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -649,27 +649,34 @@ class GeneralType(Cached, models.Model):
         self.generate_key(force=True)
         return obj
 
-    def add_key(self, key, force=False):
+    def add_key(self, key, force=False, importer=None):
         content_type = ContentType.objects.get_for_model(self.__class__)
-        if not force and ItemKey.objects.filter(
+        if not importer and not force and ItemKey.objects.filter(
                 key=key, content_type=content_type).count():
             return
         if force:
-            ItemKey.objects.filter(key=key, content_type=content_type)\
+            ItemKey.objects.filter(key=key, content_type=content_type,
+                                   importer=importer)\
                            .exclude(object_id=self.pk).delete()
-        ItemKey.objects.get_or_create(object_id=self.pk, key=key,
-                                      content_type=content_type)
+        ItemKey.objects.get_or_create(
+            object_id=self.pk, key=key, content_type=content_type,
+            importer=importer
+        )
 
     def generate_key(self, force=False):
         for key in (slugify(self.label), self.txt_idx):
             self.add_key(key)
 
-    def get_keys(self):
+    def get_keys(self, importer_id=None):
         keys = [self.txt_idx]
         content_type = ContentType.objects.get_for_model(self.__class__)
-        for ik in ItemKey.objects.filter(
-                content_type=content_type, object_id=self.pk).exclude(
-                key=self.txt_idx).all():
+        query = Q(content_type=content_type, object_id=self.pk,
+                  importer__isnull=True)
+        if importer_id:
+            query |= Q(content_type=content_type, object_id=self.pk,
+                       importer__pk=importer_id)
+        q = ItemKey.objects.filter(query)
+        for ik in q.exclude(key=self.txt_idx).all():
             keys.append(ik.key)
         return keys
 
@@ -1783,7 +1790,7 @@ class ImporterType(models.Model):
     def __unicode__(self):
         return self.name
 
-    def get_importer_class(self):
+    def get_importer_class(self, import_instance=None):
         if self.slug and self.slug in IMPORTER_CLASSES:
             cls = import_class(IMPORTER_CLASSES[self.slug])
             return cls
@@ -1806,7 +1813,8 @@ class ImporterType(models.Model):
             force_news = []
             concat_str = []
             for target in column.targets.all():
-                ft = target.formater_type.get_formater_type(target)
+                ft = target.formater_type.get_formater_type(
+                    target, import_instance=import_instance)
                 if not ft:
                     continue
                 formater_types.append(ft)
@@ -2107,10 +2115,10 @@ class TargetKey(models.Model):
                 try:
                     v = self.target.associated_model.objects.get(
                         txt_idx=unicode(self.value))
-                except (self.target.associated_model.DoesNotExist):
+                except self.target.associated_model.DoesNotExist:
                     pass
             if v:
-                v.add_key(self.key)
+                v.add_key(self.key, importer=self.associated_import)
         return obj
 
 TARGET_MODELS = [
@@ -2204,10 +2212,10 @@ class FormaterType(models.Model):
         if self.format_type in IMPORTER_TYPES_CHOICES:
             return IMPORTER_TYPES_CHOICES[self.format_type]
 
-    def get_formater_type(self, target):
+    def get_formater_type(self, target, import_instance=None):
         if self.formater_type not in IMPORTER_TYPES_DCT.keys():
             return
-        kwargs = {'db_target': target}
+        kwargs = {'db_target': target, 'import_instance': import_instance}
         if self.many_split:
             kwargs['many_split'] = self.many_split
         if self.formater_type == 'TypeFormater':
@@ -2346,7 +2354,7 @@ class Import(models.Model):
         return IMPORT_STATE_DCT[self.state]
 
     def get_importer_instance(self):
-        return self.importer_type.get_importer_class()(
+        return self.importer_type.get_importer_class(import_instance=self)(
             skip_lines=self.skip_lines, import_instance=self,
             conservative_import=self.conservative_import)
 
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 82acb1904..8953e3c1b 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -798,7 +798,7 @@ class ImportTest(TestCase):
         # town should be deleted
         self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0)
 
-    def testKeys(self):
+    def test_keys(self):
         content_type = ContentType.objects.get_for_model(
             models.OrganizationType)