Importers: fix target key initialization (manage target key not related to current import) - refs #3725

3 files changed, 69 insertions, 35 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index fa8c6a2e0..b3fb682ea 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -91,20 +91,20 @@ class ImportFormater(object):
         self.force_new = force_new
         self.label = label
 
-    def reinit_db_target(self, db_target, nb=0):
+    def reinit_db_target(self, db_target, nb=0, user=None):
         if not self.formater:
             return
         if type(db_target) in (list, tuple):
             db_target = db_target[nb]
         if type(self.formater) not in (list, tuple):
             self.formater.db_target = db_target
-            self.formater.init_db_target()
+            self.formater.init_db_target(user=user)
         else:
             for idx, formater in enumerate(self.formater):
                 formater.db_target = db_target
-                formater.init_db_target()
+                formater.init_db_target(user=user)
 
-    def init_db_target(self):
+    def init_db_target(self, user=None):
         pass
 
     def __unicode__(self):
@@ -117,7 +117,7 @@ class ImportFormater(object):
         return
 
     def init(self, vals, output=None, choose_default=False,
-             import_instance=None):
+             import_instance=None, user=None):
         try:
             lst = iter(self.formater)
         except TypeError:
@@ -126,7 +126,8 @@ class ImportFormater(object):
             if formater:
                 formater.check(vals, output, self.comment,
                                choose_default=choose_default,
-                               import_instance=import_instance)
+                               import_instance=import_instance,
+                               user=user)
 
     def post_process(self, obj, context, value, owner=None):
         raise NotImplemented()
@@ -152,12 +153,39 @@ class Formater(object):
         return value
 
     def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None):
+              import_instance=None, user=None):
         return
 
-    def init_db_target(self):
+    def init_db_target(self, user=None):
         pass
 
+    def _base_target_filter(self, user=None):
+        # set for all users
+        q_or = (
+            Q(associated_import__isnull=True) &
+            Q(associated_user__isnull=True) &
+            Q(associated_group__isnull=True)
+        )
+        if hasattr(self, 'import_instance') and self.import_instance:
+            # set for current import
+            q_or = q_or | Q(associated_import=self.import_instance)
+            if self.import_instance.associated_group:
+                # set for associated group
+                q_or = q_or | Q(
+                    associated_group=self.import_instance.associated_group)
+        if user:
+            # set for current user
+            q_or = q_or | Q(associated_user=user)
+        return q_or
+
+    def get_db_target_query(self, user=None):
+        if not self.db_target:
+            return
+        q = self.db_target.keys.filter(is_set=True)
+        q_or = self._base_target_filter(user)
+        q = q.filter(q_or)
+        return q
+
 
 class ChoiceChecker(object):
     def report_new(self, comment):
@@ -307,13 +335,12 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                 self.equiv_dict[value] = v
         self.init_db_target()
 
-    def init_db_target(self):
+    def init_db_target(self, user=None):
         if not self.db_target:
             return
-        q = self.db_target.keys.filter(is_set=True)
-        if self.import_instance:
-            q = q.filter(Q(associated_import=self.import_instance) |
-                         Q(associated_import__isnull=True))
+
+        q = self.get_db_target_query(user)
+
         for target_key in q.all():
             key = target_key.key
             if not self.strict:
@@ -347,7 +374,8 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         return msgstr, idx
 
     def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None):
+              import_instance=None, user=None):
+        """
         from ishtar_common.models import TargetKey
         if self.db_target:
             q = {'target': self.db_target,
@@ -365,7 +393,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                         t, created = TargetKey.objects.get_or_create(**q)
                     except IntegrityError:
                         pass
-
+        """
         if (not output or output == 'silent') and not choose_default:
             return
         if self.many_split:
@@ -412,16 +440,16 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                 self.equiv_dict[value] = None
             if self.equiv_dict[value] and self.db_target:
                 from ishtar_common.models import TargetKey
-                q = {'target': self.db_target, 'key': value,
-                     'associated_import': import_instance,
-                     }
+                q = {'target': self.db_target, 'key': value}
                 query = TargetKey.objects.filter(**q)
+                query = query.filter(self._base_target_filter(user))
                 if query.count():
                     target = query.all()[0]
                     target.value = self.equiv_dict[value]
                     target.is_set = True
                     target.save()
                 else:
+                    q['associated_import'] = import_instance
                     with transaction.atomic():
                         q['value'] = self.equiv_dict[value]
                         q['is_set'] = True
@@ -432,11 +460,13 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         if output == 'db' and self.db_target:
             from ishtar_common.models import TargetKey
             for missing in self.missings:
-                q = {'target': self.db_target, 'key': missing,
-                     'associated_import': import_instance}
-                if TargetKey.objects.filter(**q).count():
+                q = {'target': self.db_target, 'key': missing}
+                query = TargetKey.objects.filter(**q)
+                query = query.filter(self._base_target_filter(user))
+                if query.count():
                     continue
                 with transaction.atomic():
+                    q['associated_import'] = import_instance
                     try:
                         TargetKey.objects.create(**q)
                     except IntegrityError:
@@ -555,15 +585,16 @@ class StrToBoolean(Formater, ChoiceChecker):
         self.strict = strict
         self.db_target = db_target
         self.missings = set()
-        self.init_db_target()
         self.match_table = {}
         self.new_keys = {}
         self.import_instance = import_instance
+        self.init_db_target()
 
-    def init_db_target(self):
+    def init_db_target(self, user=None):
         if not self.db_target:
             return
-        for target_key in self.db_target.keys.filter(is_set=True).all():
+        q = self.get_db_target_query(user)
+        for target_key in q.all():
             key = self.prepare(target_key.key)
             if key in self.dct:
                 continue
@@ -577,7 +608,7 @@ class StrToBoolean(Formater, ChoiceChecker):
         return value
 
     def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None):
+              import_instance=None, user=None):
         if (not output or output == 'silent') and not choose_default:
             return
         msgstr = comment + u" - "
@@ -846,14 +877,16 @@ class Importer(object):
             getattr(item, func)(context, value)
         return item
 
-    def initialize(self, table, output='silent', choose_default=False):
+    def initialize(self, table, output='silent', choose_default=False,
+                   user=None):
         """
         copy vals in columns and initialize formaters
         * output:
-         - 'silent': no associations
-         - 'cli': output by command line interface and stocked in the database
-         - 'db': output on the database with no interactive association
+         - silent: no associations
+         - cli: output by command line interface and stocked in the database
+         - db: output on the database with no interactive association
            (further exploitation by web interface)
+         - user: associated user
         """
         assert output in ('silent', 'cli', 'db')
         vals = []
@@ -878,10 +911,11 @@ class Importer(object):
                         db_targets.append(
                             self.DB_TARGETS["{}-{}".format(
                                 idx + 1, field_name)])
-                    formater.reinit_db_target(db_targets)
+                    formater.reinit_db_target(db_targets, user=user)
 
                 formater.init(vals[idx], output, choose_default=choose_default,
-                              import_instance=self.import_instance)
+                              import_instance=self.import_instance,
+                              user=user)
 
     def importation(self, table, initialize=True, choose_default=False):
         if initialize:
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 6a313a95d..a6fcd89c6 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -708,7 +708,6 @@ class FormaterType(models.Model):
                     "**WARN FormaterType.get_formater_type**: {} "
                     "is not in TARGET_MODELS_KEYS".format(self.options))
                 return
-            model = None
             if self.options in dir():
                 model = dir()[self.options]
             else:
@@ -882,10 +881,11 @@ class Import(models.Model):
             shutil.rmtree(tmpdir)
         return []
 
-    def initialize(self):
+    def initialize(self, user=None):
         self.state = 'AP'
         self.save()
-        self.get_importer_instance().initialize(self.data_table, output='db')
+        self.get_importer_instance().initialize(self.data_table,
+                                                user=user, output='db')
         self.state = 'A'
         self.save()
 
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index d2ff49a36..b31a8730e 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1707,7 +1707,7 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
                 return HttpResponseRedirect(reverse('import_delete',
                                                     kwargs={'pk': imprt.pk}))
             elif action == 'A':
-                imprt.initialize()
+                imprt.initialize(user=self.request.user.ishtaruser)
             elif action == 'I':
                 imprt.importation()
             elif action == 'AC':