Import: add management of link between user data and db data

1 files changed, 64 insertions, 10 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c2924a541..194a9a5fa 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -88,6 +88,9 @@ class ImporterError(Exception):
         return self.msg
 
 class Formater(object):
+    def __init__(self, *args, **kwargs):
+        self.db_target = kwargs.get('db_target', None)
+
     def format(self, value):
         return value
 
@@ -95,8 +98,10 @@ class Formater(object):
         return
 
 class UnicodeFormater(Formater):
-    def __init__(self, max_length, clean=False, re_filter=None, notnull=False):
+    def __init__(self, max_length, clean=False, re_filter=None, notnull=False,
+                 db_target=None):
         self.max_length = max_length
+        self.db_target = db_target
         self.clean = clean
         self.re_filter = re_filter
         self.notnull = notnull
@@ -182,12 +187,13 @@ class IntegerFormater(Formater):
 
 class StrChoiceFormater(Formater):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
-                 cli=False, many_split=''):
+                 cli=False, many_split='', db_target=None):
         self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
         self.cli = cli
         self.model = model
+        self.db_target = db_target
         self.create = False
         self.missings = set()
         self.many_split = many_split
@@ -200,6 +206,17 @@ class StrChoiceFormater(Formater):
                 if model and v:
                     v = model.objects.get(pk=v)
                 self.equiv_dict[value] = v
+        if self.db_target:
+            for target_key in self.db_target.keys.filter(is_set=True).all():
+                value = target_key.value
+                if not self.strict:
+                    value = slugify(value)
+                if value in self.equiv_dict:
+                    continue
+                v = target_key.key
+                if model and v:
+                    v = model.objects.get(pk=v)
+                self.equiv_dict[value] = v
 
     def prepare(self, value):
         return unicode(value).strip()
@@ -257,6 +274,13 @@ class StrChoiceFormater(Formater):
                                      unicode(self.equiv_dict[value])))
             else:
                 self.equiv_dict[value] = None
+        if output == 'db' and self.db_target:
+            for missing in missings:
+                try:
+                    q = {'target':self.db_target, 'value':missing}
+                    models.TargetKey.objects.create(**q)
+                except IntegrityError:
+                    pass
 
     def new(self, value):
         return
@@ -272,12 +296,14 @@ class StrChoiceFormater(Formater):
             return self.equiv_dict[value]
 
 class TypeFormater(StrChoiceFormater):
-    def __init__(self, model, cli=False, defaults={}, many_split=False):
+    def __init__(self, model, cli=False, defaults={}, many_split=False,
+                 db_target=None):
         self.create = True
         self.strict = False
         self.model = model
         self.defaults = defaults
         self.many_split = many_split
+        self.db_target = db_target
         self.missings = set()
         self.equiv_dict, self.choices = {}, []
         for item in model.objects.all():
@@ -304,8 +330,9 @@ class TypeFormater(StrChoiceFormater):
         return self.model.objects.create(**values)
 
 class DateFormater(Formater):
-    def __init__(self, date_format="%d/%m/%Y"):
+    def __init__(self, date_format="%d/%m/%Y", db_target=None):
         self.date_format = date_format
+        self.db_target = db_target
 
     def format(self, value):
         value = value.strip()
@@ -318,11 +345,26 @@ class DateFormater(Formater):
                                                            'value':value})
 
 class StrToBoolean(Formater):
-    def __init__(self, choices={}, cli=False, strict=False):
+    def __init__(self, choices={}, cli=False, strict=False, db_target=None):
         self.dct = copy.copy(choices)
         self.cli = cli
         self.strict= strict
+        self.db_target = db_target
         self.missings = set()
+        if self.db_target:
+            for target_key in self.db_target.keys.filter(is_set=True).all():
+                value = target_key.value
+                value = self.prepare(value)
+                if value in self.dct:
+                    continue
+                v = target_key.key
+                if v in ('False', '0'):
+                    v = False
+                elif v:
+                    v = True
+                else:
+                    v = None
+                self.dct[value] = v
 
     def prepare(self, value):
         value = unicode(value).strip()
@@ -359,6 +401,13 @@ class StrToBoolean(Formater):
                 self.dct[value] = False
             else:
                 self.dct[value] = None
+        if output == 'db' and self.db_target:
+            for missing in missings:
+                try:
+                    q = {'target':self.db_target, 'value':missing}
+                    models.TargetKey.objects.create(**q)
+                except IntegrityError:
+                    pass
 
     def format(self, value):
         value = self.prepare(value)
@@ -388,8 +437,9 @@ class Importer(object):
         }
 
     def __init__(self, skip_lines=0, reference_header=None,
-                 check_col_num=False, test=False, check_validity=True,
-                 history_modifier=None, output='silent'):
+             check_col_num=False, test=False, check_validity=True,
+             history_modifier=None, output='silent',
+             importer_instance=None):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
@@ -408,14 +458,18 @@ class Importer(object):
         self.check_col_num = check_col_num
         self.check_validity = check_validity
         self.line_format = copy.copy(self.LINE_FORMAT)
+        self.importer_instance = importer_instance
         self._initialized = False
         self._defaults = self.DEFAULTS.copy()
         self.history_modifier = history_modifier
         self.output = output
         if not self.history_modifier:
-            # get the first admin
-            self.history_modifier = User.objects.filter(is_superuser=True
-                                                ).order_by('pk')[0]
+            if self.importer_instance:
+                self.history_modifier = self.importer_instance.user
+            else:
+                # import made by the CLI: get the first admin
+                self.history_modifier = User.objects.filter(
+                                is_superuser=True).order_by('pk')[0]
 
     def initialize(self, table, output='silent'):
         """