Imports: manage soft import (update) with unicity keys - script to initialize SRA file import

1 files changed, 87 insertions, 41 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 6239bf4c0..3d623e7a6 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -177,7 +177,8 @@ class UnicodeFormater(Formater):
                                      'length': self.max_length})
         if self.notnull and not value:
             return
-        value = self.prefix + value
+        if value:
+            value = self.prefix + value
         return value
 
 
@@ -603,7 +604,6 @@ class Importer(object):
     UNICITY_KEYS = []
     EXTRA_DEFAULTS = {}
     DEFAULTS = {}
-    STR_CUT = {}
     ERRORS = {
         'header_check': _(
             u"The given file is not correct. Check the file "
@@ -630,9 +630,13 @@ class Importer(object):
         model_name = self.OBJECT_CLS.__module__ + '.' + \
             self.OBJECT_CLS.__name__
 
+        unicity_keys = ''
+        if self.UNICITY_KEYS:
+            unicity_keys = ";".join(self.UNICITY_KEYS)
+
         importer = models.ImporterType.objects.create(
             slug=self.SLUG, name=name, description=self.DESC,
-            associated_models=model_name)
+            associated_models=model_name, unicity_keys=unicity_keys)
 
         for default in self.DEFAULTS:
             values = self.DEFAULTS[default]
@@ -691,13 +695,15 @@ class Importer(object):
                 models.ImportTarget.objects.get_or_create(
                     column=column, target=target, formater_type=formater_model,
                     force_new=getattr(formater, 'force_new', False),
+                    concat=getattr(formater, 'concat', False),
                     regexp_filter=regexp_filter,
                     comment=line.comment)
         return True
 
     def __init__(self, skip_lines=0, reference_header=None,
                  check_col_num=False, test=False, history_modifier=None,
-                 output='silent', import_instance=None):
+                 output='silent', import_instance=None,
+                 conservative_import=False):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
@@ -715,8 +721,12 @@ class Importer(object):
         self.line_format = copy.copy(self.LINE_FORMAT)
         self.import_instance = import_instance
         self.archive = None
+        self.conservative_import = conservative_import
+        # for a conservative_import UNICITY_KEYS should be defined
+        assert not self.conservative_import or bool(self.UNICITY_KEYS)
         self.DB_TARGETS = {}
         self.match_table = {}
+        self.concats = set()
         if import_instance and import_instance.imported_images:
             self.archive = import_instance.imported_images
         self._defaults = self.DEFAULTS.copy()
@@ -883,6 +893,7 @@ class Importer(object):
                 self.errors.append((idx_line, None, msg))
 
     def _line_processing(self, idx_line, line):
+        self.idx_line = idx_line
         if self.skip_lines > idx_line:
             self.validity.append(line)
             return
@@ -927,17 +938,11 @@ class Importer(object):
         if self.test:
             return
         # manage unicity of items (mainly for updates)
-        if self.UNICITY_KEYS:
-            data['defaults'] = {}
-            for k in data.keys():
-                if k not in self.UNICITY_KEYS \
-                   and k != 'defaults':
-                    data['defaults'][k] = data.pop(k)
-
         if 'history_modifier' in \
                 self.OBJECT_CLS._meta.get_all_field_names():
             data['history_modifier'] = self.history_modifier
 
+        self.plouf = 0
         obj, created = self.get_object(self.OBJECT_CLS, data)
         if self.import_instance and hasattr(obj, 'imports') \
            and created:
@@ -1014,10 +1019,10 @@ class Importer(object):
                     self.errors.append(
                         (idx_line + 1, idx_col + 1,
                          self.ERRORS['value_required']))
+                    self.c_errors = True
                 elif not val.strip():
                     c_row.append("")
                     return
-                self.c_errors = True
                 val = val.replace(NEW_LINE_BREAK, '\n')
                 self.errors.append(
                     (idx_line + 1, idx_col + 1,
@@ -1049,6 +1054,8 @@ class Importer(object):
                 field_name = field_name[idx_v]
             if type(force_new) in (list, tuple):
                 force_new = force_new[idx_v]
+            if formater.concat:
+                self.concats.add(field_name)
 
             if self.DB_TARGETS:
                 formater.reinit_db_target(
@@ -1192,14 +1199,18 @@ class Importer(object):
                         m2ms.append((attribute, v))
         elif hasattr(field_object, 'rel') and field_object.rel:
             if type(data[attribute]) == dict:
-                c_path.append(attribute)
                 # put history_modifier for every created item
                 if 'history_modifier' in \
                    field_object.rel.to._meta.get_all_field_names():
                     data[attribute]['history_modifier'] = \
                         self.history_modifier
-                data[attribute], created = self.get_object(
-                    field_object.rel.to, data[attribute], c_path)
+                try:
+                    c_path.append(attribute)
+                    data[attribute], created = self.get_object(
+                        field_object.rel.to, data[attribute].copy(), c_path)
+                except ImporterError, msg:
+                    self.errors.append((self.idx_line, None, msg))
+                    data[attribute] = None
             elif type(data[attribute]) == list:
                 data[attribute] = data[attribute][0]
 
@@ -1207,37 +1218,37 @@ class Importer(object):
         m2ms = []
         if data and type(data) == dict:
             c_path = path[:]
-            for attribute in data.keys():
+
+            # get all related fields
+            for attribute in list(data.keys()):
+                c_c_path = c_path[:]
                 if not attribute:
                     data.pop(attribute)
                     continue
                 if not data[attribute]:
                     continue
                 if attribute != '__force_new':
-                    self.get_field(cls, attribute, data, m2ms, c_path)
-            # default values
-            path = tuple(path)
-            if path in self._defaults:
-                for k in self._defaults[path]:
-                    if k not in data or not data[k]:
-                        data[k] = self._defaults[path][k]
-
-            # pre treatment
-            if path in self.STR_CUT:
-                for k in self.STR_CUT[path]:
-                    if k in data and data[k]:
-                        data[k] = unicode(data[k])[:self.STR_CUT[path][k]]
+                    self.get_field(cls, attribute, data, m2ms, c_c_path)
 
-            # filter default values
+            # filter uncessary default values
             create_dict = copy.deepcopy(data)
             for k in create_dict.keys():
                 if type(create_dict[k]) == dict:
                     create_dict.pop(k)
+
+            # default values
+            path = tuple(path)
             defaults = {}
+            if path in self._defaults:
+                for k in self._defaults[path]:
+                    if (k not in data or not data[k]):
+                        defaults[k] = self._defaults[path][k]
+
             if 'history_modifier' in create_dict:
-                defaults = {
+                defaults.update({
                     'history_modifier': create_dict.pop('history_modifier')
-                }
+                })
+
             try:
                 try:
                     dct = create_dict.copy()
@@ -1248,10 +1259,44 @@ class Importer(object):
                         created = dct.pop('__force_new')
                         if not [k for k in dct if dct[k] is not None]:
                             return None, created
-                        obj = cls.objects.create(**dct)
+                        new_dct = defaults.copy()
+                        new_dct.update(dct)
+                        obj = cls.objects.create(**new_dct)
                     else:
-                        dct['defaults'] = defaults
+                        # manage UNICITY_KEYS - only level 1
+                        if not path and self.UNICITY_KEYS:
+                            for k in dct.keys():
+                                if k not in self.UNICITY_KEYS \
+                                   and k != 'defaults':
+                                    defaults[k] = dct.pop(k)
+
+                        dct['defaults'] = defaults.copy()
                         obj, created = cls.objects.get_or_create(**dct)
+
+                        if not created and not path and self.UNICITY_KEYS:
+                            changed = False
+                            if self.conservative_import:
+                                for k in dct['defaults']:
+                                    new_val = dct['defaults'][k]
+                                    if new_val is None or new_val == '':
+                                        continue
+                                    val = getattr(obj, k)
+                                    if val is None or val == '':
+                                        changed = True
+                                        setattr(obj, k, new_val)
+                                    elif k in self.concats \
+                                            and type(val) == unicode \
+                                            and type(new_val) == unicode:
+                                        setattr(obj, k, val + u" - " + new_val)
+                            else:
+                                for k in dct['defaults']:
+                                    new_val = dct['defaults'][k]
+                                    if new_val is None or new_val == '':
+                                        continue
+                                    changed = True
+                                    setattr(obj, k, new_val)
+                            if changed:
+                                obj.save()
                     if self.import_instance and hasattr(obj, 'imports') \
                        and created:
                         obj.imports.add(self.import_instance)
@@ -1286,24 +1331,25 @@ class Importer(object):
                 except UnicodeDecodeError:
                     data = ''
                 raise ImporterError(
-                    "Erreur d'import %s, contexte : %s, erreur : %s"
-                    % (unicode(cls), unicode(data), message))
+                    "Erreur d'import %s %s, contexte : %s, erreur : %s"
+                    % (unicode(cls), unicode("__".join(path)),
+                       unicode(data), message))
             return obj, created
         return data
 
-    def _format_csv_line(self, values):
+    def _format_csv_line(self, values, empty=u"-"):
         return u'"' + u'","'.join(
-            [(v and unicode(v).replace('"', '""')) or u'-'
+            [(v and unicode(v).replace('"', '""')) or empty
              for v in values]) + u'"'
 
-    def _get_csv(self, rows, header=[]):
+    def _get_csv(self, rows, header=[], empty=u"-"):
         if not rows:
             return ""
         csv_v = []
         if header:
-            csv_v.append(self._format_csv_line(header))
+            csv_v.append(self._format_csv_line(header, empty=empty))
         for values in rows:
-            csv_v.append(self._format_csv_line(values))
+            csv_v.append(self._format_csv_line(values, empty=empty))
         return u"\n".join(csv_v)
 
     def get_csv_errors(self):