Step by step import - WIP - refs #3975

1 files changed, 31 insertions, 14 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index ffe6c221d..420e38008 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -18,9 +18,7 @@
 # See the file COPYING for details.
 
 import copy
-import csv
 import datetime
-import io
 import os
 import logging
 import re
@@ -715,7 +713,6 @@ class Importer(object):
     DESC = ""
     LINE_FORMAT = []
     OBJECT_CLS = None
-    IMPORTED_LINE_FIELD = None
     UNICITY_KEYS = []
     # if set only models inside this list can be created
     MODEL_CREATION_LIMIT = []
@@ -873,6 +870,8 @@ class Importer(object):
         self.line_format = copy.copy(self.LINE_FORMAT)
         self.import_instance = import_instance
         self.archive = None
+        self.simulate = False
+        self.current_csv_line = None
         self.conservative_import = conservative_import
         # for a conservative_import UNICITY_KEYS should be defined
         assert not self.conservative_import or bool(self.UNICITY_KEYS)
@@ -953,12 +952,19 @@ class Importer(object):
                               import_instance=self.import_instance,
                               user=user)
 
+    def get_formaters(self):
+        return self.line_format
+
     def importation(self, table, initialize=True, choose_default=False,
-                    user=None):
+                    user=None, line_to_process=None, simulate=False):
         if initialize:
             self.initialize(table, self.output,
                             choose_default=choose_default, user=user)
-        self._importation(table)
+        self.simulate = simulate
+        return self._importation(table, line_to_process=line_to_process)
+
+    def get_current_values(self, obj):
+        return obj
 
     def _associate_db_target_to_formaters(self):
         if not self.import_instance:
@@ -1023,7 +1029,7 @@ class Importer(object):
                 current_data = current_data[key]
         return data
 
-    def _importation(self, table):
+    def _importation(self, table, line_to_process=None):
         self.match_table = {}
         table = list(table)
         if not table or not table[0]:
@@ -1034,6 +1040,7 @@ class Importer(object):
         self.errors = []
         self.validity = []
         self.number_imported = 0
+        idx_last_col = 0
         # index of the last required column
         for idx_last_col, formater in enumerate(reversed(self.line_format)):
             if formater and formater.required:
@@ -1053,7 +1060,13 @@ class Importer(object):
         total = len(table)
         if self.output == 'cli':
             sys.stdout.write("\n")
+        results = []
         for idx_line, line in enumerate(table):
+            if line_to_process is not None:
+                if line_to_process != idx_line:
+                    continue
+                if idx_line > line_to_process:
+                    return results
             if self.output == 'cli':
                 left = None
                 if idx_line > 10:
@@ -1067,9 +1080,10 @@ class Importer(object):
                 sys.stdout.write(txt.encode('utf-8'))
                 sys.stdout.flush()
             try:
-                self._line_processing(idx_line, line)
+                results.append(self._line_processing(idx_line, line))
             except ImporterError, msg:
                 self.errors.append((idx_line, None, msg))
+        return results
 
     def _line_processing(self, idx_line, line):
         self.idx_line = idx_line
@@ -1084,12 +1098,10 @@ class Importer(object):
         self._item_post_processing = []
         data = {}
 
-        # keep in database the raw line for testing purpose
-        if self.IMPORTED_LINE_FIELD:
-            output = io.StringIO()
-            writer = csv.writer(output)
-            writer.writerow(line)
-            data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+        self.current_csv_line = None
+        # raw line for simulation
+        if self.simulate:
+            self.current_csv_line = line
 
         n = datetime.datetime.now()
         logger.debug('%s - Processing line %d' % (unicode(n - self.now),
@@ -1117,6 +1129,8 @@ class Importer(object):
         n2 = n
         if self.test:
             return
+        if self.simulate:
+            return data
         # manage unicity of items (mainly for updates)
         if 'history_modifier' in get_all_field_names(self.OBJECT_CLS):
             data['history_modifier'] = self.history_modifier
@@ -1188,6 +1202,7 @@ class Importer(object):
             formater.post_process(obj, data, val, owner=self.history_modifier)
 
         self.post_processing(idx_line, obj)
+        return data
 
     def _row_processing(self, c_row, idx_col, idx_line, val, data):
         if idx_col >= len(self.line_format):
@@ -1557,7 +1572,9 @@ class Importer(object):
             self.errors.append((self.idx_line, None, msg))
             data[attribute] = None
 
-    def get_object(self, cls, data, path=[]):
+    def get_object(self, cls, data, path=None):
+        if not path:
+            path = []
         m2ms = []
         if type(data) != dict:
             # if data is not a dict we don't know what to do