diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 45 |
1 files changed, 31 insertions, 14 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index ffe6c221d..420e38008 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -18,9 +18,7 @@ # See the file COPYING for details. import copy -import csv import datetime -import io import os import logging import re @@ -715,7 +713,6 @@ class Importer(object): DESC = "" LINE_FORMAT = [] OBJECT_CLS = None - IMPORTED_LINE_FIELD = None UNICITY_KEYS = [] # if set only models inside this list can be created MODEL_CREATION_LIMIT = [] @@ -873,6 +870,8 @@ class Importer(object): self.line_format = copy.copy(self.LINE_FORMAT) self.import_instance = import_instance self.archive = None + self.simulate = False + self.current_csv_line = None self.conservative_import = conservative_import # for a conservative_import UNICITY_KEYS should be defined assert not self.conservative_import or bool(self.UNICITY_KEYS) @@ -953,12 +952,19 @@ class Importer(object): import_instance=self.import_instance, user=user) + def get_formaters(self): + return self.line_format + def importation(self, table, initialize=True, choose_default=False, - user=None): + user=None, line_to_process=None, simulate=False): if initialize: self.initialize(table, self.output, choose_default=choose_default, user=user) - self._importation(table) + self.simulate = simulate + return self._importation(table, line_to_process=line_to_process) + + def get_current_values(self, obj): + return obj def _associate_db_target_to_formaters(self): if not self.import_instance: @@ -1023,7 +1029,7 @@ class Importer(object): current_data = current_data[key] return data - def _importation(self, table): + def _importation(self, table, line_to_process=None): self.match_table = {} table = list(table) if not table or not table[0]: @@ -1034,6 +1040,7 @@ class Importer(object): self.errors = [] self.validity = [] self.number_imported = 0 + idx_last_col = 0 # index of the last required column for idx_last_col, formater in enumerate(reversed(self.line_format)): if formater and formater.required: @@ -1053,7 +1060,13 @@ class Importer(object): total = len(table) if self.output == 'cli': sys.stdout.write("\n") + results = [] for idx_line, line in enumerate(table): + if line_to_process is not None: + if line_to_process != idx_line: + continue + if idx_line > line_to_process: + return results if self.output == 'cli': left = None if idx_line > 10: @@ -1067,9 +1080,10 @@ class Importer(object): sys.stdout.write(txt.encode('utf-8')) sys.stdout.flush() try: - self._line_processing(idx_line, line) + results.append(self._line_processing(idx_line, line)) except ImporterError, msg: self.errors.append((idx_line, None, msg)) + return results def _line_processing(self, idx_line, line): self.idx_line = idx_line @@ -1084,12 +1098,10 @@ class Importer(object): self._item_post_processing = [] data = {} - # keep in database the raw line for testing purpose - if self.IMPORTED_LINE_FIELD: - output = io.StringIO() - writer = csv.writer(output) - writer.writerow(line) - data[self.IMPORTED_LINE_FIELD] = output.getvalue() + self.current_csv_line = None + # raw line for simulation + if self.simulate: + self.current_csv_line = line n = datetime.datetime.now() logger.debug('%s - Processing line %d' % (unicode(n - self.now), @@ -1117,6 +1129,8 @@ class Importer(object): n2 = n if self.test: return + if self.simulate: + return data # manage unicity of items (mainly for updates) if 'history_modifier' in get_all_field_names(self.OBJECT_CLS): data['history_modifier'] = self.history_modifier @@ -1188,6 +1202,7 @@ class Importer(object): formater.post_process(obj, data, val, owner=self.history_modifier) self.post_processing(idx_line, obj) + return data def _row_processing(self, c_row, idx_col, idx_line, val, data): if idx_col >= len(self.line_format): @@ -1557,7 +1572,9 @@ class Importer(object): self.errors.append((self.idx_line, None, msg)) data[attribute] = None - def get_object(self, cls, data, path=[]): + def get_object(self, cls, data, path=None): + if not path: + path = [] m2ms = [] if type(data) != dict: # if data is not a dict we don't know what to do |