diff options
| -rw-r--r-- | ishtar_common/data_importer.py | 87 | ||||
| -rw-r--r-- | ishtar_common/models.py | 17 | ||||
| -rw-r--r-- | ishtar_common/templates/ishtar/import_list.html | 3 |
3 files changed, 52 insertions, 55 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 24c8e166d..8d53d78f1 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -35,8 +35,7 @@ class ImportFormater(object): def __init__(self, field_name, formater=None, required=True, through=None, through_key=None, through_dict=None, through_unicity_keys=None, duplicate_fields=[], regexp=None, regexp_formater_args=[], - reverse_for_test=None, force_value=None, post_processing=False, - concat=False, comment=""): + force_value=None, post_processing=False, concat=False, comment=""): self.field_name = field_name self.formater = formater self.required = required @@ -47,7 +46,6 @@ class ImportFormater(object): self.duplicate_fields = duplicate_fields self.regexp = regexp self.regexp_formater_args = regexp_formater_args - self.reverse_for_test = reverse_for_test # write this value even if a value exists self.force_value = force_value # post process after import @@ -442,26 +440,22 @@ class Importer(object): } def __init__(self, skip_lines=0, reference_header=None, - check_col_num=False, test=False, check_validity=True, - history_modifier=None, output='silent', - import_instance=None): + check_col_num=False, test=False, history_modifier=None, + output='silent', import_instance=None): """ * skip_line must be set if the data provided has got headers lines. * a reference_header can be provided to perform a data compliance check. It can be useful to warn about bad parsing. * test doesn't write in the database - * check_validity rewrite a CSV file to be compared """ - self.message = '' self.skip_lines = skip_lines self.reference_header = reference_header self.test = test self.errors = [] # list of (line, col, message) - self.messages = [] # list of (line, col, message) + self.validity = [] # list of (line, col, message) self.number_updated = 0 self.number_created = 0 self.check_col_num = check_col_num - self.check_validity = check_validity self.line_format = copy.copy(self.LINE_FORMAT) self.import_instance = import_instance self._defaults = self.DEFAULTS.copy() @@ -500,17 +494,9 @@ class Importer(object): formater.init(vals[idx], output) def importation(self, table, initialize=True): - self.validity_file = None if initialize: self.initialize(table, self.output) - if self.check_validity: - with NamedTemporaryFile(delete=False) as validity_file: - self.validity_file = UnicodeWriter(validity_file, - delimiter=',', quotechar='"', - quoting=csv.QUOTE_MINIMAL) - self._importation(table) - else: - self._importation(table) + self._importation(table) @classmethod def _field_name_to_data_dict(cls, field_name, value, data, @@ -545,7 +531,7 @@ class Importer(object): raise ImporterError(self.ERRORS['too_many_cols'] % { 'user_col':len(table[0]), 'ref_col':len(self.line_format)}) self.errors = [] - self.messages = [] + self.validity = [] self.number_imported = 0 # index of the last required column for idx_last_col, formater in enumerate(reversed(self.line_format)): @@ -586,12 +572,10 @@ class Importer(object): def _line_processing(self, idx_line, line): if self.skip_lines > idx_line: - if self.validity_file: - self.validity_file.writerow(line) + self.validity.append(line) return if not line: - if self.validity_file: - self.validity_file.writerow([]) + self.validity.append([]) return self._throughs = [] # list of (formater, value) self._post_processing = [] # list of (formater, value) @@ -616,8 +600,7 @@ class Importer(object): except: pass - if self.validity_file: - self.validity_file.writerow(c_row) + self.validity.append(c_row) if not self.c_errors and (idx_col + 1) < self.min_col_number: self.c_errors = True self.errors.append((idx_line+1, idx_col+1, @@ -691,8 +674,7 @@ class Importer(object): self._post_processing.append((formater, val)) if not formater or not formater.field_name: - if self.validity_file: - c_row.append(val) + c_row.append('-') return # regex management @@ -720,7 +702,6 @@ class Importer(object): c_values = [] for idx_v, v in enumerate(val_group): - self.message = '' func = formater.formater if type(func) in (list, tuple): func = func[idx_v] @@ -746,19 +727,22 @@ class Importer(object): if formater.required: self.c_errors = True self.errors.append((idx_line+1, idx_col+1, e.message)) - c_values.append(None) + c_values.append('') return formated_values.append(value) - if self.message: - self.messages.append(self.message) - value = formated_values if not many_values: value = formated_values[0] - c_values.append(value) - + printed_values = value + if type(value) not in (list, tuple): + printed_values = [value] + try: + # don't reunicode - unicoded values + c_values.append(u" ; ".join([v for v in printed_values])) + except TypeError: + c_values.append(u" ; ".join([unicode(v) for v in printed_values])) if value == None and formater.required: self.c_errors = True self.errors.append((idx_line+1, idx_col+1, @@ -780,10 +764,7 @@ class Importer(object): for field_name in field_names: self._field_name_to_data_dict(field_name, value, data, formater.force_value) - if formater.reverse_for_test: - c_row.append(formater.reverse_for_test(**c_values)) - else: - c_row.append(unicode(c_values)) + c_row.append(u" ; ".join([v for v in c_values])) def get_object(self, cls, data, path=[]): m2ms = [] @@ -850,16 +831,26 @@ class Importer(object): return obj, created return data - def get_csv_errors(self): - if not self.errors: + def _format_csv_line(self, values): + return u",".join([v and unicode(v).replace('"', '""') or u'-' + for v in values]) + + def _get_csv(self, rows, header=[]): + if not rows: return "" - csv_errors = ['"%s","%s","%s"' % (unicode(_("line")), - unicode(_("col")), unicode(_("error")))] - for line, col, error in self.errors: - csv_errors.append(u'"%s","%s","%s"' % (line and unicode(line) or '-', - col and unicode(col) or '-', - unicode(error))) - return u"\n".join(csv_errors) + csv_v = [] + if header: + csv_v.append(self._format_csv_line(header)) + for values in rows: + csv_v.append(self._format_csv_line(values)) + return u"\n".join(csv_v) + + def get_csv_errors(self): + return self._get_csv(self.errors, + header=[_("line"), _("col"), _("error")]) + + def get_csv_result(self): + return self._get_csv(self.validity) @classmethod def choices_check(cls, choices): diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 20b85e614..c7f89299e 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1400,15 +1400,18 @@ class Import(models.Model): self.save() importer = self.get_importer_instance() importer.importation(self.data_table) + # result file + filename = slugify(self.importer_type.name) + now = datetime.datetime.now().isoformat('-').replace(':','') + result_file = filename + "_result_%s.csv" % now + result_file = os.sep.join([self.result_file.storage.location, + result_file]) + with open(result_file, 'w') as fle: + fle.write(importer.get_csv_result().encode('utf-8')) + self.result_file = File(open(fle.name)) if importer.errors: self.state = 'FE' - error_file = None - - filename = slugify(self.importer_type.name) + ".csv" - now = datetime.datetime.now().isoformat('-' - ).replace(':','') - error_file = '.'.join(filename.split('.')[:-1]) \ - + "_errors_%s.csv" % now + error_file = filename + "_errors_%s.csv" % now error_file = os.sep.join([self.error_file.storage.location, error_file]) with open(error_file, 'w') as fle: diff --git a/ishtar_common/templates/ishtar/import_list.html b/ishtar_common/templates/ishtar/import_list.html index 40bc019ad..9e9b4cb61 100644 --- a/ishtar_common/templates/ishtar/import_list.html +++ b/ishtar_common/templates/ishtar/import_list.html @@ -46,6 +46,9 @@ <td>{% if import.error_file %} <a href='{{MEDIA_URL}}{{import.error_file}}'>{% trans "Error file" %}</a> {% endif %}</td> + <td>{% if import.result_file %} + <a href='{{MEDIA_URL}}{{import.result_file}}'>{% trans "Control file" %}</a> + {% endif %}</td> </tr> {% endfor %} </table> |