diff options
| -rw-r--r-- | ishtar_common/data_importer.py | 45 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 28 | ||||
| -rw-r--r-- | ishtar_common/templates/ishtar/import_step_by_step.html | 50 | ||||
| -rw-r--r-- | ishtar_common/urls.py | 2 | ||||
| -rw-r--r-- | ishtar_common/views.py | 84 |
5 files changed, 185 insertions, 24 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index ffe6c221d..420e38008 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -18,9 +18,7 @@ # See the file COPYING for details. import copy -import csv import datetime -import io import os import logging import re @@ -715,7 +713,6 @@ class Importer(object): DESC = "" LINE_FORMAT = [] OBJECT_CLS = None - IMPORTED_LINE_FIELD = None UNICITY_KEYS = [] # if set only models inside this list can be created MODEL_CREATION_LIMIT = [] @@ -873,6 +870,8 @@ class Importer(object): self.line_format = copy.copy(self.LINE_FORMAT) self.import_instance = import_instance self.archive = None + self.simulate = False + self.current_csv_line = None self.conservative_import = conservative_import # for a conservative_import UNICITY_KEYS should be defined assert not self.conservative_import or bool(self.UNICITY_KEYS) @@ -953,12 +952,19 @@ class Importer(object): import_instance=self.import_instance, user=user) + def get_formaters(self): + return self.line_format + def importation(self, table, initialize=True, choose_default=False, - user=None): + user=None, line_to_process=None, simulate=False): if initialize: self.initialize(table, self.output, choose_default=choose_default, user=user) - self._importation(table) + self.simulate = simulate + return self._importation(table, line_to_process=line_to_process) + + def get_current_values(self, obj): + return obj def _associate_db_target_to_formaters(self): if not self.import_instance: @@ -1023,7 +1029,7 @@ class Importer(object): current_data = current_data[key] return data - def _importation(self, table): + def _importation(self, table, line_to_process=None): self.match_table = {} table = list(table) if not table or not table[0]: @@ -1034,6 +1040,7 @@ class Importer(object): self.errors = [] self.validity = [] self.number_imported = 0 + idx_last_col = 0 # index of the last required column for idx_last_col, formater in enumerate(reversed(self.line_format)): if formater and formater.required: @@ -1053,7 +1060,13 @@ class Importer(object): total = len(table) if self.output == 'cli': sys.stdout.write("\n") + results = [] for idx_line, line in enumerate(table): + if line_to_process is not None: + if line_to_process != idx_line: + continue + if idx_line > line_to_process: + return results if self.output == 'cli': left = None if idx_line > 10: @@ -1067,9 +1080,10 @@ class Importer(object): sys.stdout.write(txt.encode('utf-8')) sys.stdout.flush() try: - self._line_processing(idx_line, line) + results.append(self._line_processing(idx_line, line)) except ImporterError, msg: self.errors.append((idx_line, None, msg)) + return results def _line_processing(self, idx_line, line): self.idx_line = idx_line @@ -1084,12 +1098,10 @@ class Importer(object): self._item_post_processing = [] data = {} - # keep in database the raw line for testing purpose - if self.IMPORTED_LINE_FIELD: - output = io.StringIO() - writer = csv.writer(output) - writer.writerow(line) - data[self.IMPORTED_LINE_FIELD] = output.getvalue() + self.current_csv_line = None + # raw line for simulation + if self.simulate: + self.current_csv_line = line n = datetime.datetime.now() logger.debug('%s - Processing line %d' % (unicode(n - self.now), @@ -1117,6 +1129,8 @@ class Importer(object): n2 = n if self.test: return + if self.simulate: + return data # manage unicity of items (mainly for updates) if 'history_modifier' in get_all_field_names(self.OBJECT_CLS): data['history_modifier'] = self.history_modifier @@ -1188,6 +1202,7 @@ class Importer(object): formater.post_process(obj, data, val, owner=self.history_modifier) self.post_processing(idx_line, obj) + return data def _row_processing(self, c_row, idx_col, idx_line, val, data): if idx_col >= len(self.line_format): @@ -1557,7 +1572,9 @@ class Importer(object): self.errors.append((self.idx_line, None, msg)) data[attribute] = None - def get_object(self, cls, data, path=[]): + def get_object(self, cls, data, path=None): + if not path: + path = [] m2ms = [] if type(data) != dict: # if data is not a dict we don't know what to do diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index c3fbaa92d..10124b024 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -170,7 +170,7 @@ class ImporterType(models.Model): if column.export_field_name: LINE_EXPORT_FORMAT.append( ImportFormater(column.export_field_name, - label=column.label) + label=column.label) ) continue force_news = [] @@ -196,6 +196,7 @@ class ImporterType(models.Model): formater_kwargs['label'] = column.label formater_kwargs['required'] = column.required formater_kwargs['force_new'] = force_news + formater_kwargs['comment'] = column.description if column.export_field_name: formater_kwargs['export_field_name'] = [ column.export_field_name] @@ -865,9 +866,11 @@ class Import(models.Model): if self.state == 'A': actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Launch import"))) + actions.append(('IS', _(u"Step by step import"))) if self.state in ('F', 'FE'): actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Re-import"))) + actions.append(('IS', _(u"Step by step import"))) actions.append(('AC', _(u"Archive"))) if self.state == 'AC': actions.append(('A', _(u"Unarchive"))) @@ -946,21 +949,22 @@ class Import(models.Model): self.save() return delayed_import(self.pk, session_key) - def importation(self, session_key=None): + def importation(self, session_key=None, line_to_process=None, + simulate=False, return_importer_and_data=False): self.state = 'IP' self.end_date = datetime.datetime.now() self.save() importer = self.get_importer_instance() try: - importer.importation(self.data_table, user=self.user) + data = importer.importation( + self.data_table, user=self.user, + line_to_process=line_to_process, simulate=simulate) except IOError: + error_message = unicode(_(u"Error on imported file: {}")).format( + self.imported_file) + importer.errors = [error_message] if session_key: - put_session_message( - session_key, - unicode(_(u"Error on imported file: {}")).format( - self.imported_file), - "warning" - ) + put_session_message(session_key, error_message, "warning") ids = get_session_var(session_key, 'current_import_id') if not ids: ids = [] @@ -968,7 +972,9 @@ class Import(models.Model): put_session_var(session_key, 'current_import_id', ids) self.state = 'FE' self.save() - return + if not return_importer_and_data: + return + return importer, None # result file filename = slugify(self.importer_type.name) now = datetime.datetime.now().isoformat('-').replace(':', '') @@ -1006,6 +1012,8 @@ class Import(models.Model): ) self.end_date = datetime.datetime.now() self.save() + if return_importer_and_data: + return importer, data def archive(self): self.state = 'AC' diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html new file mode 100644 index 000000000..4d7512de1 --- /dev/null +++ b/ishtar_common/templates/ishtar/import_step_by_step.html @@ -0,0 +1,50 @@ +{% extends "base.html" %} +{% load i18n inline_formset %} + +{% block content %} +<h2>{% trans "Import step by step" %}</h2> +<h3>{{import.name}} – {% trans "line " %} {{line_number_displayed}}</h3> +{% if errors %} +<div class="alert alert-danger" role="alert"> + <p>{% trans "The following error(s) has been encountered while parsing the source file:" %}</p> + <ul>{% for error in errors %} + <li>{{error}}</li> + {% endfor %}</ul> +</div> + +{% if values %} +<table class="table table-striped"> + {% for header, value in values %} + <tr> + <th>{{header}}</th> + <td>{{value}}</td> + </tr> + {% endfor %} +</table> +{% endif %} + +{% else %} + +<div> + <table class="table table-striped"> + <tr> + <th>{% trans "Column number" %}</th> + <th>{% trans "Name" %}</th> + <th>{% trans "Raw CSV value" %}</th> + <th>{% trans "Current value" %}</th> + <th>{% trans "New value" %}</th> + {% comment %}<th>{% trans "Merged value" %}</th>{% endcomment %} + </tr> + {% for idx, name, raw, interpreted in values %}<tr> + <td>{{idx}}</td> + <td>{{name|safe}}</td> + <td>{{raw}}</td> + <td></td> + <td>{{interpreted}}</td> + </tr>{% endfor %} + </table> + +</div> +{% endif %} +{% endblock %} + diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py index 5aa3bb409..1a7051b17 100644 --- a/ishtar_common/urls.py +++ b/ishtar_common/urls.py @@ -117,6 +117,8 @@ urlpatterns = [ views.ImportDeleteView.as_view(), name='import_delete'), url(r'^import-link-unmatched/(?P<pk>[0-9]+)/$', views.ImportLinkView.as_view(), name='import_link_unmatched'), + url(r'^import-step-by-step/(?P<pk>[0-9]+)/(?P<line_number>[0-9]+)/$', + views.ImportStepByStepView.as_view(), name='import_step_by_step'), ] actions = [] diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 8b0d51601..1af2f58ae 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -1819,6 +1819,14 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView): imprt.delayed_importation(request.session.session_key) else: imprt.importation() + elif action == 'IS': + if imprt.current_line is None: + imprt.current_line = imprt.skip_lines + imprt.save() + return HttpResponseRedirect( + reverse('import_step_by_step', + args=[imprt.pk, imprt.current_line + 1]) + ) elif action == 'AC': imprt.archive() return HttpResponseRedirect(reverse(self.current_url)) @@ -1829,6 +1837,82 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView): return dct +class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): + template_name = 'ishtar/import_step_by_step.html' + page_name = _(u"Import step by step") + current_url = 'import_step_by_step' + + def get(self, request, *args, **kwargs): + try: + self.imprt_obj = models.Import.objects.get( + pk=int(self.kwargs['pk']) + ) + except (models.Import.DoesNotExist, ValueError): + raise Http404 + if not self.request.user.is_superuser: + # user can only edit his own imports + user = models.IshtarUser.objects.get(pk=self.request.user.pk) + if self.imprt_obj.user != user: + raise Http404 + self.current_line_number = int(self.kwargs['line_number']) - 1 + self.imprt = None + try: + self.imprt, data = self.imprt_obj.importation( + simulate=True, + line_to_process=self.current_line_number, + return_importer_and_data=True + ) + except IOError as e: + self.errors = [e.message] + return + self.errors, self.new_data = None, None + if not data: + self.errors = self.imprt.errors + else: + self.new_data = data[:] + return super(ImportStepByStepView, self).get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + dct = super(ImportStepByStepView, self).get_context_data(**kwargs) + dct['import'] = self.imprt_obj + dct['line_number_displayed'] = self.kwargs['line_number'] + dct['errors'] = self.errors + if self.errors: + if self.imprt.current_csv_line: + headers = [f.label for f in self.imprt.get_formaters()] + dct['values'] = zip(headers, self.imprt.current_csv_line) + return dct + headers, interpreted_values = [], [] + for formater in self.imprt.get_formaters(): + lbl = formater.label + if formater.comment: + lbl += u' <i data-toggle="tooltip" class="fa ' + lbl += u'fa-question-circle"' + lbl += u' aria-hidden="true" title="{}">'.format( + formater.comment.replace(u'"', u'"')) + lbl += u'</i>' + headers.append(lbl) + field_name = formater.field_name[0] + if formater.export_field_name: + field_name = formater.export_field_name[0] + value = self.new_data[0].copy() + for key in field_name.split('__'): + if key in value: + value = value[key] + else: + value = u"-" + if value in (None, [], [None]): + value = _(u"* empty *") + if isinstance(value, list): + value = u" ; ".join(value) + interpreted_values.append(value) + dct['values'] = zip( + range(1, len(headers) + 1), headers, self.imprt.current_csv_line, + interpreted_values + ) + return dct + + class ImportListTableView(ImportListView): template_name = 'ishtar/import_table.html' current_url = 'current_imports_table' |