From 15503c3362d47a5b3490dc23d14477812fc258b3 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 23 Mar 2018 20:19:04 +0100 Subject: Manage CSV edtion and line by line import (refs #3975) --- ishtar_common/static/media/styles.css | 4 + .../templates/ishtar/import_step_by_step.html | 215 +++++++++++++++++++-- ishtar_common/utils.py | 6 +- ishtar_common/views.py | 88 ++++++++- 4 files changed, 285 insertions(+), 28 deletions(-) diff --git a/ishtar_common/static/media/styles.css b/ishtar_common/static/media/styles.css index 9507674b7..6860ca93c 100644 --- a/ishtar_common/static/media/styles.css +++ b/ishtar_common/static/media/styles.css @@ -224,6 +224,10 @@ table.dataTable thead th, table.dataTable thead td { overflow-x: auto; } +.step-by-step-csv textarea { + height: 50px; +} + /* à adapter */ #message, #message div{ diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html index c827156c5..293ce3dce 100644 --- a/ishtar_common/templates/ishtar/import_step_by_step.html +++ b/ishtar_common/templates/ishtar/import_step_by_step.html @@ -2,28 +2,109 @@ {% load i18n inline_formset link_to_window %} {% block content %} + +{% comment %} + +{% endcomment %} + +

{% trans "Import step by step" %} – {{import.name}} – {% trans "line " %} {{line_number_displayed}}

{% if errors %} {% if values %} - - {% for header, value in values %} - - - - - {% endfor %} -
{{header}}{{value}}
+
+ {% csrf_token %} + + + + + + + + {% for idx, name, raw in values %} + + + + + {% endfor %} +
{% trans "Column number" %}{% trans "Name" %}{% trans "Raw value" %}{% trans "New value" %}
{{idx}}{{name|safe}}{{raw}}
+ + +
{% endif %} {% else %} +{% if not have_change %} + +{% else %} + +{% endif %} + {% if new_objects %}

{% trans "Objects to be created" %}

{% for path, cls, values in new_objects %} @@ -39,7 +120,7 @@ {% for k, value in values.items %} {{k|safe}} - {{value}} + {{value|safe}} {% endfor %} @@ -49,6 +130,7 @@ {% endif %} {% if updated_objects %} +

{% trans "Objects to be updated" %}

{% for path, obj, values, old_and_updated in updated_objects %} @@ -70,7 +152,6 @@ {% endfor %} -
{% trans "Updated values" %}
@@ -91,31 +172,129 @@ +
+{% endfor %} +{% endif %} + +{% if matched_objects %} +

{% trans "Objects matched with no changes" %}

+{% for path, obj, values in matched_objects %} + +
+
+
{{path}} – {{obj}} {{obj|link_to_window}} ({{obj.get_verbose_name}})
+
+
+ +
{% trans "Matching values" %}
+
+ + + + + {% for k, value in values.items %} + + + {% endfor %} +
{% trans "Key" %}{% trans "Value" %}
{{k|safe}}{{value}}
+ + +
{% endfor %} {% endif %} -

{% trans "CSV values" %}

-
+ +

{% trans "Source file" %}

+ +
+ {% csrf_token %} - + - {% comment %}{% endcomment %} + {% for idx, name, raw, interpreted in values %} + {% endfor %}
{% trans "Column number" %} {% trans "Name" %}{% trans "Raw CSV value" %}{% trans "Raw value" %} {% trans "Interpreted value" %}{% trans "Merged value" %}{% trans "New value" %}
{{idx}} {{name|safe}} {{raw}} {{interpreted|safe}}
-
+ + + + {% endif %} + {% endblock %} +{% block footer %} +{% endblock %} diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index 5a03f7f5f..c25cdf605 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -28,6 +28,7 @@ from django.conf import settings from django.contrib.gis.geos import GEOSGeometry from django.contrib.sessions.backends.db import SessionStore from django.core.cache import cache +from django.core.exceptions import FieldDoesNotExist from django.core.urlresolvers import reverse from django.utils.datastructures import MultiValueDict as BaseMultiValueDict from django.utils.safestring import mark_safe @@ -406,7 +407,10 @@ def get_field_labels_from_path(model, path): """ labels = [] for key in path: - field = model._meta.get_field(key) + try: + field = model._meta.get_field(key) + except: + return labels if hasattr(field, 'verbose_name'): labels.append(field.verbose_name) if field.one_to_many or field.one_to_one or field.many_to_many: diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 1fa968a95..d6d16568a 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -29,6 +29,7 @@ from tempfile import NamedTemporaryFile from tidylib import tidy_document as tidy import unicodedata from unidecode import unidecode +import unicodecsv from weasyprint import HTML, CSS from weasyprint.fonts import FontConfiguration @@ -1844,7 +1845,7 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): page_name = _(u"Import step by step") current_url = 'import_step_by_step' - def get(self, request, *args, **kwargs): + def get_import(self): try: self.imprt_obj = models.Import.objects.get( pk=int(self.kwargs['pk']) @@ -1857,18 +1858,66 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): if self.imprt_obj.user != user: raise Http404 self.current_line_number = int(self.kwargs['line_number']) - 1 - self.imprt = None + + def update_csv(self, request): + prefix = 'col-' + submited_line = [(int(k[len(prefix):]), request.POST[k]) + for k in request.POST if k.startswith(prefix)] + updated_line = [value for line, value in sorted(submited_line)] + filename = self.imprt_obj.imported_file.path + with open(filename, 'r') as f: + reader = unicodecsv.reader( + f, encoding=self.imprt_obj.encoding) + lines = [] + for idx, line in enumerate(reader): + if idx == self.current_line_number: + line = updated_line + line = [v.encode(self.imprt_obj.encoding, errors='replace') + for v in line] + lines.append(line) + with open(filename, 'w') as f: + writer = csv.writer(f, **CSV_OPTIONS) + writer.writerows(lines) + + def import_line(self, request, *args, **kwargs): try: self.imprt, data = self.imprt_obj.importation( - simulate=True, line_to_process=self.current_line_number, return_importer_and_data=True ) except IOError as e: self.errors = [e.message] - return + return super(ImportStepByStepView, self).get(request, *args, + **kwargs) + + def post(self, request, *args, **kwargs): + if not request.POST or request.POST.get('valid', None) not in ( + 'change-csv', 'import'): + return self.get(request, *args, **kwargs) + + self.get_import() + if request.POST.get('valid') == 'change-csv': + self.update_csv(request) + return self.get(request, *args, **kwargs) + self.import_line(request, *args, **kwargs) + # todo redirect to next + return self.get(request, *args, **kwargs) + + def get(self, request, *args, **kwargs): + self.get_import() + self.imprt = None self.errors, self.new_data = None, None - if not data: + try: + self.imprt, data = self.imprt_obj.importation( + simulate=True, + line_to_process=self.current_line_number, + return_importer_and_data=True + ) + except IOError as e: + self.errors = [None, None, e.message] + return super(ImportStepByStepView, self).get(request, *args, + **kwargs) + if not data or not data[0]: self.errors = self.imprt.errors else: self.new_data = data[:] @@ -1882,7 +1931,10 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): if self.errors: if self.imprt.current_csv_line: headers = [f.label for f in self.imprt.get_formaters()] - dct['values'] = zip(headers, self.imprt.current_csv_line) + dct['values'] = zip( + range(1, len(headers) + 1), headers, + self.imprt.current_csv_line + ) return dct headers, self.path_to_column, interpreted_values = [], {}, [] for idx, formater in enumerate(self.imprt.get_formaters()): @@ -1934,7 +1986,12 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): continue created_dict[k] = val # check if it is not previously created - key = (cls, tuple(sorted(created_dict.items()))) + dct_key = [] + for key, value in created_dict.items(): + if isinstance(value, list): + value = tuple(sorted(value)) + dct_key.append((key, value)) + key = (cls, tuple(sorted(dct_key))) if key in new_objects: # regroup it new_objects[key][0].append(label) @@ -1954,10 +2011,11 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): ] updated_objects = [] + main_changed = False for path, obj, values, updated_values in self.imprt.updated_objects: # transform path to explicit label - label = self.transform_path_to_label(obj.__class__, path) + label = self.transform_path_to_label(self.imprt.OBJECT_CLS, path) # transform key into explicit label values = self.transform_keys_to_label(path, obj.__class__, values) @@ -1979,6 +2037,7 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): changed = current_value != updated_value current_value = self.get_value(current_value) updated_value = self.get_value(updated_value) + main_changed |= changed old_and_updated[k] = [current_value, updated_value, changed] @@ -1987,7 +2046,18 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView): old_and_updated) updated_objects.append((label, obj, values, old_and_updated)) - dct['updated_objects'] = updated_objects + dct['have_change'] = main_changed + dct['updated_objects'] = [] + dct['matched_objects'] = [] + for path, obj, values, old_and_updated in updated_objects: + if old_and_updated: + dct['updated_objects'].append( + (path, obj, values, old_and_updated) + ) + else: + dct['matched_objects'].append( + (path, obj, values) + ) dct['ambiguous_objects'] = self.imprt.ambiguous_objects dct['not_find_objects'] = self.imprt.not_find_objects return dct -- cgit v1.2.3