Step by step import - WIP - refs #3975

5 files changed, 185 insertions, 24 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index ffe6c221d..420e38008 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -18,9 +18,7 @@
 # See the file COPYING for details.
 
 import copy
-import csv
 import datetime
-import io
 import os
 import logging
 import re
@@ -715,7 +713,6 @@ class Importer(object):
     DESC = ""
     LINE_FORMAT = []
     OBJECT_CLS = None
-    IMPORTED_LINE_FIELD = None
     UNICITY_KEYS = []
     # if set only models inside this list can be created
     MODEL_CREATION_LIMIT = []
@@ -873,6 +870,8 @@ class Importer(object):
         self.line_format = copy.copy(self.LINE_FORMAT)
         self.import_instance = import_instance
         self.archive = None
+        self.simulate = False
+        self.current_csv_line = None
         self.conservative_import = conservative_import
         # for a conservative_import UNICITY_KEYS should be defined
         assert not self.conservative_import or bool(self.UNICITY_KEYS)
@@ -953,12 +952,19 @@ class Importer(object):
                               import_instance=self.import_instance,
                               user=user)
 
+    def get_formaters(self):
+        return self.line_format
+
     def importation(self, table, initialize=True, choose_default=False,
-                    user=None):
+                    user=None, line_to_process=None, simulate=False):
         if initialize:
             self.initialize(table, self.output,
                             choose_default=choose_default, user=user)
-        self._importation(table)
+        self.simulate = simulate
+        return self._importation(table, line_to_process=line_to_process)
+
+    def get_current_values(self, obj):
+        return obj
 
     def _associate_db_target_to_formaters(self):
         if not self.import_instance:
@@ -1023,7 +1029,7 @@ class Importer(object):
                 current_data = current_data[key]
         return data
 
-    def _importation(self, table):
+    def _importation(self, table, line_to_process=None):
         self.match_table = {}
         table = list(table)
         if not table or not table[0]:
@@ -1034,6 +1040,7 @@ class Importer(object):
         self.errors = []
         self.validity = []
         self.number_imported = 0
+        idx_last_col = 0
         # index of the last required column
         for idx_last_col, formater in enumerate(reversed(self.line_format)):
             if formater and formater.required:
@@ -1053,7 +1060,13 @@ class Importer(object):
         total = len(table)
         if self.output == 'cli':
             sys.stdout.write("\n")
+        results = []
         for idx_line, line in enumerate(table):
+            if line_to_process is not None:
+                if line_to_process != idx_line:
+                    continue
+                if idx_line > line_to_process:
+                    return results
             if self.output == 'cli':
                 left = None
                 if idx_line > 10:
@@ -1067,9 +1080,10 @@ class Importer(object):
                 sys.stdout.write(txt.encode('utf-8'))
                 sys.stdout.flush()
             try:
-                self._line_processing(idx_line, line)
+                results.append(self._line_processing(idx_line, line))
             except ImporterError, msg:
                 self.errors.append((idx_line, None, msg))
+        return results
 
     def _line_processing(self, idx_line, line):
         self.idx_line = idx_line
@@ -1084,12 +1098,10 @@ class Importer(object):
         self._item_post_processing = []
         data = {}
 
-        # keep in database the raw line for testing purpose
-        if self.IMPORTED_LINE_FIELD:
-            output = io.StringIO()
-            writer = csv.writer(output)
-            writer.writerow(line)
-            data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+        self.current_csv_line = None
+        # raw line for simulation
+        if self.simulate:
+            self.current_csv_line = line
 
         n = datetime.datetime.now()
         logger.debug('%s - Processing line %d' % (unicode(n - self.now),
@@ -1117,6 +1129,8 @@ class Importer(object):
         n2 = n
         if self.test:
             return
+        if self.simulate:
+            return data
         # manage unicity of items (mainly for updates)
         if 'history_modifier' in get_all_field_names(self.OBJECT_CLS):
             data['history_modifier'] = self.history_modifier
@@ -1188,6 +1202,7 @@ class Importer(object):
             formater.post_process(obj, data, val, owner=self.history_modifier)
 
         self.post_processing(idx_line, obj)
+        return data
 
     def _row_processing(self, c_row, idx_col, idx_line, val, data):
         if idx_col >= len(self.line_format):
@@ -1557,7 +1572,9 @@ class Importer(object):
             self.errors.append((self.idx_line, None, msg))
             data[attribute] = None
 
-    def get_object(self, cls, data, path=[]):
+    def get_object(self, cls, data, path=None):
+        if not path:
+            path = []
         m2ms = []
         if type(data) != dict:
             # if data is not a dict we don't know what to do
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index c3fbaa92d..10124b024 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -170,7 +170,7 @@ class ImporterType(models.Model):
                 if column.export_field_name:
                     LINE_EXPORT_FORMAT.append(
                         ImportFormater(column.export_field_name,
-                                        label=column.label)
+                                       label=column.label)
                     )
                 continue
             force_news = []
@@ -196,6 +196,7 @@ class ImporterType(models.Model):
             formater_kwargs['label'] = column.label
             formater_kwargs['required'] = column.required
             formater_kwargs['force_new'] = force_news
+            formater_kwargs['comment'] = column.description
             if column.export_field_name:
                 formater_kwargs['export_field_name'] = [
                     column.export_field_name]
@@ -865,9 +866,11 @@ class Import(models.Model):
         if self.state == 'A':
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Launch import")))
+            actions.append(('IS', _(u"Step by step import")))
         if self.state in ('F', 'FE'):
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Re-import")))
+            actions.append(('IS', _(u"Step by step import")))
             actions.append(('AC', _(u"Archive")))
         if self.state == 'AC':
             actions.append(('A', _(u"Unarchive")))
@@ -946,21 +949,22 @@ class Import(models.Model):
         self.save()
         return delayed_import(self.pk, session_key)
 
-    def importation(self, session_key=None):
+    def importation(self, session_key=None, line_to_process=None,
+                    simulate=False, return_importer_and_data=False):
         self.state = 'IP'
         self.end_date = datetime.datetime.now()
         self.save()
         importer = self.get_importer_instance()
         try:
-            importer.importation(self.data_table, user=self.user)
+            data = importer.importation(
+                self.data_table, user=self.user,
+                line_to_process=line_to_process, simulate=simulate)
         except IOError:
+            error_message = unicode(_(u"Error on imported file: {}")).format(
+                self.imported_file)
+            importer.errors = [error_message]
             if session_key:
-                put_session_message(
-                    session_key,
-                    unicode(_(u"Error on imported file: {}")).format(
-                        self.imported_file),
-                    "warning"
-                )
+                put_session_message(session_key, error_message, "warning")
                 ids = get_session_var(session_key, 'current_import_id')
                 if not ids:
                     ids = []
@@ -968,7 +972,9 @@ class Import(models.Model):
                 put_session_var(session_key, 'current_import_id', ids)
             self.state = 'FE'
             self.save()
-            return
+            if not return_importer_and_data:
+                return
+            return importer, None
         # result file
         filename = slugify(self.importer_type.name)
         now = datetime.datetime.now().isoformat('-').replace(':', '')
@@ -1006,6 +1012,8 @@ class Import(models.Model):
             )
         self.end_date = datetime.datetime.now()
         self.save()
+        if return_importer_and_data:
+            return importer, data
 
     def archive(self):
         self.state = 'AC'
diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html
new file mode 100644
index 000000000..4d7512de1
--- /dev/null
+++ b/ishtar_common/templates/ishtar/import_step_by_step.html
@@ -0,0 +1,50 @@
+{% extends "base.html" %}
+{% load i18n inline_formset %}
+
+{% block content %}
+<h2>{% trans "Import step by step" %}</h2>
+<h3>{{import.name}} &ndash; {% trans "line " %} {{line_number_displayed}}</h3>
+{% if errors %}
+<div class="alert alert-danger" role="alert">
+    <p>{% trans "The following error(s) has been encountered while parsing the source file:" %}</p>
+    <ul>{% for error in errors %}
+        <li>{{error}}</li>
+    {% endfor %}</ul>
+</div>
+
+{% if values %}
+<table class="table table-striped">
+    {% for header, value in values %}
+    <tr>
+        <th>{{header}}</th>
+        <td>{{value}}</td>
+    </tr>
+    {% endfor %}
+</table>
+{% endif %}
+
+{% else %}
+
+<div>
+    <table class="table table-striped">
+        <tr>
+            <th>{% trans "Column number" %}</th>
+            <th>{% trans "Name" %}</th>
+            <th>{% trans "Raw CSV value" %}</th>
+            <th>{% trans "Current value" %}</th>
+            <th>{% trans "New value" %}</th>
+            {% comment %}<th>{% trans "Merged value" %}</th>{% endcomment %}
+        </tr>
+        {% for idx, name, raw, interpreted in values %}<tr>
+            <td>{{idx}}</td>
+            <td>{{name|safe}}</td>
+            <td>{{raw}}</td>
+        <td></td>
+            <td>{{interpreted}}</td>
+        </tr>{% endfor %}
+    </table>
+
+</div>
+{% endif %}
+{% endblock %}
+
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index 5aa3bb409..1a7051b17 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -117,6 +117,8 @@ urlpatterns = [
         views.ImportDeleteView.as_view(), name='import_delete'),
     url(r'^import-link-unmatched/(?P<pk>[0-9]+)/$',
         views.ImportLinkView.as_view(), name='import_link_unmatched'),
+    url(r'^import-step-by-step/(?P<pk>[0-9]+)/(?P<line_number>[0-9]+)/$',
+        views.ImportStepByStepView.as_view(), name='import_step_by_step'),
 ]
 
 actions = []
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 8b0d51601..1af2f58ae 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1819,6 +1819,14 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
                     imprt.delayed_importation(request.session.session_key)
                 else:
                     imprt.importation()
+            elif action == 'IS':
+                if imprt.current_line is None:
+                    imprt.current_line = imprt.skip_lines
+                    imprt.save()
+                return HttpResponseRedirect(
+                    reverse('import_step_by_step',
+                            args=[imprt.pk, imprt.current_line + 1])
+                )
             elif action == 'AC':
                 imprt.archive()
         return HttpResponseRedirect(reverse(self.current_url))
@@ -1829,6 +1837,82 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
         return dct
 
 
+class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
+    template_name = 'ishtar/import_step_by_step.html'
+    page_name = _(u"Import step by step")
+    current_url = 'import_step_by_step'
+
+    def get(self, request, *args, **kwargs):
+        try:
+            self.imprt_obj = models.Import.objects.get(
+                pk=int(self.kwargs['pk'])
+            )
+        except (models.Import.DoesNotExist, ValueError):
+            raise Http404
+        if not self.request.user.is_superuser:
+            # user can only edit his own imports
+            user = models.IshtarUser.objects.get(pk=self.request.user.pk)
+            if self.imprt_obj.user != user:
+                raise Http404
+        self.current_line_number = int(self.kwargs['line_number']) - 1
+        self.imprt = None
+        try:
+            self.imprt, data = self.imprt_obj.importation(
+                simulate=True,
+                line_to_process=self.current_line_number,
+                return_importer_and_data=True
+            )
+        except IOError as e:
+            self.errors = [e.message]
+            return
+        self.errors, self.new_data = None, None
+        if not data:
+            self.errors = self.imprt.errors
+        else:
+            self.new_data = data[:]
+        return super(ImportStepByStepView, self).get(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        dct = super(ImportStepByStepView, self).get_context_data(**kwargs)
+        dct['import'] = self.imprt_obj
+        dct['line_number_displayed'] = self.kwargs['line_number']
+        dct['errors'] = self.errors
+        if self.errors:
+            if self.imprt.current_csv_line:
+                headers = [f.label for f in self.imprt.get_formaters()]
+                dct['values'] = zip(headers, self.imprt.current_csv_line)
+            return dct
+        headers, interpreted_values = [], []
+        for formater in self.imprt.get_formaters():
+            lbl = formater.label
+            if formater.comment:
+                lbl += u' <i data-toggle="tooltip" class="fa '
+                lbl += u'fa-question-circle"'
+                lbl += u' aria-hidden="true" title="{}">'.format(
+                    formater.comment.replace(u'"', u'&quot;'))
+                lbl += u'</i>'
+            headers.append(lbl)
+            field_name = formater.field_name[0]
+            if formater.export_field_name:
+                field_name = formater.export_field_name[0]
+            value = self.new_data[0].copy()
+            for key in field_name.split('__'):
+                if key in value:
+                    value = value[key]
+                else:
+                    value = u"-"
+            if value in (None, [], [None]):
+                value = _(u"* empty *")
+            if isinstance(value, list):
+                value = u" ; ".join(value)
+            interpreted_values.append(value)
+        dct['values'] = zip(
+            range(1, len(headers) + 1), headers, self.imprt.current_csv_line,
+            interpreted_values
+        )
+        return dct
+
+
 class ImportListTableView(ImportListView):
     template_name = 'ishtar/import_table.html'
     current_url = 'current_imports_table'