Step by step: large improvments on step display (refs #3975)

Show modified, sheet access for existing items, link between created,
updated fields and associated column.

7 files changed, 248 insertions, 68 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 4519241af..3bcd62415 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -1632,7 +1632,10 @@ class Importer(object):
         for k in create_dict.keys():
             # filter unnecessary default values but not the json field
             if type(create_dict[k]) == dict and k != 'data':
-                create_dict.pop(k)
+                if self.simulate:
+                    create_dict[k] = _(u"* created *")
+                else:
+                    create_dict.pop(k)
             # File doesn't like deepcopy
             elif type(create_dict[k]) == File:
                 create_dict[k] = copy.copy(data[k])
@@ -1776,16 +1779,26 @@ class Importer(object):
                         # force post save script
                         v.save()
             if self.simulate:
+                # put m2m result in data dict
+                current_data = data
+                if m2ms:
+                    for item in path:
+                        if item not in current_data:
+                            current_data[item] = {}
+                        current_data = current_data[item]
+                    for key, value in m2ms:
+                        if not isinstance(value, list) and \
+                                not isinstance(value, tuple):
+                            value = [value]
+                        current_data[key] = value
+
                 if created:
-                    for k in dct.keys():
-                        # do not present empty value
-                        if dct[k] in (None, ''):
-                            dct.pop(k)
-                    return _(u"* created item *"), True
+                    return dct, True
                 else:
                     # defaults are not presented as matching data
                     dct.pop('defaults')
                     return self.updated_objects[-1][1], False
+
             if m2ms:
                 # force post save script
                 obj.save()
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 883c12ab9..6518ea763 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -459,6 +459,10 @@ class GeneralType(Cached, models.Model):
     def natural_key(self):
         return (self.txt_idx, )
 
+    @property
+    def explicit_label(self):
+        return u"{} ({})".format(self.label, self._meta.verbose_name)
+
     @classmethod
     def create_default_for_test(cls):
         return [cls.objects.create(label='Test %d' % i) for i in range(5)]
diff --git a/ishtar_common/static/media/styles.css b/ishtar_common/static/media/styles.css
index 4ab36afe2..9507674b7 100644
--- a/ishtar_common/static/media/styles.css
+++ b/ishtar_common/static/media/styles.css
@@ -14,6 +14,10 @@ html{
     background-position: right 30px top 150px;
 }
 
+.text-large{
+    font-size: 1.8em;
+}
+
 .form-thumbnail{
     max-width: 120px;
     max-height: 120px;
@@ -89,6 +93,10 @@ button:hover{
     cursor: pointer;
 }
 
+.fa-question-circle{
+    cursor: help;
+}
+
 div#foot{
     font-size: 0.75em;
     text-align: center;
diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html
index 233889df4..c827156c5 100644
--- a/ishtar_common/templates/ishtar/import_step_by_step.html
+++ b/ishtar_common/templates/ishtar/import_step_by_step.html
@@ -1,9 +1,8 @@
 {% extends "base.html" %}
-{% load i18n inline_formset %}
+{% load i18n inline_formset link_to_window %}
 
 {% block content %}
-<h2>{% trans "Import step by step" %}</h2>
-<h3>{{import.name}} &ndash; {% trans "line " %} {{line_number_displayed}}</h3>
+<h2>{% trans "Import step by step" %} &ndash; {{import.name}} &ndash; {% trans "line " %} {{line_number_displayed}}</h2>
 {% if errors %}
 <div class="alert alert-danger" role="alert">
     <p>{% trans "The following error(s) has been encountered while parsing the source file:" %}</p>
@@ -26,74 +25,93 @@
 {% else %}
 
 {% if new_objects %}
-<h4>{% trans "Objects to be created" %}</h4>
-<div>
-    <table class="table table-striped">
-        <tr>
-            <th>{% trans "Path" %}</th>
-            <th>{% trans "Type" %}</th>
-            <th>{% trans "Values" %}</th>
-        </tr>
+<h3>{% trans "Objects to be created" %}</h3>
 {% for path, cls, values in new_objects %}
+<div class="card">
+  <div class="card-body">
+    <h5 class="card-title">{{path|safe}} ({{cls.get_verbose_name}})</h5>
+  </div>
+  <div class="card-body">
+    <table class="table table-striped">
         <tr>
-            <td>{{path}}</td>
-            <td>{{cls.get_verbose_name}}</td>
-            <td>{% for k, value in values.items %}<div class="row">
-                <div class="col-6"><strong>{{k}}{% trans ":"%}</strong></div>
-                <div class="col-6">{{value}}</div>
-            </div>{% endfor %}
-            </td>
+            <th>{% trans "Key" %}</th>
+            <th>{% trans "Value" %}</th>
         </tr>
-{% endfor %}
+        {% for k, value in values.items %}<tr>
+            <th>{{k|safe}}</th>
+            <td>{{value}}</td>
+        </tr>{% endfor %}
     </table>
+  </div>
 </div>
+<hr/>
+{% endfor %}
 {% endif %}
 
 {% if updated_objects %}
-<h4>{% trans "Objects to be updated" %}</h4>
-<div>
-    <table class="table table-striped">
-        <tr>
-            <th>{% trans "Path" %}</th>
-            <th>{% trans "Object" %}</th>
-            <th>{% trans "Matching values" %}</th>
-            <th>{% trans "Updated values" %}</th>
-        </tr>
-        {% for path, obj, values, updated_values in updated_objects %}
-        <tr>
-            <td>{{path}}</td>
-            <td>{{obj}} ({{obj.get_verbose_name}})</td>
-            <td>{% for k, value in values.items %}<div class="row">
-                <div class="col-6"><strong>{{k}}{% trans ":"%}</strong></div>
-                <div class="col-6">{{value}}</div>
-            </div>{% endfor %}
-            <td>{% for k, value in updated_values.items %}<div class="row">
-                <div class="col-6"><strong>{{k}}{% trans ":"%}</strong></div>
-                <div class="col-6">{{value}}</div>
-            </div>{% endfor %}
-            </td>
-        </tr>
-        {% endfor %}
-    </table>
+<h3>{% trans "Objects to be updated" %}</h3>
+{% for path, obj, values, old_and_updated in updated_objects %}
+
+<div class="card">
+    <div class="card-body">
+        <h5 class="card-title">{{path}} &ndash; {{obj}} {{obj|link_to_window}} ({{obj.get_verbose_name}})</h5>
+    </div>
+    <div class="card-body">
+
+        <h5>{% trans "Matching values" %}</h5>
+        <table class="table table-striped">
+            <tr>
+                <th>{% trans "Key" %}</th>
+                <th>{% trans "Value" %}</th>
+            </tr>
+            {% for k, value in values.items %}<tr>
+                <th>{{k|safe}}</th>
+                <td>{{value}}</td>
+            </tr>{% endfor %}
+        </table>
+
+
+        <h5>{% trans "Updated values" %}</h5>
+        <table class="table table-striped">
+            <tr>
+                <th>{% trans "Key" %}</th>
+                <th>{% trans "Old value" %}</th>
+                <th>{% trans "New value" %}</th>
+                <th>{% trans "Changed" %}</th>
+            </tr>
+            {% for k, value in old_and_updated.items %}<tr>
+                <th>{{k|safe}}</th>
+                <td>{{value.0|safe}}</td>
+                <td>{{value.1|safe}}</td>
+                <td class="text-center">{% if value.2%}
+                    <i class="fa fa-check-circle-o text-info text-large"
+                       aria-hidden="true"></i>{% else %}&ndash;{% endif %}</td>
+            </tr>{% endfor %}
+        </table>
+    </div>
 </div>
+
+<hr/>
+{% endfor %}
+
 {% endif %}
 
+<h3>{% trans "CSV values" %}</h3>
+
 <div>
     <table class="table table-striped">
         <tr>
             <th>{% trans "Column number" %}</th>
             <th>{% trans "Name" %}</th>
             <th>{% trans "Raw CSV value" %}</th>
-            <th>{% trans "Current value" %}</th>
-            <th>{% trans "New value" %}</th>
+            <th>{% trans "Interpreted value" %}</th>
             {% comment %}<th>{% trans "Merged value" %}</th>{% endcomment %}
         </tr>
-        {% for idx, name, raw, interpreted in values %}<tr>
+        {% for idx, name, raw, interpreted in values %}<tr id="col-{{idx}}">
             <td>{{idx}}</td>
             <td>{{name|safe}}</td>
             <td>{{raw}}</td>
-        <td></td>
-            <td>{{interpreted}}</td>
+            <td>{{interpreted|safe}}</td>
         </tr>{% endfor %}
     </table>
 
diff --git a/ishtar_common/templatetags/link_to_window.py b/ishtar_common/templatetags/link_to_window.py
index f157b6255..fca5a9f91 100644
--- a/ishtar_common/templatetags/link_to_window.py
+++ b/ishtar_common/templatetags/link_to_window.py
@@ -10,7 +10,7 @@ register = Library()
 
 @register.filter
 def link_to_window(item):
-    if not item:
+    if not hasattr(item, 'SHOW_URL'):
         return ""
     return mark_safe(
         u' <a class="display_details" href="#" '
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 948604c16..5a03f7f5f 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -396,3 +396,19 @@ def get_session_var(session_key, key):
     if key not in session:
         return
     return session[key]
+
+
+def get_field_labels_from_path(model, path):
+    """
+    :param model: base model
+    :param path: list of attribute starting from the base model
+    :return: list of labels
+    """
+    labels = []
+    for key in path:
+        field = model._meta.get_field(key)
+        if hasattr(field, 'verbose_name'):
+            labels.append(field.verbose_name)
+        if field.one_to_many or field.one_to_one or field.many_to_many:
+            model = field.model
+    return labels
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index a45a9f87c..1fa968a95 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -71,12 +71,14 @@ from archaeological_finds.forms import DashboardTreatmentForm, \
 from ishtar_common.forms import FinalForm, FinalDeleteForm
 from ishtar_common.widgets import JQueryAutoComplete
 from ishtar_common.utils import get_random_item_image_link, shortify, \
-    get_all_field_names
+    get_all_field_names, get_field_labels_from_path
 from ishtar_common import forms_common as forms
 from ishtar_common import wizards
 from ishtar_common.models import HistoryError, PRIVATE_FIELDS, \
     get_current_profile
 
+from ishtar_common.templatetags.link_to_window import link_to_window
+
 import models
 
 CSV_OPTIONS = {'delimiter': ',', 'quotechar': '"', 'quoting': csv.QUOTE_ALL}
@@ -1882,8 +1884,8 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
                 headers = [f.label for f in self.imprt.get_formaters()]
                 dct['values'] = zip(headers, self.imprt.current_csv_line)
             return dct
-        headers, interpreted_values = [], []
-        for formater in self.imprt.get_formaters():
+        headers, self.path_to_column, interpreted_values = [], {}, []
+        for idx, formater in enumerate(self.imprt.get_formaters()):
             lbl = formater.label
             if formater.comment:
                 lbl += u' <i data-toggle="tooltip" class="fa '
@@ -1896,26 +1898,145 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
             if formater.export_field_name:
                 field_name = formater.export_field_name[0]
             value = self.new_data[0].copy()
-            for key in field_name.split('__'):
-                if value and key in value:
+
+            field_name_tuple = field_name.split(u'__')
+            # associate each path level to this column
+            while field_name_tuple:
+                current_field_name = u'__'.join(field_name_tuple)
+                if current_field_name not in self.path_to_column:
+                    self.path_to_column[current_field_name] = []
+                self.path_to_column[current_field_name].append(idx)
+                field_name_tuple.pop()
+
+            for idx, key in enumerate(field_name.split('__')):
+                if isinstance(value, dict) and key in value:
                     value = value[key]
+                elif not idx:
+                    # no key reach
+                    value = None
                 else:
-                    value = u"-"
-            if value in (None, [], [None]):
-                value = _(u"* empty *")
-            if isinstance(value, list):
-                value = u" ; ".join(value)
+                    break
+            value = self.get_value(value)
             interpreted_values.append(value)
         dct['values'] = zip(
             range(1, len(headers) + 1), headers, self.imprt.current_csv_line,
             interpreted_values
         )
-        dct['new_objects'] = self.imprt.new_objects
-        dct['updated_objects'] = self.imprt.updated_objects
+
+        new_objects = {}
+        for path, cls, new_dct in self.imprt.new_objects:
+            # transform path to explicit label
+            label = self.transform_path_to_label(self.imprt.OBJECT_CLS, path)
+
+            created_dict = {}
+            for k, val in new_dct.items():
+                if val in ('', None, [], [None]):
+                    continue
+                created_dict[k] = val
+            # check if it is not previously created
+            key = (cls, tuple(sorted(created_dict.items())))
+            if key in new_objects:
+                # regroup it
+                new_objects[key][0].append(label)
+                continue
+
+            # values - transform key to explicit label
+            value_dct = self.transform_keys_to_label(path, cls, created_dict)
+
+            for k in value_dct.keys():
+                value_dct[k] = self.get_value(value_dct[k])
+
+            new_objects[key] = ([label], cls, value_dct)
+
+        dct['new_objects'] = [
+            [u" &ndash; ".join(lbls), cls, new_dct]
+            for lbls, cls, new_dct in new_objects.values()
+        ]
+
+        updated_objects = []
+        for path, obj, values, updated_values in self.imprt.updated_objects:
+
+            # transform path to explicit label
+            label = self.transform_path_to_label(obj.__class__, path)
+
+            # transform key into explicit label
+            values = self.transform_keys_to_label(path, obj.__class__, values)
+
+            # add current values and changed bool
+            old_and_updated = {}
+            for k in updated_values.keys():
+                current_value = getattr(obj, k)
+                updated_value = updated_values[k]
+                if hasattr(current_value, 'all'):
+                    current_value = list(current_value.all())
+                    changed = False
+                    for v in updated_value:
+                        if v not in current_value:
+                            changed = True
+                    current_value = self.list_to_html(current_value)
+                    updated_value = self.list_to_html(updated_value)
+                else:
+                    changed = current_value != updated_value
+                    current_value = self.get_value(current_value)
+                    updated_value = self.get_value(updated_value)
+                old_and_updated[k] = [current_value, updated_value,
+                                      changed]
+
+            # transform key into explicit label
+            old_and_updated = self.transform_keys_to_label(path, obj.__class__,
+                                                           old_and_updated)
+            updated_objects.append((label, obj, values, old_and_updated))
+
+        dct['updated_objects'] = updated_objects
         dct['ambiguous_objects'] = self.imprt.ambiguous_objects
         dct['not_find_objects'] = self.imprt.not_find_objects
         return dct
 
+    def transform_path_to_label(self, cls, path):
+        label = u" > ".join(
+            unicode(l)
+            for l in get_field_labels_from_path(cls, path)
+        )
+        if not label:
+            label = unicode(cls._meta.verbose_name)
+        return label
+
+    def transform_keys_to_label(self, path, cls, dct):
+        value_dct = {}
+        for k in dct:
+            label = unicode(
+                get_field_labels_from_path(cls, [k])[0]
+            )
+
+            concat_path = u"__".join(list(path) + [k])
+            if concat_path in self.path_to_column:
+                for col in self.path_to_column[concat_path]:
+                    col += 1
+                    label += u" <a href=\"#col-{}\">"\
+                             u"<span class=\"badge badge-info\"> {} {} </span>"\
+                             u"</a>".format(col, _(u"Col. "), col)
+            value_dct[label] = dct[k]
+        return value_dct
+
+    def list_to_html(self, lst):
+        if not lst:
+            return _(u"* empty *")
+        return u"<ul class='list-group'><li class='list-group-item'>" + \
+               u"</li><li class='list-group-item'>".join([
+                   self.get_value(item) for item in lst
+               ]) + u"</li></ul>"
+
+    def get_value(self, item):
+        if hasattr(item, 'SHOW_URL'):
+            return u"{}{}".format(unicode(item), link_to_window(item))
+        if hasattr(item, 'explicit_label'):
+            return item.explicit_label
+        if item in (None, [], [None]):
+            return _(u"* empty *")
+        if isinstance(item, list):
+            return self.list_to_html(item)
+        return unicode(item)
+
 
 class ImportListTableView(ImportListView):
     template_name = 'ishtar/import_table.html'