✨ Imports: built-in CSV viewer

11 files changed, 282 insertions, 65 deletions

diff --git a/changelog/en/changelog_2022-06-15.md b/changelog/en/changelog_2022-06-15.md
index e178ff7b0..cf881bdaf 100644
--- a/changelog/en/changelog_2022-06-15.md
+++ b/changelog/en/changelog_2022-06-15.md
@@ -2,8 +2,10 @@ v4.0.XX - 2099-12-31
 --------------------
 
 ### Features/improvements ###
+- pre-import forms
 - imports form: reorganisation of field order
 - import table :
+  - built-in CSV viewer
   - automatic progress refresh
   - reorganization of fields
   - improved presentation
diff --git a/changelog/fr/changelog_2023-01-25.md b/changelog/fr/changelog_2023-01-25.md
index 2f23ed826..b5265c4d0 100644
--- a/changelog/fr/changelog_2023-01-25.md
+++ b/changelog/fr/changelog_2023-01-25.md
@@ -2,8 +2,10 @@ v4.0.XX - 2099-12-31
 --------------------
 
 ### Fonctionnalités/améliorations ###
+- ajout de formulaire pré-imports
 - formulaire d'imports: réorganisation de l'ordre des champs
 - table des imports :
+  - visualisateur CSV intégré
   - raffrachissement automatique de l'avancement
   - réorganisation des champs
   - amélioration de la présentation
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index ca0a38832..fce25cedd 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -1397,6 +1397,24 @@ class BaseImport(models.Model):
     class Meta:
         abstract = True
 
+    @classmethod
+    def query_can_access(cls, user):
+        """
+        Filter the query to check access permissions
+        :param user: User instance
+        :return: import query
+        """
+        q = cls.objects
+        if user.is_superuser:
+            return q
+        ishtar_user = models.IshtarUser.objects.get(pk=user.pk)
+        q = q.filter(user=ishtar_user)
+        return q
+
+    @property
+    def group_prefix(self):
+        return ""
+
     @property
     def has_pre_import_form(self) -> bool:
         raise NotImplemented()
@@ -1452,6 +1470,10 @@ class ImportGroup(BaseImport):
         return f"{self.name} ({self.importer_type.name})"
 
     @property
+    def group_prefix(self):
+        return "group-"
+
+    @property
     def has_pre_import_form(self) -> bool:
         return False
 
diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js
index 1656cb427..9797c7d4a 100644
--- a/ishtar_common/static/js/ishtar.js
+++ b/ishtar_common/static/js/ishtar.js
@@ -955,7 +955,7 @@ function toggle_window_menu(){
     return false;
 }
 
-var register_qa_on_sheet = function(){
+var register_qa = function(){
     $(".btn-qa").click(function(){
         var target = $(this).attr('data-target');
         dt_qa_open(target);
@@ -1081,6 +1081,8 @@ var dt_qa_open = function (url, modal_id){
                 }
             );
             $('#' + modal_id).modal("show");
+            let table_scroll_height = $(window).height() - 180;
+            $(".table-scroll table").height(table_scroll_height + "px");
         },
         error: function() {
             close_wait();
diff --git a/ishtar_common/templates/ishtar/blocks/import_table_buttons_view.html b/ishtar_common/templates/ishtar/blocks/import_table_buttons_view.html
new file mode 100644
index 000000000..061dfe0f4
--- /dev/null
+++ b/ishtar_common/templates/ishtar/blocks/import_table_buttons_view.html
@@ -0,0 +1,18 @@
+{% load i18n %}
+<div class="btn-group btn-group-sm" role="group">
+    <span class="btn btn-outline-secondary no-hover">
+        <i class="{{logo}}" aria-hidden="true"></i>
+        {{ file_label }}
+    </span>
+    {% if file_type %}
+    <a class="btn btn-qa btn-secondary"
+       href='#'
+       data-target="{% url 'import_display_csv' file_type current_import.group_prefix current_import.pk %}"
+       onclick=""
+       title="{% trans 'View' %}">
+        <i class="fa fa-eye" aria-hidden="true"></i>
+    </a>{% endif %}
+    <a class="btn btn-secondary" href='{{file.url}}' title="{% trans 'Download' %}">
+        <i class="fa fa-download" aria-hidden="true"></i>
+    </a>
+</div>
diff --git a/ishtar_common/templates/ishtar/blocks/view_import_csv.html b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
new file mode 100644
index 000000000..f1b089ce2
--- /dev/null
+++ b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
@@ -0,0 +1,32 @@
+{% load i18n  %}
+
+<div class="modal-dialog full modal-lg">
+    <div class="modal-content">
+        <div class="modal-header">
+            <h2>{{ title }} |
+                <i class="{{icon}}" aria-hidden="true"></i>
+                {{target}}
+            </h2>
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                <span aria-hidden="true">&times;</span>
+            </button>
+        </div>
+        <div class="table-scroll">
+           <table class="table table-striped table-bordered">
+              <thead>
+              <tr>
+                  <th>&nbsp;</th>
+                  {% for head in header %}<th>{{head}}</th>{% endfor %}
+              </tr>
+              </thead>
+              <tbody>
+              {% for line in content %}
+              <tr>
+                  <td>{{ forloop.counter }}</td>
+                  {% for cell in line %}<td>{{cell}}</td>{% endfor %}
+              </tr>{% endfor %}
+              </tbody>
+            </table>
+        </div>
+    </div>
+</div>
diff --git a/ishtar_common/templates/ishtar/blocks/window_nav.html b/ishtar_common/templates/ishtar/blocks/window_nav.html
index 41572c5f2..a8c344bae 100644
--- a/ishtar_common/templates/ishtar/blocks/window_nav.html
+++ b/ishtar_common/templates/ishtar/blocks/window_nav.html
@@ -117,7 +117,7 @@
 {% endif %}
 <script type="text/javascript">
 $(document).ready(function(){
-    register_qa_on_sheet();
+    register_qa();
 });
 </script>
 {% else %}
diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html
index 3be92d42d..57e28abc3 100644
--- a/ishtar_common/templates/ishtar/import_table.html
+++ b/ishtar_common/templates/ishtar/import_table.html
@@ -1,7 +1,14 @@
 {% load i18n l10n inline_formset %}
+{% trans "Source" as source_label %}
+{% trans "Media" as media_label %}
+{% trans "Result" as result_label %}
+{% trans "Error" as error_label %}
+{% trans "Match" as match_label %}
 {% localize off %}<script type="text/javascript">
     {% comment %}
+
     /* TODO : à effacer ? */
+
     var html = $("#message_list").html();
     {% if MESSAGES and AJAX %}{% for message, message_type in MESSAGES %}
     html += '<div class="alert alert-{{message_type}} alert-dismissible fade show"';
@@ -14,6 +21,7 @@
     html += ' </div>';
     {% endfor %}{% endif %}
     $("#message_list").html(html);
+
     {% endcomment %}
 
     $("#import-list").find('select').prop('disabled', true);
@@ -30,6 +38,7 @@
     if (import_table_update_import_ids.length) need_refresh = true;
 
     $(document).ready(function(){
+        register_qa();
         if (need_refresh) setInterval(function(){
             import_table_update_import_list(import_table_update_import_ids)
         }, 3 * 1000);
@@ -51,6 +60,7 @@
             <th>{% trans "Diagnostic files" %}</th>{% endif %}
         </tr>
         {% for import in object_list %}
+        {% with current_import=import %}
         <tr id="import-{{import.import_id}}"
              class='import-row{% if import.has_error or not import.pre_import_form_is_valid %}-error{% endif %}{% if import.pk in refreshed_pks %} bg-info{% endif %}'>
             <td><ul class="simple">
@@ -74,38 +84,57 @@
                     {% endfor%}
                 </select>
             </td>
-            <td><ul class="simple">
-                {% if import.imported_file %}<li>
-                    <i class="fa fa-fw fa-file-text-o" aria-hidden="true"></i> <a href='{{import.imported_file.url}}'>{% trans "Source" %}</a>
+            <td><ul class="simple table-import-files">
+                {% if import.imported_file %}<li class="p-1">
+                {% with file_label=source_label logo='fa fa-fw fa-file-text-o' file_type='source' file=import.imported_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
                 </li>
-                {% if import.get_imported_images %}<li>
-                    <i class="fa fa-fw fa-file-image-o" aria-hidden="true"></i> <a href="{{ import.get_imported_images.url }}">{% trans "Media" %}</a>
+                {% if import.get_imported_images %}<li class="p-1">
+                {% with file_label=media_label logo='fa fa-fw fa-file-image-o' file_type='' file=import.get_imported_images %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
                 </li>{% endif %}
                 {% elif import.archive_file %}<li>
                     <i class="fa fa-fw fa-file-archive-o" aria-hidden="true"></i> <a href='{{import.archive_file.url}}'>{% trans "Archive" context "name" %}</a>
                 </li>{% endif %}
             </ul></td>{% if not ARCHIVE_PAGE %}
-            <td><ul class="simple">
-                {% if import.has_pre_import_form %}<li>
-                    {% if not import.pre_import_form_is_valid %}
-                    <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i>
-                    {% else %}
-                    <i class="fa fa-fw fa-check-square-o" aria-hidden="true"></i>
-                    {% endif %}
-                    <a href='{% url "import_pre_import_form" import.pk %}'>{% trans "Pre-import values" %}</a>
+            <td><ul class="simple table-import-match-files">
+                {% if import.has_pre_import_form %}<li class="p-1">
+                <div class="btn-group btn-group-sm" role="group">
+                    <a class="btn btn-secondary" href='{% url "import_pre_import_form" import.pk %}'>
+                        {% if not import.pre_import_form_is_valid %}
+                        <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i>
+                        {% else %}
+                        <i class="fa fa-fw fa-check-square-o" aria-hidden="true"></i>
+                        {% endif %}
+                        {% trans "Pre-import values" %}
+                    </a>
+                </div>
                 </li>{% endif %}
-                {% if import.need_matching %}<li>
-                    <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> <a href='{% url "import_link_unmatched" import.pk %}'>{% trans "Make match" %}</a>
+                {% if import.need_matching %}<li class="p-1">
+                <div class="btn-group btn-group-sm" role="group">
+                    <a class="btn btn-secondary" href='{% url "import_link_unmatched" import.pk %}'>
+                        <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> {% trans "Make match" %}
+                    </a>
+                </div>
                 </li>{% endif %}
             </ul></td>
-            <td style="white-space: nowrap;"><ul class="simple">{% if import.error_file %}<li>
-                <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i> <a href='{{import.error_file.url}}'>{% trans "Error" %}</a>
+            <td style="white-space: nowrap;"><ul class="simple table-import-diag">
+                {% if import.error_file %}<li class="p-1">
+                {% with file_label=error_label logo='text-danger fa fa-fw fa-exclamation-triangle' file_type='error' file=import.error_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
             </li>{% endif %}
-            {% if import.result_file %}<li>
-                 <i class="fa fa-fw fa-th" aria-hidden="true"></i> <a href='{{import.result_file.url}}'>{% trans "Result" %}</a>
+            {% if import.result_file %}<li class="p-1">
+                {% with file_label=result_label logo='fa fa-fw fa-th' file_type='result' file=import.result_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
             </li>{% endif %}
-            {% if import.match_file %}<li>
-                <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> <a href='{{import.match_file.url}}'>{% trans "Match" %}</a>
+            {% if import.match_file %}<li class="p-1">
+                {% with file_label=match_label logo='fa fa-fw fa-arrows-h' file_type='match' file=import.match_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
             </li>{% endif %}</ul>
             </td>{% endif %}
         </tr>
@@ -122,46 +151,66 @@
             </td>
         </tr>
         {% endif %}
+        {% endwith %}
         {% if not import.importer_type.type_label and not ARCHIVE_PAGE  %} {# group #}
         {% for sub in import.import_list %}
+        {% with current_import=sub %}
         <tr id="import-{{sub.import_id}}">
             <td></td>
             <td>{{sub.importer_type}}</td>
             <td id="status-{{sub.import_id}}">{{sub.status}}</td>
             <td></td>
-            <td><ul class="simple">
-                {% if sub.imported_file %}<li>
-                    <i class="fa fa-fw fa-file-text-o" aria-hidden="true"></i> <a href='{{sub.imported_file.url}}'>{% trans "Source" %}</a>
+            <td><ul class="simple table-import-files">
+                {% if sub.imported_file %}<li class="p-1">
+                {% with file_label=source_label logo='fa fa-fw fa-file-text-o' file_type='source' file=sub.imported_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
                 </li>{% endif %}
-                {% if sub.get_imported_images %}<li>
-                    <i class="fa fa-fw fa-file-image-o" aria-hidden="true"></i> <a href="{{ sub.get_imported_images.url }}">{% trans "Media" %}</a>
+                {% if sub.get_imported_images %}<li class="p-1">
+                {% with file_label=media_label logo='fa fa-fw fa-file-image-o' file_type='' file=sub.get_imported_images %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
                 </li>{% endif %}
             </ul></td>{% if not ARCHIVE_PAGE %}
-            <td><ul class="simple">
-                {% if sub.has_pre_import_form %}<li>
-                {% if not sub.pre_import_form_is_valid %}
-                <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i>
-                {% else %}
-                <i class="fa fa-fw fa-check-square-o" aria-hidden="true"></i>
-                {% endif %}
-                <a href='{% url "import_pre_import_form" sub.pk %}'>{% trans "Pre-import values" %}</a>
-            </li>{% endif %}
-            {% if sub.need_matching %}<li>
-                <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> <a href='{% url "import_link_unmatched" sub.pk %}'>{% trans "Make match" %}</a>
-            </li>{% endif %}
+            <td><ul class="simple table-import-match-files">
+                {% if sub.has_pre_import_form %}<li class="p-1">
+                <div class="btn-group btn-group-sm" role="group">
+                    <a class="btn btn-secondary" href='{% url "import_pre_import_form" sub.pk %}'>
+                        {% if not sub.pre_import_form_is_valid %}
+                        <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i>
+                        {% else %}
+                        <i class="fa fa-fw fa-check-square-o" aria-hidden="true"></i>
+                        {% endif %}
+                        {% trans "Pre-import values" %}
+                    </a>
+                </div>
+                </li>{% endif %}
+                {% if sub.need_matching %}<li class="p-1">
+                <div class="btn-group btn-group-sm" role="group">
+                    <a class="btn btn-secondary" href='{% url "import_link_unmatched" sub.pk %}'>
+                        <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> {% trans "Make match" %}
+                    </a>
+                </div>
+                </li>{% endif %}
             </ul></td>
-            <td><ul class="simple">
-            {% if sub.error_file %}<li>
-                <i class="text-danger fa fa-fw fa-exclamation-triangle" aria-hidden="true"></i> <a href='{{sub.error_file.url}}'>{% trans "Error" %}</a>
-            </li>{% endif %}
-            {% if sub.result_file %}<li>
-                <i class="fa fa-fw fa-th" aria-hidden="true"></i> <a href='{{sub.result_file.url}}'>{% trans "Result" %}</a>
+            <td style="white-space: nowrap;"><ul class="simple table-import-diag">
+                {% if sub.error_file %}<li class="p-1">
+                {% with file_label=error_label logo='text-danger fa fa-fw fa-exclamation-triangle' file_type='error' file=sub.error_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
             </li>{% endif %}
-            {% if sub.match_file %}<li>
-                <i class="fa fa-fw fa-arrows-h" aria-hidden="true"></i> <a href='{{sub.match_file.url}}'>{% trans "Match" %}</a>
+                {% if sub.result_file %}<li class="p-1">
+                {% with file_label=result_label logo='fa fa-fw fa-th' file_type='result' file=sub.result_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
             </li>{% endif %}
-            </ul>
-            </td>{% endif %}
+                {% if sub.match_file %}<li class="p-1">
+                {% with file_label=match_label logo='fa fa-fw fa-arrows-h' file_type='match' file=sub.match_file %}
+                {% include "ishtar/blocks/import_table_buttons_view.html" %}
+                {% endwith %}
+            </li>{% endif %}</ul>
+            </td>
+            {% endif %}
         </tr>
         <tr></tr>{# only for even and odd style #}
         <tr id="progress-display-{{sub.id}}"{% if sub.state != 'IP' and sub.state != 'PP' %} style="display:none"{% endif %}>
@@ -174,6 +223,7 @@
                 </div>
             </td>
         </tr>
+        {% endwith %}
         {% endfor %}
         {% endif %}
         {% endfor %}
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index ebcb132b8..2dbc13434 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -264,6 +264,11 @@ urlpatterns = [
         name="import_link_unmatched",
     ),
     url(
+        r"^import-csv-view/(?P<target>source|result|match|error)/(?P<group>group\-)?(?P<pk>[0-9]+)/$",
+        views.ImportCSVView.as_view(),
+        name="import_display_csv",
+    ),
+    url(
         r"^import-step-by-step/all/(?P<pk>[0-9]+)/(?P<line_number>[0-9]+)/$",
         views.ImportStepByStepView.as_view(),
         name="import_step_by_step_all",
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 4a76207f6..ca9df098c 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1553,13 +1553,12 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
         return query.exclude(state="AC")
 
     def get_queryset(self):
-        q1 = self._queryset_filter(self.model.objects)
-        q2 = self._queryset_filter(models.ImportGroup.objects)
-        if not self.request.user.is_superuser:
-            user = models.IshtarUser.objects.get(pk=self.request.user.pk)
-            q1 = q1.filter(user=user)
-            q2 = q2.filter(user=user)
+        user = self.request.user
+        if not user.pk:
+            raise Http404()
+        q1 = self._queryset_filter(self.model.query_can_access(user))
         q1 = q1.filter(group__isnull=True).order_by("-creation_date", "-pk")
+        q2 = self._queryset_filter(models.ImportGroup.query_can_access(user))
         q2 = q2.order_by("-creation_date", "-pk")
         return list(reversed(sorted(list(q1) + list(q2), key=lambda x: x.creation_date)))
 
@@ -2143,6 +2142,55 @@ class ImportGroupDeleteView(IshtarMixin, LoginRequiredMixin, DeleteView):
         return reverse("current_imports")
 
 
+class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView):
+    template_name = "ishtar/blocks/view_import_csv.html"
+    ATTRIBUTES = {
+        "source": "imported_file",
+        "error": "error_file",
+        "result": "result_file",
+        "match": "match_file"
+    }
+    TITLES = {
+        "source": ("fa fa-file-text-o", _("Source")),
+        "error": ("text-danger fa fa-exclamation-triangle", _("Error")),
+        "result": ("fa fa-th", _("Result")) ,
+        "match": ("fa fa-arrows-h", _("Match")),
+    }
+
+    def get(self, request, *args, **kwargs):
+        user = self.request.user
+        if not user.pk:
+            raise Http404()
+        model = models.ImportGroup if kwargs.get("group", None) else models.Import
+        q = model.query_can_access(self.request.user).filter(pk=kwargs.get("pk", -1))
+        if not q.count():
+            raise Http404()
+        self.import_item = q.all()[0]
+        attribute = self.ATTRIBUTES[kwargs["target"]]
+        self.csv_file = getattr(self.import_item, attribute)
+        if not self.csv_file:
+            raise Http404()
+        return super().get(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        data = super().get_context_data(**kwargs)
+        encoding = "utf-8"
+        if self.kwargs["target"] == "source" and self.import_item.encoding:
+            encoding = self.import_item.encoding
+        data["icon"], data["target"] = self.TITLES[self.kwargs["target"]]
+        data["title"] = str(self.import_item)
+        data["content"] = []
+        with open(self.csv_file.path, "r", encoding=encoding) as f:
+            reader = csv.reader(f)
+            for idx, line in enumerate(reader):
+                if not idx:
+                    data["header"] = line
+                    continue
+                data["content"].append(line)
+        data["window_id"] = "csv-view-" + (self.kwargs.get("group", "") or "") + str(self.import_item.pk)
+        return data
+
+
 class PersonCreate(LoginRequiredMixin, CreateView):
     model = models.Person
     form_class = forms.BasePersonForm
diff --git a/scss/custom.scss b/scss/custom.scss
index 934d67937..c5ee6d060 100644
--- a/scss/custom.scss
+++ b/scss/custom.scss
@@ -123,6 +123,16 @@ input[type="file"].form-control{
 	border-collapse: separate;
 }
 
+.btn-outline-secondary.no-hover:hover {
+  color: inherit;
+  background-color: inherit;
+  border-color: inherit;
+}
+
+.btn.no-hover {
+    color: #212529;
+}
+
 .has-previous-value .form-control,
 .has-previous-value .btn,
 .has-previous-value .input-group-text{
@@ -289,6 +299,41 @@ pre {
     background-color: white;
 }
 
+.import-row-error,
+.table-striped tbody tr:nth-of-type(2n+1).import-row-error {
+    background-color: lighten(red, 40%);
+}
+
+
+#import-container {
+    overflow: scroll;
+}
+
+#import-list li {
+    white-space: nowrap;
+}
+
+ul.table-import-files > li  span.btn{
+    width: 6rem;
+}
+
+ul.table-import-diag > li  span.btn{
+    width: 10rem;
+}
+
+.table-scroll {
+    padding: 1em;
+
+    table {
+        display:block;
+        overflow-x : scroll;
+        overflow-y : scroll;
+    }
+    thead {
+        position: sticky;
+        top: -1px;
+    }
+}
 
 .tab-content{
     padding-top: 0;
@@ -672,15 +717,6 @@ div#validation-bar{
     color: darken(red, 20%);
 }
 
-.import-row-error,
-.table-striped tbody tr:nth-of-type(2n+1).import-row-error {
-    background-color: lighten(red, 40%);
-}
-
-#import-list li {
-white-space: nowrap;
-}
-
 /* context menu */
 #shortcut-menu {
     width: 700px;