✨ imports: allow errors to be pointed out line by line

14 files changed, 156 insertions, 22 deletions

diff --git a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
index 6c9a5e085..b68f81937 100644
--- a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
+++ b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
 
 from django.db import migrations, models
 
@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0230_auto_20231024_1045'),
         ('archaeological_context_records', '0112_migrate_created'),
     ]
 
diff --git a/archaeological_files/migrations/0112_file_imports_updated.py b/archaeological_files/migrations/0112_file_imports_updated.py
index d227e0487..ab076fbc8 100644
--- a/archaeological_files/migrations/0112_file_imports_updated.py
+++ b/archaeological_files/migrations/0112_file_imports_updated.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
 
 from django.db import migrations, models
 
@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0231_default_mandatory_keys'),
         ('archaeological_files', '0111_migrate_created'),
     ]
 
diff --git a/archaeological_finds/migrations/0113_auto_20231018_1553.py b/archaeological_finds/migrations/0113_auto_20231024_1050.py
index e9bfad7fd..a1ede77e6 100644
--- a/archaeological_finds/migrations/0113_auto_20231018_1553.py
+++ b/archaeological_finds/migrations/0113_auto_20231024_1050.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
 
 from django.db import migrations, models
 
@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0231_default_mandatory_keys'),
         ('archaeological_finds', '0112_migrate_created'),
     ]
 
diff --git a/archaeological_operations/migrations/0113_auto_20231018_1551.py b/archaeological_operations/migrations/0113_auto_20231024_1045.py
index 11d1143d0..bb0b8ac0e 100644
--- a/archaeological_operations/migrations/0113_auto_20231018_1551.py
+++ b/archaeological_operations/migrations/0113_auto_20231024_1045.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
 
 from django.db import migrations, models
 
@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0230_auto_20231024_1045'),
         ('archaeological_operations', '0112_migrate_created'),
     ]
 
diff --git a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py
index b84564711..5b908736d 100644
--- a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py
+++ b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
 
 from django.db import migrations, models
 
@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0231_default_mandatory_keys'),
         ('archaeological_warehouse', '0118_auto_20230807_1106'),
     ]
 
diff --git a/ishtar_common/migrations/0230_auto_20231018_1551.py b/ishtar_common/migrations/0230_auto_20231024_1045.py
index c2fc2abe0..f9715bd65 100644
--- a/ishtar_common/migrations/0230_auto_20231018_1551.py
+++ b/ishtar_common/migrations/0230_auto_20231024_1045.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
 
 import django.core.validators
 from django.db import migrations, models
@@ -153,6 +153,21 @@ class Migration(migrations.Migration):
             field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'),
         ),
         migrations.CreateModel(
+            name='ImportLineError',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('line', models.PositiveIntegerField(verbose_name='Line')),
+                ('ignored', models.BooleanField(default=False, verbose_name='Ignored')),
+                ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='error_lines', to='ishtar_common.Import')),
+            ],
+            options={
+                'verbose_name': 'Import - Ignored error',
+                'verbose_name_plural': 'Import - Ignored error',
+                'ordering': ('import_item', 'line'),
+                'unique_together': {('line', 'import_item')},
+            },
+        ),
+        migrations.CreateModel(
             name='ImporterGroupImporter',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
diff --git a/ishtar_common/migrations/0231_default_mandatory_keys.py b/ishtar_common/migrations/0231_default_mandatory_keys.py
index 568e60b9a..cd245322a 100644
--- a/ishtar_common/migrations/0231_default_mandatory_keys.py
+++ b/ishtar_common/migrations/0231_default_mandatory_keys.py
@@ -30,7 +30,7 @@ def migrate(apps, __):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ishtar_common', '0230_auto_20231018_1551'),
+        ('ishtar_common', '0230_auto_20231024_1045'),
     ]
 
     operations = [
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a38fc3c3e..d7a546696 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -107,6 +107,7 @@ from ishtar_common.model_managers import (
 )
 from ishtar_common.model_merging import merge_model_objects
 from ishtar_common.models_imports import (
+    ImportLineError,
     ImporterModel,
     ImporterType,
     ImporterGroup,
@@ -181,6 +182,7 @@ __all__ = [
     "ImporterType",
     "ImporterGroup",
     "ImportGroup",
+    "ImportLineError",
     "ImporterGroupImporter",
     "ImporterDefault",
     "ImporterDefaultValues",
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index a240f4326..77dad558e 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -47,6 +47,7 @@ from django.core.exceptions import ValidationError, SuspiciousOperation
 from django.core.files import File
 from django.core.files.base import ContentFile
 from django.core.validators import validate_comma_separated_integer_list, MinValueValidator
+from django.db.models import Q
 from django.db.models.base import ModelBase
 from django.db.models.signals import pre_delete
 from django.template.defaultfilters import slugify
@@ -1423,7 +1424,7 @@ class BaseImport(models.Model, OwnPerms, SheetItem):
             return q
         IshtarUser = apps.get_model("ishtar_common", "IshtarUser")
         ishtar_user = IshtarUser.objects.get(pk=user.pk)
-        q = q.filter(user=ishtar_user)
+        q = q.filter(Q(user=ishtar_user) | Q(importer_type__users__pk=ishtar_user.pk))
         return q
 
     @classmethod
@@ -1924,9 +1925,8 @@ class Import(BaseImport):
         return "{} | {}".format(self.name or "-", self.importer_type)
 
     def __init__(self, *args, **kwargs):
-        returned = super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._initial_imported_file = self.imported_file.path if self.imported_file else ""
-        return returned
 
     def is_available(self, ishtar_user) -> bool:
         if ishtar_user.is_superuser or self.user == ishtar_user:
@@ -1981,7 +1981,11 @@ class Import(BaseImport):
 
     @property
     def has_error(self) -> bool:
-        return bool(self.error_file)
+        if not self.error_file:
+            return False
+        if self.error_file and not self.error_lines.count():
+            self.parse_error_file()
+        return bool(self.error_lines.filter(ignored=False).count())
 
     @property
     def pre_import_form_is_valid(self) -> bool:
@@ -2686,6 +2690,17 @@ class Import(BaseImport):
     def get_all_updated(self):
         return self._get_all_related("import_updated_")
 
+    def parse_error_file(self):
+        if not self.error_file or not self.error_file.path:
+            ImportLineError.objects.filter(import_item=self).delete()
+            return
+        with open(self.error_file.path, "r") as error_file:
+            reader = csv.reader(error_file)
+            for idx, line in enumerate(reader):
+                if not idx:  # header
+                    continue
+                ImportLineError.objects.get_or_create(import_item=self, line=idx)
+
     def save(self, *args, **kwargs):
         if self.imported_file:
             if self._initial_imported_file != self.imported_file.path or not self.imported_values:
@@ -2693,6 +2708,8 @@ class Import(BaseImport):
         elif self.imported_values:
             self.imported_values = None
         super().save(*args, **kwargs)
+        if not getattr(self, "_no_parse_error_file", False):
+            self.parse_error_file()
 
 
 def pre_delete_import(sender, **kwargs):
@@ -2723,6 +2740,39 @@ def pre_delete_import(sender, **kwargs):
 pre_delete.connect(pre_delete_import, sender=Import)
 
 
+class ImportLineError(models.Model):
+    import_item = models.ForeignKey(Import, on_delete=models.CASCADE, related_name="error_lines")
+    line = models.PositiveIntegerField(_("Line"))
+    ignored = models.BooleanField(_("Ignored"), default=False)
+
+    class Meta:
+        verbose_name = _("Import - Ignored error")
+        verbose_name_plural = _("Import - Ignored error")
+        unique_together = ("line", "import_item")
+        ordering = ("import_item", "line")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._initial_ignored = self.ignored
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        # reevaluate error status of the import
+        if self._initial_ignored == self.ignored:
+            return
+        has_non_ignored_errors = self.import_item.error_lines.filter(ignored=False).count()
+        modified = False
+        if has_non_ignored_errors and self.import_item.state == "F":
+            self.import_item.state = "FE"
+            modified = True
+        elif not has_non_ignored_errors and self.import_item.state == "FE":
+            self.import_item.state = "F"
+            modified = True
+        if modified:
+            self.import_item._no_parse_error_file = True
+            self.import_item.save()
+
+
 class ImportColumnValue(models.Model):
     """
     Value in a column for pre-import columns
diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js
index 9797c7d4a..10ca7bd6c 100644
--- a/ishtar_common/static/js/ishtar.js
+++ b/ishtar_common/static/js/ishtar.js
@@ -124,8 +124,9 @@ var activate_all_search_url = '/activate-all-search/';
 var activate_own_search_url = '/activate-own-search/';
 var activate_advanced_url = '/activate-advanced-menu/';
 var activate_simple_url = '/activate-simple-menu/';
-var shortcut_menu_hide_url = '/hide-shortcut-menu/'
-var shortcut_menu_show_url = '/show-shortcut-menu/'
+var shortcut_menu_hide_url = '/hide-shortcut-menu/';
+var shortcut_menu_show_url = '/show-shortcut-menu/';
+var ignore_error_line_url = "/import-ignore-line/";
 
 function init_shortcut_menu(html){
     close_wait();
@@ -2163,5 +2164,23 @@ var import_table_refresh_import_list = function(data){
                 item["current_line"] + "/" + item["number_of_line"]
             );
         }
+        if (item["has_error"]){
+            $("#import-" + item["full_id"]).addClass("import-row-error");
+        } else {
+            $("#import-" + item["full_id"]).removeClass("import-row-error");
+        }
     }
 }
+
+var import_csv_check_ignored = function(){
+    let tr = $(this).parent().parent();
+    if (this.checked){
+        tr.addClass("import-line-ignored");
+    } else {
+        tr.removeClass("import-line-ignored");
+    }
+    $.get(url_path + ignore_error_line_url + this.value + "/", function(){
+         let import_ids = [tr.parent().parent().attr("data-import")];
+         import_table_update_import_list(import_ids);
+    });
+}
diff --git a/ishtar_common/templates/ishtar/blocks/view_import_csv.html b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
index f1b089ce2..ba9f43d50 100644
--- a/ishtar_common/templates/ishtar/blocks/view_import_csv.html
+++ b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
@@ -1,4 +1,4 @@
-{% load i18n  %}
+{% load i18n l10n %}
 
 <div class="modal-dialog full modal-lg">
     <div class="modal-content">
@@ -12,21 +12,27 @@
             </button>
         </div>
         <div class="table-scroll">
-           <table class="table table-striped table-bordered">
+           <table class="table table-striped table-bordered" data-import="{{import_id}}">
               <thead>
               <tr>
                   <th>&nbsp;</th>
+                  {% if has_line_errors %}<th>{% trans "Ignore" %}</th>{% endif %}
                   {% for head in header %}<th>{{head}}</th>{% endfor %}
               </tr>
               </thead>
               <tbody>
               {% for line in content %}
-              <tr>
+              <tr{% if has_line_errors and line.0.1 %} class="import-line-ignored"{% endif%}>
                   <td>{{ forloop.counter }}</td>
-                  {% for cell in line %}<td>{{cell}}</td>{% endfor %}
+                  {% for cell in line %}<td>
+                      {% if has_line_errors and not forloop.counter0 %}<input class="check-ignored" value="{{cell.0}}"{% if cell.1%} checked="checked"{% endif %} type="checkbox" />{% else %}{{cell}}{% endif %}
+                  </td>{% endfor %}
               </tr>{% endfor %}
               </tbody>
             </table>
         </div>
     </div>
 </div>
+<script type="text/javascript">{% localize off %}
+$(".check-ignored").click(import_csv_check_ignored);
+{% endlocalize %}</script>
\ No newline at end of file
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index 9a5ee4f1c..52e1324b2 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -284,6 +284,11 @@ urlpatterns = [
         check_rights(["change_import"])(views.ImportPreFormView.as_view()),
         name="import_pre_import_form",
     ),
+    url(
+        r"^import-ignore-line/(?P<line_id>[0-9]+)/$",
+        views.line_error,
+        name="import_ignore_line",
+    ),
     url(r"^profile(?:/(?P<pk>[0-9]+))?/$", views.ProfileEdit.as_view(), name="profile"),
     url(
         r"^save-search/(?P<app_label>[a-z-]+)/(?P<model>[a-z-]+)/$",
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 8f8ecb5a6..b91f3202c 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -2075,6 +2075,7 @@ def import_get_status(request, current_right=None):
             "full_id": ("group-" if key == "group" else "") + str(item.id),
             "state": item.state,
             "status": str(item.status),
+            "has_error": item.has_error
         }
         if key == "import":
             item_dct.update({
@@ -2182,6 +2183,16 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView):
             encoding = self.import_item.encoding
         data["icon"], data["target"] = self.TITLES[self.kwargs["target"]]
         data["title"] = str(self.import_item)
+        line_errors = []
+        has_line_errors = False
+        if self.kwargs["target"] == "error":
+            has_line_errors = True
+            q = models.ImportLineError.objects.filter(import_item=self.import_item)
+            if not q.count():
+                self.import_item.parse_error_file()
+            line_errors = models.ImportLineError.objects.filter(
+                import_item=self.import_item).values_list("pk", "ignored")
+        data["has_line_errors"] = has_line_errors
         data["content"] = []
         with open(self.csv_file.path, "r", encoding=encoding) as f:
             reader = csv.reader(f)
@@ -2189,11 +2200,33 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView):
                 if not idx:
                     data["header"] = line
                     continue
+                if line_errors and len(line_errors) >= idx:
+                    line = [line_errors[idx - 1]] + line
                 data["content"].append(line)
         data["window_id"] = "csv-view-" + (self.kwargs.get("group", "") or "") + str(self.import_item.pk)
+        data["import_id"] = self.import_item.pk
         return data
 
 
+def line_error(request, line_id):
+    """
+    Set or unset ignored state of a csv error file
+    """
+    user = request.user
+    if not user.pk:
+        raise Http404()
+    q = models.ImportLineError.objects.filter(pk=line_id)
+    if not q.count():
+        return
+    line = q.all()[0]
+    q = models.Import.query_can_access(request.user).filter(pk=line.import_item_id)
+    if not q.count():
+        raise Http404()
+    line.ignored = not line.ignored
+    line.save()
+    return HttpResponse(content_type="text/plain")
+
+
 class PersonCreate(LoginRequiredMixin, CreateView):
     model = models.Person
     form_class = forms.BasePersonForm
diff --git a/scss/custom.scss b/scss/custom.scss
index 3b6d803b3..749d2becf 100644
--- a/scss/custom.scss
+++ b/scss/custom.scss
@@ -304,6 +304,10 @@ pre {
     background-color: lighten(red, 40%);
 }
 
+.import-line-ignored {
+    color: $gray-500;
+}
+
 
 #import-container {
     overflow: scroll;