✨ imports: allow errors to be pointed out line by line

1 files changed, 54 insertions, 4 deletions

diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index a240f4326..77dad558e 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -47,6 +47,7 @@ from django.core.exceptions import ValidationError, SuspiciousOperation
 from django.core.files import File
 from django.core.files.base import ContentFile
 from django.core.validators import validate_comma_separated_integer_list, MinValueValidator
+from django.db.models import Q
 from django.db.models.base import ModelBase
 from django.db.models.signals import pre_delete
 from django.template.defaultfilters import slugify
@@ -1423,7 +1424,7 @@ class BaseImport(models.Model, OwnPerms, SheetItem):
             return q
         IshtarUser = apps.get_model("ishtar_common", "IshtarUser")
         ishtar_user = IshtarUser.objects.get(pk=user.pk)
-        q = q.filter(user=ishtar_user)
+        q = q.filter(Q(user=ishtar_user) | Q(importer_type__users__pk=ishtar_user.pk))
         return q
 
     @classmethod
@@ -1924,9 +1925,8 @@ class Import(BaseImport):
         return "{} | {}".format(self.name or "-", self.importer_type)
 
     def __init__(self, *args, **kwargs):
-        returned = super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._initial_imported_file = self.imported_file.path if self.imported_file else ""
-        return returned
 
     def is_available(self, ishtar_user) -> bool:
         if ishtar_user.is_superuser or self.user == ishtar_user:
@@ -1981,7 +1981,11 @@ class Import(BaseImport):
 
     @property
     def has_error(self) -> bool:
-        return bool(self.error_file)
+        if not self.error_file:
+            return False
+        if self.error_file and not self.error_lines.count():
+            self.parse_error_file()
+        return bool(self.error_lines.filter(ignored=False).count())
 
     @property
     def pre_import_form_is_valid(self) -> bool:
@@ -2686,6 +2690,17 @@ class Import(BaseImport):
     def get_all_updated(self):
         return self._get_all_related("import_updated_")
 
+    def parse_error_file(self):
+        if not self.error_file or not self.error_file.path:
+            ImportLineError.objects.filter(import_item=self).delete()
+            return
+        with open(self.error_file.path, "r") as error_file:
+            reader = csv.reader(error_file)
+            for idx, line in enumerate(reader):
+                if not idx:  # header
+                    continue
+                ImportLineError.objects.get_or_create(import_item=self, line=idx)
+
     def save(self, *args, **kwargs):
         if self.imported_file:
             if self._initial_imported_file != self.imported_file.path or not self.imported_values:
@@ -2693,6 +2708,8 @@ class Import(BaseImport):
         elif self.imported_values:
             self.imported_values = None
         super().save(*args, **kwargs)
+        if not getattr(self, "_no_parse_error_file", False):
+            self.parse_error_file()
 
 
 def pre_delete_import(sender, **kwargs):
@@ -2723,6 +2740,39 @@ def pre_delete_import(sender, **kwargs):
 pre_delete.connect(pre_delete_import, sender=Import)
 
 
+class ImportLineError(models.Model):
+    import_item = models.ForeignKey(Import, on_delete=models.CASCADE, related_name="error_lines")
+    line = models.PositiveIntegerField(_("Line"))
+    ignored = models.BooleanField(_("Ignored"), default=False)
+
+    class Meta:
+        verbose_name = _("Import - Ignored error")
+        verbose_name_plural = _("Import - Ignored error")
+        unique_together = ("line", "import_item")
+        ordering = ("import_item", "line")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._initial_ignored = self.ignored
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        # reevaluate error status of the import
+        if self._initial_ignored == self.ignored:
+            return
+        has_non_ignored_errors = self.import_item.error_lines.filter(ignored=False).count()
+        modified = False
+        if has_non_ignored_errors and self.import_item.state == "F":
+            self.import_item.state = "FE"
+            modified = True
+        elif not has_non_ignored_errors and self.import_item.state == "FE":
+            self.import_item.state = "F"
+            modified = True
+        if modified:
+            self.import_item._no_parse_error_file = True
+            self.import_item.save()
+
+
 class ImportColumnValue(models.Model):
     """
     Value in a column for pre-import columns