✨ imports: manage ignore_errors

6 files changed, 50 insertions, 24 deletions

diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 312d1385f..202452ef8 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -985,6 +985,12 @@ class ImportFindTest(ImportTest, FindInit, TestCase):
         ).format("Area", "43", "Warehouse test")
         for error in impt.errors:
             self.assertEqual(error["error"], error_msg)
+        # test ignore errors
+        impt.importer_type.ignore_errors = error_msg[:-5]
+        impt.importer_type.save()
+        impt.importation()
+        # check errors
+        self.assertEqual(len(impt.errors), 0)
 
     def tearDown(self):
         self.tmpdir.cleanup()
diff --git a/changelog/en/changelog_2022-06-15.md b/changelog/en/changelog_2022-06-15.md
index 72013ebe4..35a3d60ce 100644
--- a/changelog/en/changelog_2022-06-15.md
+++ b/changelog/en/changelog_2022-06-15.md
@@ -10,6 +10,7 @@ v4.0.XX - 2099-12-31
     - reorganization of fields
     - improved presentation
 - import sheet (links from other sheet to the import sheet)
+- import: allow to exclude some errors messages
 
 ### Technical ###
 - update relationship between imports and main items
diff --git a/changelog/fr/changelog_2023-01-25.md b/changelog/fr/changelog_2023-01-25.md
index 171c09bc6..f60f7331b 100644
--- a/changelog/fr/changelog_2023-01-25.md
+++ b/changelog/fr/changelog_2023-01-25.md
@@ -10,6 +10,7 @@ v4.0.XX - 2099-12-31
     - réorganisation des champs
     - amélioration de la présentation
 - fiche d'import (liens des autres fiches vers la fiche import)
+- import: possibilité d'ignorer certains messages d'erreur
 
 ### Technique ###
 - relation de mise à jour entre imports et les éléments principaux
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c4805eed6..df6b52c7c 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -44,8 +44,6 @@ from ishtar_common.utils import get_all_field_names, update_data, get_current_pr
 
 NEW_LINE_BREAK = "#####@@@#####"
 
-RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
-
 
 def post_importer_action(func):
     def wrapper(self, context, value):
@@ -748,6 +746,7 @@ class Importer(object):
     MODEL_CREATION_LIMIT = []
     DEFAULTS = {}
     PRE_IMPORT_VALUES = {}  # values from a form before the import
+    IGNORE_ERRORS = tuple()
     ERRORS = {
         "header_check": _(
             "The given file is not correct. Check the file "
@@ -860,8 +859,10 @@ class Importer(object):
                         self._add_to_post_save(rel.__class__, rel.pk, idx_line)
             except IntegrityError as e:
                 self.errors.append((idx_line, None, str(e)))
-            except ImporterError as msg:
-                self.errors.append((idx_line, None, msg))
+            except ImporterError as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((idx_line, None, msg))
         return item
 
     def post_import(self):
@@ -1083,8 +1084,10 @@ class Importer(object):
                 sys.stdout.flush()
             try:
                 results.append(self._line_processing(idx_line, line))
-            except ImporterError as msg:
-                self.errors.append((idx_line, None, msg))
+            except ImporterError as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((idx_line, None, msg))
         self.post_import()
         for item in self.to_be_close:
             item.close()
@@ -1277,8 +1280,10 @@ class Importer(object):
                 else:
                     item = GeoVectorData.objects.create(**geodata)
                     created = True
-            except Exception as e:
-                self.errors.append((self.idx_line, None, str(e)))
+            except Exception as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((self.idx_line, None, msg))
                 return
             if self.import_instance and created:
                 item.imports.add(self.import_instance)
@@ -1402,13 +1407,15 @@ class Importer(object):
                     c_row.append("")
                     return data
                 val = val.replace(NEW_LINE_BREAK, "\n")
-                self.errors.append(
-                    (
-                        idx_line + 1,
-                        idx_col + 1,
-                        str(self.ERRORS["regex_not_match"]) + val,
+                msg = str(self.ERRORS["regex_not_match"]) + val
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append(
+                        (
+                            idx_line + 1,
+                            idx_col + 1,
+                            msg,
+                        )
                     )
-                )
                 c_row.append("")
                 return data
             val_group = []
@@ -1475,10 +1482,12 @@ class Importer(object):
                         value = func.format(v, archive=self.archive)
                     else:
                         value = func.format(v)
-                except ValueError as e:
+                except ValueError as import_error:
                     if formater.required:
                         self.c_errors = True
-                    self.errors.append((idx_line + 1, idx_col + 1, str(e)))
+                    msg = str(import_error)
+                    if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                        self.errors.append((idx_line + 1, idx_col + 1, msg))
                     c_values.append("")
                     return data
                 if formater.value_format and value is not None and value != "":
@@ -1776,7 +1785,9 @@ class Importer(object):
                     idx_line=idx_line,
                 )
             except Exception as e:
-                self.errors.append((self.idx_line, None, str(e)))
+                msg = str(e)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((self.idx_line, None, msg))
             return
         if not hasattr(field_object, "related_model") or not field_object.related_model:
             return
@@ -1804,8 +1815,10 @@ class Importer(object):
                 field_object.remote_field.model, data[attribute].copy(), c_path,
                 idx_line=idx_line
             )
-        except ImporterError as msg:
-            self.errors.append((self.idx_line, None, msg))
+        except ImporterError as import_error:
+            msg = str(import_error)
+            if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                self.errors.append((self.idx_line, None, msg))
             data[attribute] = None
 
     def get_object(self, cls, data, path=None, idx_line=None):
@@ -2111,8 +2124,9 @@ class Importer(object):
                             v._timestamp = self.timestamp
                             v._queue = "low_priority"
                             v.save()
-                        except DatabaseError as e:
-                            raise IntegrityError(e.message)
+                        except DatabaseError as import_error:
+                            msg = str(import_error)
+                            raise IntegrityError(msg)
             if self.simulate:
                 # put m2m result in data dict
                 current_data = data
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 6c60051fc..3f687b93d 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -313,6 +313,9 @@ class ImporterType(models.Model):
             defaults = import_instance.default_values
         else:
             defaults = self.get_default_values()
+        ignore_errors = tuple()
+        if self.ignore_errors:
+            ignore_errors = tuple([error.strip() for error in self.ignore_errors.split("\n")])
         LINE_FORMAT = []
         LINE_EXPORT_FORMAT = []
         idx = 0
@@ -377,6 +380,7 @@ class ImporterType(models.Model):
             "DESC": self.description,
             "DEFAULTS": defaults,
             "PRE_IMPORT_VALUES": pre_import_values,
+            "IGNORE_ERRORS": ignore_errors,
             "LINE_FORMAT": LINE_FORMAT,
             "UNICITY_KEYS": UNICITY_KEYS,
             "LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT,
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 15e0158da..aec8fa9f4 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1558,10 +1558,10 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
         if not user.pk:
             raise Http404()
         q1 = self._queryset_filter(self.model.query_can_access(user))
-        q1 = q1.filter(group__isnull=True).order_by("-creation_date", "-pk")
+        q1 = q1.filter(group__isnull=True).order_by("-end_date", "-creation_date", "-pk")
         q2 = self._queryset_filter(models.ImportGroup.query_can_access(user))
-        q2 = q2.order_by("-creation_date", "-pk")
-        return list(reversed(sorted(list(q1) + list(q2), key=lambda x: x.creation_date)))
+        q2 = q2.order_by("-end_date", "-creation_date", "-pk")
+        return list(reversed(sorted(list(q1) + list(q2), key=lambda x: (x.end_date or x.creation_date))))
 
     def post(self, request, *args, **kwargs):
         for field in request.POST: