✨ imports: manage ignore_errors

1 files changed, 35 insertions, 21 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c4805eed6..df6b52c7c 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -44,8 +44,6 @@ from ishtar_common.utils import get_all_field_names, update_data, get_current_pr
 
 NEW_LINE_BREAK = "#####@@@#####"
 
-RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
-
 
 def post_importer_action(func):
     def wrapper(self, context, value):
@@ -748,6 +746,7 @@ class Importer(object):
     MODEL_CREATION_LIMIT = []
     DEFAULTS = {}
     PRE_IMPORT_VALUES = {}  # values from a form before the import
+    IGNORE_ERRORS = tuple()
     ERRORS = {
         "header_check": _(
             "The given file is not correct. Check the file "
@@ -860,8 +859,10 @@ class Importer(object):
                         self._add_to_post_save(rel.__class__, rel.pk, idx_line)
             except IntegrityError as e:
                 self.errors.append((idx_line, None, str(e)))
-            except ImporterError as msg:
-                self.errors.append((idx_line, None, msg))
+            except ImporterError as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((idx_line, None, msg))
         return item
 
     def post_import(self):
@@ -1083,8 +1084,10 @@ class Importer(object):
                 sys.stdout.flush()
             try:
                 results.append(self._line_processing(idx_line, line))
-            except ImporterError as msg:
-                self.errors.append((idx_line, None, msg))
+            except ImporterError as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((idx_line, None, msg))
         self.post_import()
         for item in self.to_be_close:
             item.close()
@@ -1277,8 +1280,10 @@ class Importer(object):
                 else:
                     item = GeoVectorData.objects.create(**geodata)
                     created = True
-            except Exception as e:
-                self.errors.append((self.idx_line, None, str(e)))
+            except Exception as import_error:
+                msg = str(import_error)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((self.idx_line, None, msg))
                 return
             if self.import_instance and created:
                 item.imports.add(self.import_instance)
@@ -1402,13 +1407,15 @@ class Importer(object):
                     c_row.append("")
                     return data
                 val = val.replace(NEW_LINE_BREAK, "\n")
-                self.errors.append(
-                    (
-                        idx_line + 1,
-                        idx_col + 1,
-                        str(self.ERRORS["regex_not_match"]) + val,
+                msg = str(self.ERRORS["regex_not_match"]) + val
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append(
+                        (
+                            idx_line + 1,
+                            idx_col + 1,
+                            msg,
+                        )
                     )
-                )
                 c_row.append("")
                 return data
             val_group = []
@@ -1475,10 +1482,12 @@ class Importer(object):
                         value = func.format(v, archive=self.archive)
                     else:
                         value = func.format(v)
-                except ValueError as e:
+                except ValueError as import_error:
                     if formater.required:
                         self.c_errors = True
-                    self.errors.append((idx_line + 1, idx_col + 1, str(e)))
+                    msg = str(import_error)
+                    if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                        self.errors.append((idx_line + 1, idx_col + 1, msg))
                     c_values.append("")
                     return data
                 if formater.value_format and value is not None and value != "":
@@ -1776,7 +1785,9 @@ class Importer(object):
                     idx_line=idx_line,
                 )
             except Exception as e:
-                self.errors.append((self.idx_line, None, str(e)))
+                msg = str(e)
+                if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                    self.errors.append((self.idx_line, None, msg))
             return
         if not hasattr(field_object, "related_model") or not field_object.related_model:
             return
@@ -1804,8 +1815,10 @@ class Importer(object):
                 field_object.remote_field.model, data[attribute].copy(), c_path,
                 idx_line=idx_line
             )
-        except ImporterError as msg:
-            self.errors.append((self.idx_line, None, msg))
+        except ImporterError as import_error:
+            msg = str(import_error)
+            if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+                self.errors.append((self.idx_line, None, msg))
             data[attribute] = None
 
     def get_object(self, cls, data, path=None, idx_line=None):
@@ -2111,8 +2124,9 @@ class Importer(object):
                             v._timestamp = self.timestamp
                             v._queue = "low_priority"
                             v.save()
-                        except DatabaseError as e:
-                            raise IntegrityError(e.message)
+                        except DatabaseError as import_error:
+                            msg = str(import_error)
+                            raise IntegrityError(msg)
             if self.simulate:
                 # put m2m result in data dict
                 current_data = data