summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--archaeological_finds/tests.py6
-rw-r--r--changelog/en/changelog_2022-06-15.md1
-rw-r--r--changelog/fr/changelog_2023-01-25.md1
-rw-r--r--ishtar_common/data_importer.py56
-rw-r--r--ishtar_common/models_imports.py4
-rw-r--r--ishtar_common/views.py6
6 files changed, 50 insertions, 24 deletions
diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 312d1385f..202452ef8 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -985,6 +985,12 @@ class ImportFindTest(ImportTest, FindInit, TestCase):
).format("Area", "43", "Warehouse test")
for error in impt.errors:
self.assertEqual(error["error"], error_msg)
+ # test ignore errors
+ impt.importer_type.ignore_errors = error_msg[:-5]
+ impt.importer_type.save()
+ impt.importation()
+ # check errors
+ self.assertEqual(len(impt.errors), 0)
def tearDown(self):
self.tmpdir.cleanup()
diff --git a/changelog/en/changelog_2022-06-15.md b/changelog/en/changelog_2022-06-15.md
index 72013ebe4..35a3d60ce 100644
--- a/changelog/en/changelog_2022-06-15.md
+++ b/changelog/en/changelog_2022-06-15.md
@@ -10,6 +10,7 @@ v4.0.XX - 2099-12-31
- reorganization of fields
- improved presentation
- import sheet (links from other sheet to the import sheet)
+- import: allow to exclude some errors messages
### Technical ###
- update relationship between imports and main items
diff --git a/changelog/fr/changelog_2023-01-25.md b/changelog/fr/changelog_2023-01-25.md
index 171c09bc6..f60f7331b 100644
--- a/changelog/fr/changelog_2023-01-25.md
+++ b/changelog/fr/changelog_2023-01-25.md
@@ -10,6 +10,7 @@ v4.0.XX - 2099-12-31
- réorganisation des champs
- amélioration de la présentation
- fiche d'import (liens des autres fiches vers la fiche import)
+- import: possibilité d'ignorer certains messages d'erreur
### Technique ###
- relation de mise à jour entre imports et les éléments principaux
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c4805eed6..df6b52c7c 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -44,8 +44,6 @@ from ishtar_common.utils import get_all_field_names, update_data, get_current_pr
NEW_LINE_BREAK = "#####@@@#####"
-RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
-
def post_importer_action(func):
def wrapper(self, context, value):
@@ -748,6 +746,7 @@ class Importer(object):
MODEL_CREATION_LIMIT = []
DEFAULTS = {}
PRE_IMPORT_VALUES = {} # values from a form before the import
+ IGNORE_ERRORS = tuple()
ERRORS = {
"header_check": _(
"The given file is not correct. Check the file "
@@ -860,8 +859,10 @@ class Importer(object):
self._add_to_post_save(rel.__class__, rel.pk, idx_line)
except IntegrityError as e:
self.errors.append((idx_line, None, str(e)))
- except ImporterError as msg:
- self.errors.append((idx_line, None, msg))
+ except ImporterError as import_error:
+ msg = str(import_error)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((idx_line, None, msg))
return item
def post_import(self):
@@ -1083,8 +1084,10 @@ class Importer(object):
sys.stdout.flush()
try:
results.append(self._line_processing(idx_line, line))
- except ImporterError as msg:
- self.errors.append((idx_line, None, msg))
+ except ImporterError as import_error:
+ msg = str(import_error)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((idx_line, None, msg))
self.post_import()
for item in self.to_be_close:
item.close()
@@ -1277,8 +1280,10 @@ class Importer(object):
else:
item = GeoVectorData.objects.create(**geodata)
created = True
- except Exception as e:
- self.errors.append((self.idx_line, None, str(e)))
+ except Exception as import_error:
+ msg = str(import_error)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((self.idx_line, None, msg))
return
if self.import_instance and created:
item.imports.add(self.import_instance)
@@ -1402,13 +1407,15 @@ class Importer(object):
c_row.append("")
return data
val = val.replace(NEW_LINE_BREAK, "\n")
- self.errors.append(
- (
- idx_line + 1,
- idx_col + 1,
- str(self.ERRORS["regex_not_match"]) + val,
+ msg = str(self.ERRORS["regex_not_match"]) + val
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append(
+ (
+ idx_line + 1,
+ idx_col + 1,
+ msg,
+ )
)
- )
c_row.append("")
return data
val_group = []
@@ -1475,10 +1482,12 @@ class Importer(object):
value = func.format(v, archive=self.archive)
else:
value = func.format(v)
- except ValueError as e:
+ except ValueError as import_error:
if formater.required:
self.c_errors = True
- self.errors.append((idx_line + 1, idx_col + 1, str(e)))
+ msg = str(import_error)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((idx_line + 1, idx_col + 1, msg))
c_values.append("")
return data
if formater.value_format and value is not None and value != "":
@@ -1776,7 +1785,9 @@ class Importer(object):
idx_line=idx_line,
)
except Exception as e:
- self.errors.append((self.idx_line, None, str(e)))
+ msg = str(e)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((self.idx_line, None, msg))
return
if not hasattr(field_object, "related_model") or not field_object.related_model:
return
@@ -1804,8 +1815,10 @@ class Importer(object):
field_object.remote_field.model, data[attribute].copy(), c_path,
idx_line=idx_line
)
- except ImporterError as msg:
- self.errors.append((self.idx_line, None, msg))
+ except ImporterError as import_error:
+ msg = str(import_error)
+ if not any(1 for error in self.IGNORE_ERRORS if error in msg):
+ self.errors.append((self.idx_line, None, msg))
data[attribute] = None
def get_object(self, cls, data, path=None, idx_line=None):
@@ -2111,8 +2124,9 @@ class Importer(object):
v._timestamp = self.timestamp
v._queue = "low_priority"
v.save()
- except DatabaseError as e:
- raise IntegrityError(e.message)
+ except DatabaseError as import_error:
+ msg = str(import_error)
+ raise IntegrityError(msg)
if self.simulate:
# put m2m result in data dict
current_data = data
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 6c60051fc..3f687b93d 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -313,6 +313,9 @@ class ImporterType(models.Model):
defaults = import_instance.default_values
else:
defaults = self.get_default_values()
+ ignore_errors = tuple()
+ if self.ignore_errors:
+ ignore_errors = tuple([error.strip() for error in self.ignore_errors.split("\n")])
LINE_FORMAT = []
LINE_EXPORT_FORMAT = []
idx = 0
@@ -377,6 +380,7 @@ class ImporterType(models.Model):
"DESC": self.description,
"DEFAULTS": defaults,
"PRE_IMPORT_VALUES": pre_import_values,
+ "IGNORE_ERRORS": ignore_errors,
"LINE_FORMAT": LINE_FORMAT,
"UNICITY_KEYS": UNICITY_KEYS,
"LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT,
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 15e0158da..aec8fa9f4 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1558,10 +1558,10 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
if not user.pk:
raise Http404()
q1 = self._queryset_filter(self.model.query_can_access(user))
- q1 = q1.filter(group__isnull=True).order_by("-creation_date", "-pk")
+ q1 = q1.filter(group__isnull=True).order_by("-end_date", "-creation_date", "-pk")
q2 = self._queryset_filter(models.ImportGroup.query_can_access(user))
- q2 = q2.order_by("-creation_date", "-pk")
- return list(reversed(sorted(list(q1) + list(q2), key=lambda x: x.creation_date)))
+ q2 = q2.order_by("-end_date", "-creation_date", "-pk")
+ return list(reversed(sorted(list(q1) + list(q2), key=lambda x: (x.end_date or x.creation_date))))
def post(self, request, *args, **kwargs):
for field in request.POST:
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-16 17:07:40 +0000