diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 56 |
1 files changed, 35 insertions, 21 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index c4805eed6..df6b52c7c 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -44,8 +44,6 @@ from ishtar_common.utils import get_all_field_names, update_data, get_current_pr NEW_LINE_BREAK = "#####@@@#####" -RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)") - def post_importer_action(func): def wrapper(self, context, value): @@ -748,6 +746,7 @@ class Importer(object): MODEL_CREATION_LIMIT = [] DEFAULTS = {} PRE_IMPORT_VALUES = {} # values from a form before the import + IGNORE_ERRORS = tuple() ERRORS = { "header_check": _( "The given file is not correct. Check the file " @@ -860,8 +859,10 @@ class Importer(object): self._add_to_post_save(rel.__class__, rel.pk, idx_line) except IntegrityError as e: self.errors.append((idx_line, None, str(e))) - except ImporterError as msg: - self.errors.append((idx_line, None, msg)) + except ImporterError as import_error: + msg = str(import_error) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((idx_line, None, msg)) return item def post_import(self): @@ -1083,8 +1084,10 @@ class Importer(object): sys.stdout.flush() try: results.append(self._line_processing(idx_line, line)) - except ImporterError as msg: - self.errors.append((idx_line, None, msg)) + except ImporterError as import_error: + msg = str(import_error) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((idx_line, None, msg)) self.post_import() for item in self.to_be_close: item.close() @@ -1277,8 +1280,10 @@ class Importer(object): else: item = GeoVectorData.objects.create(**geodata) created = True - except Exception as e: - self.errors.append((self.idx_line, None, str(e))) + except Exception as import_error: + msg = str(import_error) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((self.idx_line, None, msg)) return if self.import_instance and created: item.imports.add(self.import_instance) @@ -1402,13 +1407,15 @@ class Importer(object): c_row.append("") return data val = val.replace(NEW_LINE_BREAK, "\n") - self.errors.append( - ( - idx_line + 1, - idx_col + 1, - str(self.ERRORS["regex_not_match"]) + val, + msg = str(self.ERRORS["regex_not_match"]) + val + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append( + ( + idx_line + 1, + idx_col + 1, + msg, + ) ) - ) c_row.append("") return data val_group = [] @@ -1475,10 +1482,12 @@ class Importer(object): value = func.format(v, archive=self.archive) else: value = func.format(v) - except ValueError as e: + except ValueError as import_error: if formater.required: self.c_errors = True - self.errors.append((idx_line + 1, idx_col + 1, str(e))) + msg = str(import_error) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((idx_line + 1, idx_col + 1, msg)) c_values.append("") return data if formater.value_format and value is not None and value != "": @@ -1776,7 +1785,9 @@ class Importer(object): idx_line=idx_line, ) except Exception as e: - self.errors.append((self.idx_line, None, str(e))) + msg = str(e) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((self.idx_line, None, msg)) return if not hasattr(field_object, "related_model") or not field_object.related_model: return @@ -1804,8 +1815,10 @@ class Importer(object): field_object.remote_field.model, data[attribute].copy(), c_path, idx_line=idx_line ) - except ImporterError as msg: - self.errors.append((self.idx_line, None, msg)) + except ImporterError as import_error: + msg = str(import_error) + if not any(1 for error in self.IGNORE_ERRORS if error in msg): + self.errors.append((self.idx_line, None, msg)) data[attribute] = None def get_object(self, cls, data, path=None, idx_line=None): @@ -2111,8 +2124,9 @@ class Importer(object): v._timestamp = self.timestamp v._queue = "low_priority" v.save() - except DatabaseError as e: - raise IntegrityError(e.message) + except DatabaseError as import_error: + msg = str(import_error) + raise IntegrityError(msg) if self.simulate: # put m2m result in data dict current_data = data |