diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 318 |
1 files changed, 172 insertions, 146 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index a03f4de34..426d32a7a 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -29,9 +29,10 @@ import zipfile from django.conf import settings from django.contrib.auth.models import User -from django.core.exceptions import ImproperlyConfigured +from django.db.models.fields import FieldDoesNotExist from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction +from django.db.models import Q from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ @@ -148,13 +149,15 @@ class ChoiceChecker(object): class UnicodeFormater(Formater): def __init__(self, max_length=None, clean=False, re_filter=None, - notnull=False, prefix=u'', db_target=None): + notnull=False, prefix=u'', db_target=None, + import_instance=None): self.max_length = max_length self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull self.prefix = prefix + self.import_instance = import_instance def format(self, value): try: @@ -255,7 +258,8 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater, ChoiceChecker): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split='', db_target=None): + cli=False, many_split='', db_target=None, + import_instance=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) @@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.new_keys = {} self.match_table = {} self.many_split = many_split + self.import_instance = None for key, value in self.choices: value = unicode(value) if not self.strict: @@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker): def init_db_target(self): if not self.db_target: return - for target_key in self.db_target.keys.filter(is_set=True).all(): + q = self.db_target.keys.filter(is_set=True) + if self.import_instance: + q = q.filter(Q(associated_import=self.import_instance) | + Q(associated_import__isnull=True)) + for target_key in q.all(): key = target_key.key if not self.strict: key = slugify(key) @@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): class TypeFormater(StrChoiceFormater): def __init__(self, model, cli=False, defaults={}, many_split=False, - db_target=None): + db_target=None, import_instance=None): self.create = True self.strict = False self.model = model @@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater): self.equiv_dict, self.choices = {}, [] self.match_table = {} self.new_keys = {} + self.import_instance = import_instance for item in model.objects.all(): self.choices.append((item.pk, unicode(item))) - for key in item.get_keys(): + for key in item.get_keys(importer_id=import_instance.pk): self.equiv_dict[key] = item def prepare(self, value): @@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater): class DateFormater(Formater): - def __init__(self, date_formats=["%d/%m/%Y"], db_target=None): + def __init__(self, date_formats=["%d/%m/%Y"], db_target=None, + import_instance=None): self.date_formats = date_formats if type(date_formats) not in (list, tuple): self.date_formats = [self.date_formats] self.db_target = db_target + self.import_instance = import_instance def format(self, value): value = value.strip() @@ -511,7 +523,8 @@ class FileFormater(Formater): class StrToBoolean(Formater, ChoiceChecker): - def __init__(self, choices={}, cli=False, strict=False, db_target=None): + def __init__(self, choices={}, cli=False, strict=False, db_target=None, + import_instance=None): self.dct = copy.copy(choices) self.cli = cli self.strict = strict @@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.init_db_target() self.match_table = {} self.new_keys = {} + self.import_instance = import_instance def init_db_target(self): if not self.db_target: @@ -733,7 +747,7 @@ class Importer(object): q = ImporterModel.objects.filter(klass=cls_name) if q.count(): cls_name = q.all()[0].name - return ImproperlyConfigured( + return ImporterError( unicode(self.ERRORS['improperly_configured']).format(cls_name)) def _get_does_not_exist_in_db_error(self, model, data): @@ -827,7 +841,7 @@ class Importer(object): vals[idx_col].append(val) for idx, formater in enumerate(self.line_format): if formater and idx < len(vals): - + formater.import_instance = self.import_instance if self.DB_TARGETS: field_names = formater.field_name if type(field_names) not in (list, tuple): @@ -1143,6 +1157,7 @@ class Importer(object): self.concat_str[field_name] = concat_str if self.DB_TARGETS: + formater.import_instance = self.import_instance formater.reinit_db_target( self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)], idx_v) @@ -1217,8 +1232,13 @@ class Importer(object): c_row.append(u" ; ".join([v for v in c_values])) def get_field(self, cls, attribute, data, m2ms, c_path, new_created): - field_object, model, direct, m2m = \ - cls._meta.get_field_by_name(attribute) + try: + field_object, model, direct, m2m = \ + cls._meta.get_field_by_name(attribute) + except FieldDoesNotExist: + raise ImporterError(unicode( + _(u"Importer configuration error: field \"{}\" does not exist " + u"for {}.")).format(attribute, cls._meta.verbose_name)) if m2m: many_values = data.pop(attribute) if hasattr(field_object, 'rel'): @@ -1361,146 +1381,152 @@ class Importer(object): def get_object(self, cls, data, path=[]): m2ms = [] - if data and type(data) == dict: - c_path = path[:] - - # get all related fields - new_created = {} - for attribute in list(data.keys()): - c_c_path = c_path[:] - if not attribute: - data.pop(attribute) - continue - if not data[attribute]: - continue - if attribute != '__force_new': - self.get_field(cls, attribute, data, m2ms, c_c_path, - new_created) - - create_dict = copy.deepcopy(data) - for k in create_dict.keys(): - # filter unnecessary default values - if type(create_dict[k]) == dict: - create_dict.pop(k) - # File doesn't like deepcopy - if type(create_dict[k]) == File: - create_dict[k] = copy.copy(data[k]) - - # default values - path = tuple(path) - defaults = {} - if path in self._defaults: - for k in self._defaults[path]: - if (k not in data or not data[k]): - defaults[k] = self._defaults[path][k] - - if 'history_modifier' in create_dict: - defaults.update({ - 'history_modifier': create_dict.pop('history_modifier') - }) - - created = False + if type(data) != dict: + return data, False + is_empty = not bool( + [k for k in data if k not in ('history_modifier', 'defaults') + and data[k]]) + if is_empty: + return None, False + + c_path = path[:] + + # get all related fields + new_created = {} + for attribute in list(data.keys()): + c_c_path = c_path[:] + if not attribute: + data.pop(attribute) + continue + if not data[attribute]: + continue + if attribute != '__force_new': + self.get_field(cls, attribute, data, m2ms, c_c_path, + new_created) + + create_dict = copy.deepcopy(data) + for k in create_dict.keys(): + # filter unnecessary default values + if type(create_dict[k]) == dict: + create_dict.pop(k) + # File doesn't like deepcopy + if type(create_dict[k]) == File: + create_dict[k] = copy.copy(data[k]) + + # default values + path = tuple(path) + defaults = {} + if path in self._defaults: + for k in self._defaults[path]: + if k not in data or not data[k]: + defaults[k] = self._defaults[path][k] + + if 'history_modifier' in create_dict: + defaults.update({ + 'history_modifier': create_dict.pop('history_modifier') + }) + + created = False + try: try: - try: - dct = create_dict.copy() - for key in dct: - if callable(dct[key]): - dct[key] = dct[key]() - if '__force_new' in dct: - created = dct.pop('__force_new') - if not [k for k in dct if dct[k] is not None]: - return None, created - new_dct = defaults.copy() - new_dct.update(dct) - if self.MODEL_CREATION_LIMIT and \ - cls not in self.MODEL_CREATION_LIMIT: - raise self._get_improperly_conf_error(cls) - obj = cls.objects.create(**new_dct) + dct = create_dict.copy() + for key in dct: + if callable(dct[key]): + dct[key] = dct[key]() + if '__force_new' in dct: + created = dct.pop('__force_new') + if not [k for k in dct if dct[k] is not None]: + return None, created + new_dct = defaults.copy() + new_dct.update(dct) + if self.MODEL_CREATION_LIMIT and \ + cls not in self.MODEL_CREATION_LIMIT: + raise self._get_improperly_conf_error(cls) + obj = cls.objects.create(**new_dct) + else: + # manage UNICITY_KEYS - only level 1 + if not path and self.UNICITY_KEYS: + for k in dct.keys(): + if k not in self.UNICITY_KEYS \ + and k != 'defaults': + defaults[k] = dct.pop(k) + if not self.MODEL_CREATION_LIMIT or \ + cls in self.MODEL_CREATION_LIMIT: + dct['defaults'] = defaults.copy() + obj, created = cls.objects.get_or_create(**dct) else: - # manage UNICITY_KEYS - only level 1 - if not path and self.UNICITY_KEYS: - for k in dct.keys(): - if k not in self.UNICITY_KEYS \ - and k != 'defaults': - defaults[k] = dct.pop(k) - if not self.MODEL_CREATION_LIMIT or \ - cls in self.MODEL_CREATION_LIMIT: + try: + obj = cls.objects.get(**dct) dct['defaults'] = defaults.copy() - obj, created = cls.objects.get_or_create(**dct) - else: - try: - obj = cls.objects.get(**dct) - dct['defaults'] = defaults.copy() - except cls.DoesNotExist: - raise self._get_does_not_exist_in_db_error( - cls, dct) - - if not created and not path and self.UNICITY_KEYS: - changed = False - if self.conservative_import: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue - val = getattr(obj, k) - if val is None or val == '': - changed = True - setattr(obj, k, new_val) - elif k in self.concats \ - and type(val) == unicode \ - and type(new_val) == unicode: - setattr(obj, k, val + u"\n" + new_val) - else: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue + except cls.DoesNotExist: + raise self._get_does_not_exist_in_db_error( + cls, dct) + + if not created and not path and self.UNICITY_KEYS: + changed = False + if self.conservative_import: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + val = getattr(obj, k) + if val is None or val == '': changed = True setattr(obj, k, new_val) - if changed: - obj.save() - if self.import_instance and hasattr(obj, 'imports') \ - and created: - obj.imports.add(self.import_instance) - except ValueError as e: - raise IntegrityError(e.message) - except IntegrityError as e: - raise IntegrityError(e.message) - except DatabaseError as e: - raise IntegrityError(e.message) - except cls.MultipleObjectsReturned as e: - created = False - if 'defaults' in dct: - dct.pop('defaults') - raise IntegrityError(e.message) - # obj = cls.objects.filter(**dct).all()[0] - for attr, value in m2ms: - values = [value] - if type(value) in (list, tuple): - values = value - for v in values: - getattr(obj, attr).add(v) - # force post save script - v.save() - if m2ms: - # force post save script - obj.save() + elif k in self.concats \ + and type(val) == unicode \ + and type(new_val) == unicode: + setattr(obj, k, val + u"\n" + new_val) + else: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + changed = True + setattr(obj, k, new_val) + if changed: + obj.save() + if self.import_instance and hasattr(obj, 'imports') \ + and created: + obj.imports.add(self.import_instance) + except ValueError as e: + raise IntegrityError(e.message) except IntegrityError as e: - message = e.message - try: - message = e.message.decode('utf-8') - except (UnicodeDecodeError, UnicodeDecodeError): - message = '' - try: - data = unicode(data) - except UnicodeDecodeError: - data = '' - raise ImporterError( - "Erreur d'import %s %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode("__".join(path)), - unicode(data), message)) - return obj, created - return data + raise IntegrityError(e.message) + except DatabaseError as e: + raise IntegrityError(e.message) + except cls.MultipleObjectsReturned as e: + created = False + if 'defaults' in dct: + dct.pop('defaults') + raise IntegrityError(e.message) + # obj = cls.objects.filter(**dct).all()[0] + for attr, value in m2ms: + values = [value] + if type(value) in (list, tuple): + values = value + for v in values: + getattr(obj, attr).add(v) + # force post save script + v.save() + if m2ms: + # force post save script + obj.save() + except IntegrityError as e: + message = e.message + try: + message = e.message.decode('utf-8') + except (UnicodeDecodeError, UnicodeDecodeError): + message = '' + try: + data = unicode(data) + except UnicodeDecodeError: + data = '' + raise ImporterError( + "Erreur d'import %s %s, contexte : %s, erreur : %s" + % (unicode(cls), unicode("__".join(path)), + unicode(data), message)) + return obj, created def _format_csv_line(self, values, empty=u"-"): return u'"' + u'","'.join( |