From d147e85723e8c9d3466c9b3d56288696b21a6ad8 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Tue, 14 Feb 2017 20:47:35 +0100 Subject: Importer: report bad configuration in a cleaner way --- ishtar_common/data_importer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'ishtar_common/data_importer.py') diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index a03f4de34..0b6662e1c 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -29,7 +29,7 @@ import zipfile from django.conf import settings from django.contrib.auth.models import User -from django.core.exceptions import ImproperlyConfigured +from django.db.models.fields import FieldDoesNotExist from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction from django.template.defaultfilters import slugify @@ -733,7 +733,7 @@ class Importer(object): q = ImporterModel.objects.filter(klass=cls_name) if q.count(): cls_name = q.all()[0].name - return ImproperlyConfigured( + return ImporterError( unicode(self.ERRORS['improperly_configured']).format(cls_name)) def _get_does_not_exist_in_db_error(self, model, data): @@ -1217,8 +1217,13 @@ class Importer(object): c_row.append(u" ; ".join([v for v in c_values])) def get_field(self, cls, attribute, data, m2ms, c_path, new_created): - field_object, model, direct, m2m = \ - cls._meta.get_field_by_name(attribute) + try: + field_object, model, direct, m2m = \ + cls._meta.get_field_by_name(attribute) + except FieldDoesNotExist: + raise ImporterError(unicode( + _(u"Importer configuration error: field \"{}\" does not exist " + u"for {}.")).format(attribute, cls._meta.verbose_name)) if m2m: many_values = data.pop(attribute) if hasattr(field_object, 'rel'): -- cgit v1.2.3 From 7d69b2b90a7c831477a2b1668bb9b7346120aaca Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 9 Feb 2017 23:16:25 +0100 Subject: Item keys are now related to the current importer (refs #3451) --- archaeological_operations/tests.py | 88 +++++++++++++++++++++++--------------- ishtar_common/data_importer.py | 31 ++++++++++---- ishtar_common/forms_common.py | 4 +- ishtar_common/models.py | 40 ++++++++++------- ishtar_common/tests.py | 2 +- 5 files changed, 105 insertions(+), 60 deletions(-) (limited to 'ishtar_common/data_importer.py') diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 7c141af93..296236029 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -34,7 +34,7 @@ import models from archaeological_operations import views -from ishtar_common.models import OrganizationType, Organization, \ +from ishtar_common.models import OrganizationType, Organization, ItemKey, \ ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, \ Town, ImporterColumn from archaeological_context_records.models import Unit @@ -74,26 +74,30 @@ class ImportTest(object): def init_ope_targetkey(self, imp): # doing manually connections - tg = TargetKey.objects.filter(target__target='operation_type' - ).order_by('-pk').all()[0] - tg.value = models.OperationType.objects.get( + target = TargetKey.objects.filter( + target__target='operation_type').order_by('-pk').all()[0] + target.value = models.OperationType.objects.get( txt_idx='prog_excavation').pk - tg.is_set = True - tg.save() - - target = TargetKey.objects.get(key='gallo-romain', - associated_import=imp) - gallo = models.Period.objects.get(txt_idx='gallo-roman') - target.value = gallo.pk target.is_set = True + target.associated_import = imp target.save() - target = TargetKey.objects.get(key='age-du-fer', - associated_import=imp) + target2 = TargetKey.objects.get(key='gallo-romain', + associated_import=imp) + gallo = models.Period.objects.get(txt_idx='gallo-roman') + target2.value = gallo.pk + target2.is_set = True + target2.associated_import = imp + target2.save() + + target3 = TargetKey.objects.get(key='age-du-fer', + associated_import=imp) iron = models.Period.objects.get(txt_idx='iron_age') - target.value = iron.pk - target.is_set = True - target.save() + target3.value = iron.pk + target3.is_set = True + target3.associated_import = imp + target3.save() + return [target, target2, target3] def init_ope(self): importer, form = self.init_ope_import() @@ -186,7 +190,7 @@ class ImportOperationTest(ImportTest, TestCase): impt.importation() # a new operation has now been imported current_ope_nb = models.Operation.objects.count() - self.assertTrue(current_ope_nb == (first_ope_nb + 1)) + self.assertEqual(current_ope_nb, first_ope_nb + 1) # and well imported last_ope = models.Operation.objects.order_by('-pk').all()[0] self.assertEqual(last_ope.name, u"Oppìdum de Paris") @@ -200,9 +204,28 @@ class ImportOperationTest(ImportTest, TestCase): # a second importation will be not possible: no two same patriarche # code impt.importation() - models.Operation.objects.count() - self.assertTrue(last_ope == - models.Operation.objects.order_by('-pk').all()[0]) + self.assertEqual(last_ope, + models.Operation.objects.order_by('-pk').all()[0]) + + def test_keys_limitation(self): + # each key association is associated to the import + init_ope_number = models.Operation.objects.count() + importer, form = self.init_ope_import() + impt = form.save(self.ishtar_user) + impt.initialize() + self.init_ope_targetkey(imp=impt) + + importer, form = self.init_ope_import() + other_imp = form.save(self.ishtar_user) + # associate with another import + for ik in ItemKey.objects.filter(importer=impt).all(): + ik.importer = other_imp + ik.save() + + impt.importation() + current_ope_nb = models.Operation.objects.count() + # no new operation + self.assertEqual(current_ope_nb, init_ope_number) def test_bad_configuration(self): importer, form = self.init_ope_import() @@ -809,7 +832,6 @@ class OperationSearchTest(TestCase, OperationInitTest): self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content)['total'], 1) - def testOwnSearch(self): c = Client() response = c.get(reverse('get-operation'), {'year': '2010'}) @@ -911,12 +933,10 @@ class OperationWizardCreationTest(WizardTest, OperationInitTest, TestCase): return super(OperationWizardCreationTest, self).pre_wizard() town = self.create_towns()[0] town_data = {'town': town.pk} - self.form_datas[0].form_datas['townsgeneral-operation_creation'].append( - town_data - ) - self.form_datas[1].form_datas['townsgeneral-operation_creation'].append( - town_data - ) + self.form_datas[0].form_datas[ + 'townsgeneral-operation_creation'].append(town_data) + self.form_datas[1].form_datas[ + 'townsgeneral-operation_creation'].append(town_data) parcel_data = { 'town': town.pk, 'year': 2017, 'section': 'S', 'parcel_number': '42'} @@ -997,7 +1017,8 @@ class OperationWizardClosingTest(OperationWizardCreationTest): self.assertFalse(ope.is_active()) self.assertEqual( ope.closing()['date'].strftime('%Y-%d-%m'), - self.form_datas[0].form_datas['date-operation_closing']['end_date']) + self.form_datas[0].form_datas['date-operation_closing']['end_date'] + ) class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest, @@ -1017,12 +1038,11 @@ class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest, FormData( "Admin act creation", form_datas={ - 'selec-operation_administrativeactop': { - }, - 'administrativeact-operation_administrativeactop': { - 'signature_date': str(datetime.date.today()) - } - }, + 'selec-operation_administrativeactop': {}, + 'administrativeact-operation_administrativeactop': { + 'signature_date': str(datetime.date.today()) + } + }, ) ] diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 0b6662e1c..fbf19def2 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -32,6 +32,7 @@ from django.contrib.auth.models import User from django.db.models.fields import FieldDoesNotExist from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction +from django.db.models import Q from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ @@ -148,13 +149,15 @@ class ChoiceChecker(object): class UnicodeFormater(Formater): def __init__(self, max_length=None, clean=False, re_filter=None, - notnull=False, prefix=u'', db_target=None): + notnull=False, prefix=u'', db_target=None, + import_instance=None): self.max_length = max_length self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull self.prefix = prefix + self.import_instance = import_instance def format(self, value): try: @@ -255,7 +258,8 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater, ChoiceChecker): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split='', db_target=None): + cli=False, many_split='', db_target=None, + import_instance=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) @@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.new_keys = {} self.match_table = {} self.many_split = many_split + self.import_instance = None for key, value in self.choices: value = unicode(value) if not self.strict: @@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker): def init_db_target(self): if not self.db_target: return - for target_key in self.db_target.keys.filter(is_set=True).all(): + q = self.db_target.keys.filter(is_set=True) + if self.import_instance: + q = q.filter(Q(associated_import=self.import_instance) | + Q(associated_import__isnull=True)) + for target_key in q.all(): key = target_key.key if not self.strict: key = slugify(key) @@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): class TypeFormater(StrChoiceFormater): def __init__(self, model, cli=False, defaults={}, many_split=False, - db_target=None): + db_target=None, import_instance=None): self.create = True self.strict = False self.model = model @@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater): self.equiv_dict, self.choices = {}, [] self.match_table = {} self.new_keys = {} + self.import_instance = import_instance for item in model.objects.all(): self.choices.append((item.pk, unicode(item))) - for key in item.get_keys(): + for key in item.get_keys(importer_id=import_instance.pk): self.equiv_dict[key] = item def prepare(self, value): @@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater): class DateFormater(Formater): - def __init__(self, date_formats=["%d/%m/%Y"], db_target=None): + def __init__(self, date_formats=["%d/%m/%Y"], db_target=None, + import_instance=None): self.date_formats = date_formats if type(date_formats) not in (list, tuple): self.date_formats = [self.date_formats] self.db_target = db_target + self.import_instance = import_instance def format(self, value): value = value.strip() @@ -511,7 +523,8 @@ class FileFormater(Formater): class StrToBoolean(Formater, ChoiceChecker): - def __init__(self, choices={}, cli=False, strict=False, db_target=None): + def __init__(self, choices={}, cli=False, strict=False, db_target=None, + import_instance=None): self.dct = copy.copy(choices) self.cli = cli self.strict = strict @@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.init_db_target() self.match_table = {} self.new_keys = {} + self.import_instance = import_instance def init_db_target(self): if not self.db_target: @@ -827,7 +841,7 @@ class Importer(object): vals[idx_col].append(val) for idx, formater in enumerate(self.line_format): if formater and idx < len(vals): - + formater.import_instance = self.import_instance if self.DB_TARGETS: field_names = formater.field_name if type(field_names) not in (list, tuple): @@ -1143,6 +1157,7 @@ class Importer(object): self.concat_str[field_name] = concat_str if self.DB_TARGETS: + formater.import_instance = self.import_instance formater.reinit_db_target( self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)], idx_v) diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 7ab09f9f7..67299fbac 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(TargetKeyForm, self).__init__(*args, **kwargs) instance = getattr(self, 'instance', None) + self.associated_import = None if instance and instance.pk: + self.associated_import = instance.associated_import self.fields['target'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['title'] = unicode(instance) @@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm): super(TargetKeyForm, self).save(commit) if self.cleaned_data.get('value'): self.instance.is_set = True - self.associated_import = None + self.instance.associated_import = self.associated_import self.instance.save() diff --git a/ishtar_common/models.py b/ishtar_common/models.py index c1edc13c6..fa2b348bf 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -649,27 +649,34 @@ class GeneralType(Cached, models.Model): self.generate_key(force=True) return obj - def add_key(self, key, force=False): + def add_key(self, key, force=False, importer=None): content_type = ContentType.objects.get_for_model(self.__class__) - if not force and ItemKey.objects.filter( + if not importer and not force and ItemKey.objects.filter( key=key, content_type=content_type).count(): return if force: - ItemKey.objects.filter(key=key, content_type=content_type)\ + ItemKey.objects.filter(key=key, content_type=content_type, + importer=importer)\ .exclude(object_id=self.pk).delete() - ItemKey.objects.get_or_create(object_id=self.pk, key=key, - content_type=content_type) + ItemKey.objects.get_or_create( + object_id=self.pk, key=key, content_type=content_type, + importer=importer + ) def generate_key(self, force=False): for key in (slugify(self.label), self.txt_idx): self.add_key(key) - def get_keys(self): + def get_keys(self, importer_id=None): keys = [self.txt_idx] content_type = ContentType.objects.get_for_model(self.__class__) - for ik in ItemKey.objects.filter( - content_type=content_type, object_id=self.pk).exclude( - key=self.txt_idx).all(): + query = Q(content_type=content_type, object_id=self.pk, + importer__isnull=True) + if importer_id: + query |= Q(content_type=content_type, object_id=self.pk, + importer__pk=importer_id) + q = ItemKey.objects.filter(query) + for ik in q.exclude(key=self.txt_idx).all(): keys.append(ik.key) return keys @@ -1783,7 +1790,7 @@ class ImporterType(models.Model): def __unicode__(self): return self.name - def get_importer_class(self): + def get_importer_class(self, import_instance=None): if self.slug and self.slug in IMPORTER_CLASSES: cls = import_class(IMPORTER_CLASSES[self.slug]) return cls @@ -1806,7 +1813,8 @@ class ImporterType(models.Model): force_news = [] concat_str = [] for target in column.targets.all(): - ft = target.formater_type.get_formater_type(target) + ft = target.formater_type.get_formater_type( + target, import_instance=import_instance) if not ft: continue formater_types.append(ft) @@ -2107,10 +2115,10 @@ class TargetKey(models.Model): try: v = self.target.associated_model.objects.get( txt_idx=unicode(self.value)) - except (self.target.associated_model.DoesNotExist): + except self.target.associated_model.DoesNotExist: pass if v: - v.add_key(self.key) + v.add_key(self.key, importer=self.associated_import) return obj TARGET_MODELS = [ @@ -2204,10 +2212,10 @@ class FormaterType(models.Model): if self.format_type in IMPORTER_TYPES_CHOICES: return IMPORTER_TYPES_CHOICES[self.format_type] - def get_formater_type(self, target): + def get_formater_type(self, target, import_instance=None): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {'db_target': target} + kwargs = {'db_target': target, 'import_instance': import_instance} if self.many_split: kwargs['many_split'] = self.many_split if self.formater_type == 'TypeFormater': @@ -2346,7 +2354,7 @@ class Import(models.Model): return IMPORT_STATE_DCT[self.state] def get_importer_instance(self): - return self.importer_type.get_importer_class()( + return self.importer_type.get_importer_class(import_instance=self)( skip_lines=self.skip_lines, import_instance=self, conservative_import=self.conservative_import) diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 82acb1904..8953e3c1b 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -798,7 +798,7 @@ class ImportTest(TestCase): # town should be deleted self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0) - def testKeys(self): + def test_keys(self): content_type = ContentType.objects.get_for_model( models.OrganizationType) -- cgit v1.2.3 From 5921fedef065a34a97ecdd2ca4353aa72f25b116 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 16 Feb 2017 21:00:08 +0100 Subject: Imports: prevent creation of new items when data is empty --- archaeological_operations/tests.py | 25 +- .../tests/MCC-operations-example.csv | 1 + ishtar_common/data_importer.py | 274 +++++++++++---------- 3 files changed, 155 insertions(+), 145 deletions(-) (limited to 'ishtar_common/data_importer.py') diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 296236029..364cc4c8e 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -36,7 +36,7 @@ from archaeological_operations import views from ishtar_common.models import OrganizationType, Organization, ItemKey, \ ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, \ - Town, ImporterColumn + Town, ImporterColumn, Person from archaeological_context_records.models import Unit from ishtar_common import forms_common @@ -171,6 +171,7 @@ class ImportOperationTest(ImportTest, TestCase): def test_mcc_import_operation(self): first_ope_nb = models.Operation.objects.count() + first_person_nb = Person.objects.count() importer, form = self.init_ope_import() self.assertTrue(form.is_valid()) impt = form.save(self.ishtar_user) @@ -184,18 +185,20 @@ class ImportOperationTest(ImportTest, TestCase): current_ope_nb = models.Operation.objects.count() # no new operation imported because of a missing connection for # operation_type value - self.assertTrue(current_ope_nb == first_ope_nb) + self.assertEqual(current_ope_nb, first_ope_nb) self.init_ope_targetkey(imp=impt) impt.importation() - # a new operation has now been imported + # new operations have now been imported current_ope_nb = models.Operation.objects.count() - self.assertEqual(current_ope_nb, first_ope_nb + 1) + self.assertEqual(current_ope_nb, first_ope_nb + 2) + current_person_nb = Person.objects.count() + self.assertEqual(current_person_nb, first_person_nb + 1) # and well imported last_ope = models.Operation.objects.order_by('-pk').all()[0] self.assertEqual(last_ope.name, u"Oppìdum de Paris") - self.assertTrue(last_ope.code_patriarche == 4200) - self.assertTrue(last_ope.operation_type.txt_idx == 'prog_excavation') + self.assertEqual(last_ope.code_patriarche, 4200) + self.assertEqual(last_ope.operation_type.txt_idx, 'prog_excavation') self.assertEqual(last_ope.periods.count(), 2) periods = [period.txt_idx for period in last_ope.periods.all()] self.assertIn('iron_age', periods) @@ -239,7 +242,7 @@ class ImportOperationTest(ImportTest, TestCase): impt.initialize() self.init_ope_targetkey(imp=impt) impt.importation() - self.assertEqual(len(impt.errors), 1) + self.assertEqual(len(impt.errors), 2) self.assertIn("Importer configuration error", impt.errors[0]['error']) def test_model_limitation(self): @@ -253,10 +256,10 @@ class ImportOperationTest(ImportTest, TestCase): init_ope_number = models.Operation.objects.count() impt.importation() current_ope_nb = models.Operation.objects.count() - self.assertEqual(current_ope_nb, init_ope_number + 1) + self.assertEqual(current_ope_nb, init_ope_number + 2) - last_ope = models.Operation.objects.order_by('-pk').all()[0] - last_ope.delete() + for ope in models.Operation.objects.order_by('-pk').all()[:2]: + ope.delete() importer, form = self.init_ope_import() # add an inadequate model to make created_models non empty @@ -286,7 +289,7 @@ class ImportOperationTest(ImportTest, TestCase): # import of operations impt.importation() current_ope_nb = models.Operation.objects.count() - self.assertEqual(current_ope_nb, init_ope_number + 1) + self.assertEqual(current_ope_nb, init_ope_number + 2) def test_mcc_import_parcels(self): old_nb = models.Parcel.objects.count() diff --git a/archaeological_operations/tests/MCC-operations-example.csv b/archaeological_operations/tests/MCC-operations-example.csv index 432ceffca..3b9801c33 100644 --- a/archaeological_operations/tests/MCC-operations-example.csv +++ b/archaeological_operations/tests/MCC-operations-example.csv @@ -1,2 +1,3 @@ code OA,region,type operation,intitule operation,operateur,responsable operation,date debut terrain,date fin terrain,chronologie generale,identifiant document georeferencement,notice scientifique +4201,Bourgogne,Fouille programmée,Oppìdum de Paris 2,L'opérateur,,2000/01/31,2002/12/31,Age du Fer,, 4200,Bourgogne,Fouille programmée,Oppìdum de Paris,L'opérateur,Jean Sui-Resp'on Sablé,2000/01/22,2002/12/31,Age du Fer & Gallo-Romain,, diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index fbf19def2..426d32a7a 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -1381,146 +1381,152 @@ class Importer(object): def get_object(self, cls, data, path=[]): m2ms = [] - if data and type(data) == dict: - c_path = path[:] - - # get all related fields - new_created = {} - for attribute in list(data.keys()): - c_c_path = c_path[:] - if not attribute: - data.pop(attribute) - continue - if not data[attribute]: - continue - if attribute != '__force_new': - self.get_field(cls, attribute, data, m2ms, c_c_path, - new_created) - - create_dict = copy.deepcopy(data) - for k in create_dict.keys(): - # filter unnecessary default values - if type(create_dict[k]) == dict: - create_dict.pop(k) - # File doesn't like deepcopy - if type(create_dict[k]) == File: - create_dict[k] = copy.copy(data[k]) - - # default values - path = tuple(path) - defaults = {} - if path in self._defaults: - for k in self._defaults[path]: - if (k not in data or not data[k]): - defaults[k] = self._defaults[path][k] - - if 'history_modifier' in create_dict: - defaults.update({ - 'history_modifier': create_dict.pop('history_modifier') - }) - - created = False + if type(data) != dict: + return data, False + is_empty = not bool( + [k for k in data if k not in ('history_modifier', 'defaults') + and data[k]]) + if is_empty: + return None, False + + c_path = path[:] + + # get all related fields + new_created = {} + for attribute in list(data.keys()): + c_c_path = c_path[:] + if not attribute: + data.pop(attribute) + continue + if not data[attribute]: + continue + if attribute != '__force_new': + self.get_field(cls, attribute, data, m2ms, c_c_path, + new_created) + + create_dict = copy.deepcopy(data) + for k in create_dict.keys(): + # filter unnecessary default values + if type(create_dict[k]) == dict: + create_dict.pop(k) + # File doesn't like deepcopy + if type(create_dict[k]) == File: + create_dict[k] = copy.copy(data[k]) + + # default values + path = tuple(path) + defaults = {} + if path in self._defaults: + for k in self._defaults[path]: + if k not in data or not data[k]: + defaults[k] = self._defaults[path][k] + + if 'history_modifier' in create_dict: + defaults.update({ + 'history_modifier': create_dict.pop('history_modifier') + }) + + created = False + try: try: - try: - dct = create_dict.copy() - for key in dct: - if callable(dct[key]): - dct[key] = dct[key]() - if '__force_new' in dct: - created = dct.pop('__force_new') - if not [k for k in dct if dct[k] is not None]: - return None, created - new_dct = defaults.copy() - new_dct.update(dct) - if self.MODEL_CREATION_LIMIT and \ - cls not in self.MODEL_CREATION_LIMIT: - raise self._get_improperly_conf_error(cls) - obj = cls.objects.create(**new_dct) + dct = create_dict.copy() + for key in dct: + if callable(dct[key]): + dct[key] = dct[key]() + if '__force_new' in dct: + created = dct.pop('__force_new') + if not [k for k in dct if dct[k] is not None]: + return None, created + new_dct = defaults.copy() + new_dct.update(dct) + if self.MODEL_CREATION_LIMIT and \ + cls not in self.MODEL_CREATION_LIMIT: + raise self._get_improperly_conf_error(cls) + obj = cls.objects.create(**new_dct) + else: + # manage UNICITY_KEYS - only level 1 + if not path and self.UNICITY_KEYS: + for k in dct.keys(): + if k not in self.UNICITY_KEYS \ + and k != 'defaults': + defaults[k] = dct.pop(k) + if not self.MODEL_CREATION_LIMIT or \ + cls in self.MODEL_CREATION_LIMIT: + dct['defaults'] = defaults.copy() + obj, created = cls.objects.get_or_create(**dct) else: - # manage UNICITY_KEYS - only level 1 - if not path and self.UNICITY_KEYS: - for k in dct.keys(): - if k not in self.UNICITY_KEYS \ - and k != 'defaults': - defaults[k] = dct.pop(k) - if not self.MODEL_CREATION_LIMIT or \ - cls in self.MODEL_CREATION_LIMIT: + try: + obj = cls.objects.get(**dct) dct['defaults'] = defaults.copy() - obj, created = cls.objects.get_or_create(**dct) - else: - try: - obj = cls.objects.get(**dct) - dct['defaults'] = defaults.copy() - except cls.DoesNotExist: - raise self._get_does_not_exist_in_db_error( - cls, dct) - - if not created and not path and self.UNICITY_KEYS: - changed = False - if self.conservative_import: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue - val = getattr(obj, k) - if val is None or val == '': - changed = True - setattr(obj, k, new_val) - elif k in self.concats \ - and type(val) == unicode \ - and type(new_val) == unicode: - setattr(obj, k, val + u"\n" + new_val) - else: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue + except cls.DoesNotExist: + raise self._get_does_not_exist_in_db_error( + cls, dct) + + if not created and not path and self.UNICITY_KEYS: + changed = False + if self.conservative_import: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + val = getattr(obj, k) + if val is None or val == '': changed = True setattr(obj, k, new_val) - if changed: - obj.save() - if self.import_instance and hasattr(obj, 'imports') \ - and created: - obj.imports.add(self.import_instance) - except ValueError as e: - raise IntegrityError(e.message) - except IntegrityError as e: - raise IntegrityError(e.message) - except DatabaseError as e: - raise IntegrityError(e.message) - except cls.MultipleObjectsReturned as e: - created = False - if 'defaults' in dct: - dct.pop('defaults') - raise IntegrityError(e.message) - # obj = cls.objects.filter(**dct).all()[0] - for attr, value in m2ms: - values = [value] - if type(value) in (list, tuple): - values = value - for v in values: - getattr(obj, attr).add(v) - # force post save script - v.save() - if m2ms: - # force post save script - obj.save() + elif k in self.concats \ + and type(val) == unicode \ + and type(new_val) == unicode: + setattr(obj, k, val + u"\n" + new_val) + else: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + changed = True + setattr(obj, k, new_val) + if changed: + obj.save() + if self.import_instance and hasattr(obj, 'imports') \ + and created: + obj.imports.add(self.import_instance) + except ValueError as e: + raise IntegrityError(e.message) except IntegrityError as e: - message = e.message - try: - message = e.message.decode('utf-8') - except (UnicodeDecodeError, UnicodeDecodeError): - message = '' - try: - data = unicode(data) - except UnicodeDecodeError: - data = '' - raise ImporterError( - "Erreur d'import %s %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode("__".join(path)), - unicode(data), message)) - return obj, created - return data + raise IntegrityError(e.message) + except DatabaseError as e: + raise IntegrityError(e.message) + except cls.MultipleObjectsReturned as e: + created = False + if 'defaults' in dct: + dct.pop('defaults') + raise IntegrityError(e.message) + # obj = cls.objects.filter(**dct).all()[0] + for attr, value in m2ms: + values = [value] + if type(value) in (list, tuple): + values = value + for v in values: + getattr(obj, attr).add(v) + # force post save script + v.save() + if m2ms: + # force post save script + obj.save() + except IntegrityError as e: + message = e.message + try: + message = e.message.decode('utf-8') + except (UnicodeDecodeError, UnicodeDecodeError): + message = '' + try: + data = unicode(data) + except UnicodeDecodeError: + data = '' + raise ImporterError( + "Erreur d'import %s %s, contexte : %s, erreur : %s" + % (unicode(cls), unicode("__".join(path)), + unicode(data), message)) + return obj, created def _format_csv_line(self, values, empty=u"-"): return u'"' + u'","'.join( -- cgit v1.2.3