diff options
| -rw-r--r-- | archaeological_operations/tests.py | 88 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 31 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 4 | ||||
| -rw-r--r-- | ishtar_common/models.py | 40 | ||||
| -rw-r--r-- | ishtar_common/tests.py | 2 |
5 files changed, 105 insertions, 60 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 7c141af93..296236029 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -34,7 +34,7 @@ import models from archaeological_operations import views -from ishtar_common.models import OrganizationType, Organization, \ +from ishtar_common.models import OrganizationType, Organization, ItemKey, \ ImporterType, IshtarUser, TargetKey, ImporterModel, IshtarSiteProfile, \ Town, ImporterColumn from archaeological_context_records.models import Unit @@ -74,26 +74,30 @@ class ImportTest(object): def init_ope_targetkey(self, imp): # doing manually connections - tg = TargetKey.objects.filter(target__target='operation_type' - ).order_by('-pk').all()[0] - tg.value = models.OperationType.objects.get( + target = TargetKey.objects.filter( + target__target='operation_type').order_by('-pk').all()[0] + target.value = models.OperationType.objects.get( txt_idx='prog_excavation').pk - tg.is_set = True - tg.save() - - target = TargetKey.objects.get(key='gallo-romain', - associated_import=imp) - gallo = models.Period.objects.get(txt_idx='gallo-roman') - target.value = gallo.pk target.is_set = True + target.associated_import = imp target.save() - target = TargetKey.objects.get(key='age-du-fer', - associated_import=imp) + target2 = TargetKey.objects.get(key='gallo-romain', + associated_import=imp) + gallo = models.Period.objects.get(txt_idx='gallo-roman') + target2.value = gallo.pk + target2.is_set = True + target2.associated_import = imp + target2.save() + + target3 = TargetKey.objects.get(key='age-du-fer', + associated_import=imp) iron = models.Period.objects.get(txt_idx='iron_age') - target.value = iron.pk - target.is_set = True - target.save() + target3.value = iron.pk + target3.is_set = True + target3.associated_import = imp + target3.save() + return [target, target2, target3] def init_ope(self): importer, form = self.init_ope_import() @@ -186,7 +190,7 @@ class ImportOperationTest(ImportTest, TestCase): impt.importation() # a new operation has now been imported current_ope_nb = models.Operation.objects.count() - self.assertTrue(current_ope_nb == (first_ope_nb + 1)) + self.assertEqual(current_ope_nb, first_ope_nb + 1) # and well imported last_ope = models.Operation.objects.order_by('-pk').all()[0] self.assertEqual(last_ope.name, u"Oppìdum de Paris") @@ -200,9 +204,28 @@ class ImportOperationTest(ImportTest, TestCase): # a second importation will be not possible: no two same patriarche # code impt.importation() - models.Operation.objects.count() - self.assertTrue(last_ope == - models.Operation.objects.order_by('-pk').all()[0]) + self.assertEqual(last_ope, + models.Operation.objects.order_by('-pk').all()[0]) + + def test_keys_limitation(self): + # each key association is associated to the import + init_ope_number = models.Operation.objects.count() + importer, form = self.init_ope_import() + impt = form.save(self.ishtar_user) + impt.initialize() + self.init_ope_targetkey(imp=impt) + + importer, form = self.init_ope_import() + other_imp = form.save(self.ishtar_user) + # associate with another import + for ik in ItemKey.objects.filter(importer=impt).all(): + ik.importer = other_imp + ik.save() + + impt.importation() + current_ope_nb = models.Operation.objects.count() + # no new operation + self.assertEqual(current_ope_nb, init_ope_number) def test_bad_configuration(self): importer, form = self.init_ope_import() @@ -809,7 +832,6 @@ class OperationSearchTest(TestCase, OperationInitTest): self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content)['total'], 1) - def testOwnSearch(self): c = Client() response = c.get(reverse('get-operation'), {'year': '2010'}) @@ -911,12 +933,10 @@ class OperationWizardCreationTest(WizardTest, OperationInitTest, TestCase): return super(OperationWizardCreationTest, self).pre_wizard() town = self.create_towns()[0] town_data = {'town': town.pk} - self.form_datas[0].form_datas['townsgeneral-operation_creation'].append( - town_data - ) - self.form_datas[1].form_datas['townsgeneral-operation_creation'].append( - town_data - ) + self.form_datas[0].form_datas[ + 'townsgeneral-operation_creation'].append(town_data) + self.form_datas[1].form_datas[ + 'townsgeneral-operation_creation'].append(town_data) parcel_data = { 'town': town.pk, 'year': 2017, 'section': 'S', 'parcel_number': '42'} @@ -997,7 +1017,8 @@ class OperationWizardClosingTest(OperationWizardCreationTest): self.assertFalse(ope.is_active()) self.assertEqual( ope.closing()['date'].strftime('%Y-%d-%m'), - self.form_datas[0].form_datas['date-operation_closing']['end_date']) + self.form_datas[0].form_datas['date-operation_closing']['end_date'] + ) class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest, @@ -1017,12 +1038,11 @@ class OperationAdminActWizardCreationTest(WizardTest, OperationInitTest, FormData( "Admin act creation", form_datas={ - 'selec-operation_administrativeactop': { - }, - 'administrativeact-operation_administrativeactop': { - 'signature_date': str(datetime.date.today()) - } - }, + 'selec-operation_administrativeactop': {}, + 'administrativeact-operation_administrativeactop': { + 'signature_date': str(datetime.date.today()) + } + }, ) ] diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 0b6662e1c..fbf19def2 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -32,6 +32,7 @@ from django.contrib.auth.models import User from django.db.models.fields import FieldDoesNotExist from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction +from django.db.models import Q from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ @@ -148,13 +149,15 @@ class ChoiceChecker(object): class UnicodeFormater(Formater): def __init__(self, max_length=None, clean=False, re_filter=None, - notnull=False, prefix=u'', db_target=None): + notnull=False, prefix=u'', db_target=None, + import_instance=None): self.max_length = max_length self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull self.prefix = prefix + self.import_instance = import_instance def format(self, value): try: @@ -255,7 +258,8 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater, ChoiceChecker): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split='', db_target=None): + cli=False, many_split='', db_target=None, + import_instance=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) @@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.new_keys = {} self.match_table = {} self.many_split = many_split + self.import_instance = None for key, value in self.choices: value = unicode(value) if not self.strict: @@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker): def init_db_target(self): if not self.db_target: return - for target_key in self.db_target.keys.filter(is_set=True).all(): + q = self.db_target.keys.filter(is_set=True) + if self.import_instance: + q = q.filter(Q(associated_import=self.import_instance) | + Q(associated_import__isnull=True)) + for target_key in q.all(): key = target_key.key if not self.strict: key = slugify(key) @@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): class TypeFormater(StrChoiceFormater): def __init__(self, model, cli=False, defaults={}, many_split=False, - db_target=None): + db_target=None, import_instance=None): self.create = True self.strict = False self.model = model @@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater): self.equiv_dict, self.choices = {}, [] self.match_table = {} self.new_keys = {} + self.import_instance = import_instance for item in model.objects.all(): self.choices.append((item.pk, unicode(item))) - for key in item.get_keys(): + for key in item.get_keys(importer_id=import_instance.pk): self.equiv_dict[key] = item def prepare(self, value): @@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater): class DateFormater(Formater): - def __init__(self, date_formats=["%d/%m/%Y"], db_target=None): + def __init__(self, date_formats=["%d/%m/%Y"], db_target=None, + import_instance=None): self.date_formats = date_formats if type(date_formats) not in (list, tuple): self.date_formats = [self.date_formats] self.db_target = db_target + self.import_instance = import_instance def format(self, value): value = value.strip() @@ -511,7 +523,8 @@ class FileFormater(Formater): class StrToBoolean(Formater, ChoiceChecker): - def __init__(self, choices={}, cli=False, strict=False, db_target=None): + def __init__(self, choices={}, cli=False, strict=False, db_target=None, + import_instance=None): self.dct = copy.copy(choices) self.cli = cli self.strict = strict @@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.init_db_target() self.match_table = {} self.new_keys = {} + self.import_instance = import_instance def init_db_target(self): if not self.db_target: @@ -827,7 +841,7 @@ class Importer(object): vals[idx_col].append(val) for idx, formater in enumerate(self.line_format): if formater and idx < len(vals): - + formater.import_instance = self.import_instance if self.DB_TARGETS: field_names = formater.field_name if type(field_names) not in (list, tuple): @@ -1143,6 +1157,7 @@ class Importer(object): self.concat_str[field_name] = concat_str if self.DB_TARGETS: + formater.import_instance = self.import_instance formater.reinit_db_target( self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)], idx_v) diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 7ab09f9f7..67299fbac 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(TargetKeyForm, self).__init__(*args, **kwargs) instance = getattr(self, 'instance', None) + self.associated_import = None if instance and instance.pk: + self.associated_import = instance.associated_import self.fields['target'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['title'] = unicode(instance) @@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm): super(TargetKeyForm, self).save(commit) if self.cleaned_data.get('value'): self.instance.is_set = True - self.associated_import = None + self.instance.associated_import = self.associated_import self.instance.save() diff --git a/ishtar_common/models.py b/ishtar_common/models.py index c1edc13c6..fa2b348bf 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -649,27 +649,34 @@ class GeneralType(Cached, models.Model): self.generate_key(force=True) return obj - def add_key(self, key, force=False): + def add_key(self, key, force=False, importer=None): content_type = ContentType.objects.get_for_model(self.__class__) - if not force and ItemKey.objects.filter( + if not importer and not force and ItemKey.objects.filter( key=key, content_type=content_type).count(): return if force: - ItemKey.objects.filter(key=key, content_type=content_type)\ + ItemKey.objects.filter(key=key, content_type=content_type, + importer=importer)\ .exclude(object_id=self.pk).delete() - ItemKey.objects.get_or_create(object_id=self.pk, key=key, - content_type=content_type) + ItemKey.objects.get_or_create( + object_id=self.pk, key=key, content_type=content_type, + importer=importer + ) def generate_key(self, force=False): for key in (slugify(self.label), self.txt_idx): self.add_key(key) - def get_keys(self): + def get_keys(self, importer_id=None): keys = [self.txt_idx] content_type = ContentType.objects.get_for_model(self.__class__) - for ik in ItemKey.objects.filter( - content_type=content_type, object_id=self.pk).exclude( - key=self.txt_idx).all(): + query = Q(content_type=content_type, object_id=self.pk, + importer__isnull=True) + if importer_id: + query |= Q(content_type=content_type, object_id=self.pk, + importer__pk=importer_id) + q = ItemKey.objects.filter(query) + for ik in q.exclude(key=self.txt_idx).all(): keys.append(ik.key) return keys @@ -1783,7 +1790,7 @@ class ImporterType(models.Model): def __unicode__(self): return self.name - def get_importer_class(self): + def get_importer_class(self, import_instance=None): if self.slug and self.slug in IMPORTER_CLASSES: cls = import_class(IMPORTER_CLASSES[self.slug]) return cls @@ -1806,7 +1813,8 @@ class ImporterType(models.Model): force_news = [] concat_str = [] for target in column.targets.all(): - ft = target.formater_type.get_formater_type(target) + ft = target.formater_type.get_formater_type( + target, import_instance=import_instance) if not ft: continue formater_types.append(ft) @@ -2107,10 +2115,10 @@ class TargetKey(models.Model): try: v = self.target.associated_model.objects.get( txt_idx=unicode(self.value)) - except (self.target.associated_model.DoesNotExist): + except self.target.associated_model.DoesNotExist: pass if v: - v.add_key(self.key) + v.add_key(self.key, importer=self.associated_import) return obj TARGET_MODELS = [ @@ -2204,10 +2212,10 @@ class FormaterType(models.Model): if self.format_type in IMPORTER_TYPES_CHOICES: return IMPORTER_TYPES_CHOICES[self.format_type] - def get_formater_type(self, target): + def get_formater_type(self, target, import_instance=None): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {'db_target': target} + kwargs = {'db_target': target, 'import_instance': import_instance} if self.many_split: kwargs['many_split'] = self.many_split if self.formater_type == 'TypeFormater': @@ -2346,7 +2354,7 @@ class Import(models.Model): return IMPORT_STATE_DCT[self.state] def get_importer_instance(self): - return self.importer_type.get_importer_class()( + return self.importer_type.get_importer_class(import_instance=self)( skip_lines=self.skip_lines, import_instance=self, conservative_import=self.conservative_import) diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 82acb1904..8953e3c1b 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -798,7 +798,7 @@ class ImportTest(TestCase): # town should be deleted self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0) - def testKeys(self): + def test_keys(self): content_type = ContentType.objects.get_for_model( models.OrganizationType) |