diff options
| -rw-r--r-- | archaeological_context_records/fixtures/initial_data-fr.json | 34 | ||||
| -rw-r--r-- | archaeological_context_records/models.py | 2 | ||||
| -rw-r--r-- | archaeological_finds/fixtures/initial_data-fr.json | 6 | ||||
| -rw-r--r-- | archaeological_finds/tests.py | 29 | ||||
| -rw-r--r-- | archaeological_finds/tests/MCC-finds-example.csv | 10 | ||||
| -rw-r--r-- | archaeological_finds/tests/images.zip | bin | 0 -> 1696 bytes | |||
| -rw-r--r-- | archaeological_operations/fixtures/initial_data-fr.json | 14 | ||||
| -rw-r--r-- | archaeological_operations/fixtures/periods-fr.json | 8 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 37 |
9 files changed, 97 insertions, 43 deletions
diff --git a/archaeological_context_records/fixtures/initial_data-fr.json b/archaeological_context_records/fixtures/initial_data-fr.json index 00138177e..4991e7b66 100644 --- a/archaeological_context_records/fixtures/initial_data-fr.json +++ b/archaeological_context_records/fixtures/initial_data-fr.json @@ -23,7 +23,7 @@ "pk": 1, "model": "archaeological_context_records.activitytype", "fields": { - "comment": "Unit\u00e9 enregistrement qui n'est pas directement d'origine anthropique. ", + "comment": "Unit\u00e9 enregistrement qui n'est pas directement d'origine anthropique", "available": true, "txt_idx": "natural", "order": 1000, @@ -48,7 +48,7 @@ "comment": "Unit\u00e9 r\u00e9sultant de l'arr\u00eat d'anthropisation", "available": true, "txt_idx": "desertion", - "order": 1200, + "order": 1300, "label": "Abandon" } }, @@ -59,7 +59,7 @@ "comment": "Unit\u00e9 li\u00e9e \u00e0 l'anthropisation elle-m\u00eame", "available": true, "txt_idx": "occupation", - "order": 1300, + "order": 1200, "label": "Occupation" } }, @@ -385,9 +385,9 @@ "pk": 2, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "Une UE en rempli une autre (qui doit \u00eatre vraisemblablement une US de creusement)", + "comment": "Cette UE en remplit une autre (qui doit \u00eatre vraisemblablement une US de creusement)", "available": true, - "label": "Remplissage", + "label": "Remplit", "symmetrical": false, "inverse_relation": null, "order": 1, @@ -398,9 +398,9 @@ "pk": 4, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "A pr\u00e9ciser ", + "comment": "Cette UE est en relation indirecte avec une autre", "available": true, - "label": "Relation indirecte", + "label": "Est en relation indirecte avec", "symmetrical": true, "inverse_relation": null, "order": 1, @@ -411,7 +411,7 @@ "pk": 5, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "Cette Unit\u00e9 d\u2019enregistrement en inclut (regroupe) d'autres (ex: tranch\u00e9e / structures ou entre structure / US, etc.)", + "comment": "Cette UE en inclut (regroupe) d'autres (ex: tranch\u00e9e / structures ou entre structure / US, etc.)", "available": true, "label": "Inclut", "symmetrical": false, @@ -424,7 +424,7 @@ "pk": 7, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "UE est situ\u00e9e en-dessous d'une autre", + "comment": "Cette UE est situ\u00e9e en-dessous d'une autre", "available": true, "label": "Est en-dessous de", "symmetrical": false, @@ -437,7 +437,7 @@ "pk": 3, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "UE est situ\u00e9e au-dessus d'une autre", + "comment": "Cette UE est situ\u00e9e au-dessus d'une autre", "available": true, "label": "Est au-dessus de", "symmetrical": false, @@ -450,9 +450,9 @@ "pk": 8, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "UE recoup\u00e9e par une autre", + "comment": "Cette UE est recoup\u00e9e par une autre", "available": true, - "label": "Coup\u00e9 par", + "label": "Est coup\u00e9e par", "symmetrical": false, "inverse_relation": 1, "order": 1, @@ -463,7 +463,7 @@ "pk": 1, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "UE recoupant une autre", + "comment": "Cette UE en recoupe une autre", "available": true, "label": "Recoupe", "symmetrical": false, @@ -476,7 +476,7 @@ "pk": 9, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "UE \u00e9quivalant \u00e0 une autre", + "comment": "Cette UE est \u00e9quivalente \u00e0 une autre", "available": true, "label": "\u00c9quivaut \u00e0", "symmetrical": true, @@ -489,13 +489,13 @@ "pk": 6, "model": "archaeological_context_records.relationtype", "fields": { - "comment": "Cette unit\u00e9 d'enregistrement fait partie d'une autre", + "comment": "Cette UE fait partie d'une autre", "available": true, - "label": "Est inclus dans", + "label": "Est incluse dans", "symmetrical": false, "inverse_relation": 5, "order": 1, "txt_idx": "is_included" } } -]
\ No newline at end of file +] diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py index 5b22fb7f7..38c65a24a 100644 --- a/archaeological_context_records/models.py +++ b/archaeological_context_records/models.py @@ -99,7 +99,7 @@ class IdentificationType(GeneralType): class Meta: verbose_name = _(u"Identification Type") verbose_name_plural = _(u"Identification Types") - ordering = ('order',) + ordering = ('order', 'label') def __unicode__(self): return self.label diff --git a/archaeological_finds/fixtures/initial_data-fr.json b/archaeological_finds/fixtures/initial_data-fr.json index 1b26d8cd2..426a3800a 100644 --- a/archaeological_finds/fixtures/initial_data-fr.json +++ b/archaeological_finds/fixtures/initial_data-fr.json @@ -596,7 +596,7 @@ "comment": "", "available": true, "txt_idx": "recond", - "label": "A reconditionner" + "label": "\u00c0 reconditionner" } }, { @@ -606,7 +606,7 @@ "comment": "", "available": true, "txt_idx": "to_restaur", - "label": "A restaurer" + "label": "\u00c0 restaurer" } }, { @@ -616,7 +616,7 @@ "comment": "", "available": true, "txt_idx": "to_stab", - "label": "A stabiliser" + "label": "\u00c0 stabiliser" } }, { diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py index f75e9cfa3..c17e440b1 100644 --- a/archaeological_finds/tests.py +++ b/archaeological_finds/tests.py @@ -20,7 +20,8 @@ from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase -from ishtar_common.models import ImporterType, IshtarUser +from ishtar_common.models import ImporterType, IshtarUser, ImporterColumn,\ + FormaterType, ImportTarget from archaeological_finds import models @@ -42,12 +43,26 @@ class ImportFindTest(ImportContextRecordTest): self.testMCCImportContextRecords(test=False) old_nb = models.BaseFind.objects.count() + old_nb_find = models.Find.objects.count() MCC = ImporterType.objects.get(name=u"MCC - Mobilier") + + col = ImporterColumn.objects.create(col_number=25, + importer_type_id=MCC.pk) + formater = FormaterType.objects.filter( + formater_type='FileFormater').all()[0] + ImportTarget.objects.create(target='find__image', + formater_type_id=formater.pk, + column_id=col.pk) mcc_file = open( settings.ROOT_PATH + '../archaeological_finds/tests/MCC-finds-example.csv', 'rb') + mcc_images = open( + settings.ROOT_PATH + + '../archaeological_finds/tests/images.zip', 'rb') file_dict = {'imported_file': SimpleUploadedFile(mcc_file.name, - mcc_file.read())} + mcc_file.read()), + 'imported_images': SimpleUploadedFile(mcc_images.name, + mcc_images.read())} post_dict = {'importer_type': MCC.pk, 'skip_lines': 1, "encoding": 'utf-8'} form = forms_common.NewImportForm(data=post_dict, files=file_dict, @@ -60,15 +75,23 @@ class ImportFindTest(ImportContextRecordTest): # doing manual connections ceram = models.MaterialType.objects.get(txt_idx='ceramic').pk + glass = models.MaterialType.objects.get(txt_idx='glass').pk self.setTargetKey('find__material_types', 'terre-cuite', ceram) + self.setTargetKey('find__material_types', 'verre', glass) impt.importation() if not test: return # new finds has now been imported current_nb = models.BaseFind.objects.count() - self.assertTrue(current_nb == (old_nb + 4)) + self.assertEqual(current_nb, (old_nb + 4)) + current_nb = models.Find.objects.count() + self.assertEqual(current_nb, (old_nb_find + 4)) self.assertEqual( models.Find.objects.filter(material_types__pk=ceram).count(), 4) + self.assertEqual( + models.Find.objects.filter(material_types__pk=glass).count(), 1) + images = [f.image for f in models.Find.objects.all() if f.image.name] + self.assertEqual(len(images), 1) class FindInit(ContextRecordInit): diff --git a/archaeological_finds/tests/MCC-finds-example.csv b/archaeological_finds/tests/MCC-finds-example.csv index 13340b9aa..86a572657 100644 --- a/archaeological_finds/tests/MCC-finds-example.csv +++ b/archaeological_finds/tests/MCC-finds-example.csv @@ -1,5 +1,5 @@ -code OA,numero INSEE commune,identifiant parcelle,identifiant UE,identifiant materiel,objet lot,partage,composite,matiere,type,determination,numero contenant,nombre fragment,poids,unite poids,date decouverte,etat sanitaire,type preservation a envisager,commentaire,chronologie,localisation topographique,interet particulier,description,remontage -4200,75101,XXXX,HC,1,lot,non,,terre cuite,céramique,sfq,1,76,4040,g,,sais pas,,Blah,med,,,, -4200,75101,XXXX,H.-C.,1,lot,non,,terre cuite,céramique,qsdfsqfd,1,56,2280,g,,,,,med,,,, -4200,59350,YY55,US17,1,lot,non,,terre cuite,céramique,sqfdsq,2,44,5210,g,,stable,,,GR,,,, -4200,59350,YY55,US17,2,lot,non,,terre cuite,céramique,sqfdsq,45,43,1500,g,,,,,,,,, +code OA,numero INSEE commune,identifiant parcelle,identifiant UE,identifiant materiel,objet lot,partage,composite,matiere,type,determination,numero contenant,nombre fragment,poids,unite poids,date decouverte,etat sanitaire,type preservation a envisager,commentaire,chronologie,localisation topographique,interet particulier,description,remontage,photo +4200,75101,XXXX,HC,1,lot,non,,terre cuite,céramique,sfq,1,76,4040,g,,sais pas,,Blah,med,,,,, +4200,75101,XXXX,H.-C.,1,lot,non,,terre cuite,céramique,qsdfsqfd,1,56,2280,g,,,,,med,,,,, +4200,59350,YY55,US17,1,lot,non,,terre cuite,céramique,sqfdsq,2,44,5210,g,,stable,,,GR,,,,, +4200,59350,YY55,US17,2,lot,non,,terre cuite & verre,céramique,sqfdsq,45,43,1500,g,,,,,,,,,,image-1.jpg diff --git a/archaeological_finds/tests/images.zip b/archaeological_finds/tests/images.zip Binary files differnew file mode 100644 index 000000000..13b2680e7 --- /dev/null +++ b/archaeological_finds/tests/images.zip diff --git a/archaeological_operations/fixtures/initial_data-fr.json b/archaeological_operations/fixtures/initial_data-fr.json index dc70160db..03d331789 100644 --- a/archaeological_operations/fixtures/initial_data-fr.json +++ b/archaeological_operations/fixtures/initial_data-fr.json @@ -764,7 +764,7 @@ "parent": 39, "end_date": 0, "order": 2, - "label": "Epoque archa\u00efque", + "label": "\u00c9poque archa\u00efque", "start_date": 0, "txt_idx": "archaic_epoc" } @@ -792,7 +792,7 @@ "parent": 39, "end_date": 0, "order": 4, - "label": "Epoque classique", + "label": "\u00c9poque classique", "start_date": 0, "txt_idx": "classic_epoc" } @@ -806,7 +806,7 @@ "parent": 39, "end_date": 0, "order": 5, - "label": "Epoque hell\u00e9nistique", + "label": "\u00c9poque hell\u00e9nistique", "start_date": 0, "txt_idx": "hellen_epoc" } @@ -834,7 +834,7 @@ "parent": null, "end_date": 0, "order": 1000, - "label": "Epoque ind\u00e9termin\u00e9e", + "label": "\u00c9poque ind\u00e9termin\u00e9e", "start_date": 0, "txt_idx": "indetermined" } @@ -862,7 +862,7 @@ "parent": 31, "end_date": 2016, "order": 2800, - "label": "Epoque contemporaine", + "label": "\u00c9poque contemporaine", "start_date": 1815, "txt_idx": "contemporan" } @@ -876,7 +876,7 @@ "parent": 31, "end_date": 1815, "order": 2900, - "label": "Epoque moderne", + "label": "\u00c9poque moderne", "start_date": 1492, "txt_idx": "modern" } @@ -1484,4 +1484,4 @@ "label": "Re\u00e7us, un rapport d\u00e9pouill\u00e9, un autre non d\u00e9pouill\u00e9" } } -]
\ No newline at end of file +] diff --git a/archaeological_operations/fixtures/periods-fr.json b/archaeological_operations/fixtures/periods-fr.json index edb6401ff..1346cfbba 100644 --- a/archaeological_operations/fixtures/periods-fr.json +++ b/archaeological_operations/fixtures/periods-fr.json @@ -22,7 +22,7 @@ "parent": null, "end_date": 0, "order": 1000, - "label": "Epoque ind\u00e9termin\u00e9e", + "label": "\u00c9poque ind\u00e9termin\u00e9e", "start_date": 0, "txt_idx": "indetermined" } @@ -50,7 +50,7 @@ "parent": 31, "end_date": 2011, "order": 2800, - "label": "Epoque contemporaine", + "label": "\u00c9poque contemporaine", "start_date": 1815, "txt_idx": "contemporan" } @@ -64,7 +64,7 @@ "parent": 31, "end_date": 1815, "order": 2900, - "label": "Epoque moderne", + "label": "\u00c9poque moderne", "start_date": 1492, "txt_idx": "modern" } @@ -489,4 +489,4 @@ "txt_idx": "ancien_paleolithic" } } -]
\ No newline at end of file +] diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index a2d1f40d2..b669995e3 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -21,6 +21,7 @@ import copy import csv import datetime import io +import os import logging import re import sys @@ -500,6 +501,8 @@ class FileFormater(Formater): f.write(z.read()) f = open(filename, 'r') my_file = File(f) + # manualy set the file size because of an issue with TempFile + my_file.size = os.stat(filename).st_size return my_file except KeyError: raise ValueError(_(u"\"%(value)s\" is not a valid path for the " @@ -1161,7 +1164,7 @@ class Importer(object): concat_str=concat_str[idx]) c_row.append(u" ; ".join([v for v in c_values])) - def get_field(self, cls, attribute, data, m2ms, c_path): + def get_field(self, cls, attribute, data, m2ms, c_path, new_created): field_object, model, direct, m2m = \ cls._meta.get_field_by_name(attribute) if m2m: @@ -1226,9 +1229,19 @@ class Importer(object): for k in v.keys(): if k not in field_names: continue - self.get_field(model, k, v, m2m_m2ms, c_c_path) + self.get_field(model, k, v, m2m_m2ms, c_c_path, + new_created) if '__force_new' in v: created = v.pop('__force_new') + key = u";".join([u"{}-{}".format(k, v[k]) + for k in sorted(v.keys())]) + # only one forced creation + if attribute in new_created \ + and key in new_created[attribute]: + continue + if attribute not in new_created: + new_created[attribute] = [] + new_created[attribute].append(key) has_values = bool([1 for k in v if v[k]]) if has_values: v = model.objects.create(**v) @@ -1236,8 +1249,24 @@ class Importer(object): continue else: v['defaults'] = v.get('defaults', {}) + extra_fields = {} + # "File" type is a temp object and can be different + # for the same filename - it must be treated + # separatly + for field in model._meta.fields: + k = field.name + # attr_class est un attribut de FileField + if hasattr(field, 'attr_class') and k in v: + extra_fields[k] = v.pop(k) v, created = model.objects.get_or_create( **v) + changed = False + for k in extra_fields.keys(): + if extra_fields[k]: + changed = True + setattr(v, k, extra_fields[k]) + if changed: + v.save() for att, objs in m2m_m2ms: if type(objs) not in (list, tuple): objs = [objs] @@ -1270,6 +1299,7 @@ class Importer(object): c_path = path[:] # get all related fields + new_created = {} for attribute in list(data.keys()): c_c_path = c_path[:] if not attribute: @@ -1278,7 +1308,8 @@ class Importer(object): if not data[attribute]: continue if attribute != '__force_new': - self.get_field(cls, attribute, data, m2ms, c_c_path) + self.get_field(cls, attribute, data, m2ms, c_c_path, + new_created) # filter uncessary default values create_dict = copy.deepcopy(data) |