diff options
Diffstat (limited to 'archaeological_operations/data_importer.py')
| -rw-r--r-- | archaeological_operations/data_importer.py | 86 |
1 files changed, 47 insertions, 39 deletions
diff --git a/archaeological_operations/data_importer.py b/archaeological_operations/data_importer.py index 252fdcca0..9cbdca05c 100644 --- a/archaeological_operations/data_importer.py +++ b/archaeological_operations/data_importer.py @@ -17,15 +17,14 @@ # See the file COPYING for details. -import copy, datetime, re +import re from django.db import IntegrityError from django.template.defaultfilters import slugify -from django.utils.translation import ugettext_lazy as _ from ishtar_common.data_importer import * from ishtar_common.models import Town, OrganizationType, SourceType, \ - SupportType, Format, AuthorType + SupportType, Format, AuthorType from archaeological_operations import models from archaeological_operations.forms import OPERATOR @@ -33,13 +32,14 @@ from archaeological_operations.utils import parse_parcels RE_PERMIT_REFERENCE = re.compile('[A-Za-z]*(.*)') + class ImportParcelFormater(ImportFormater): - NEED = ['town',] + NEED = ['town', ] PARCEL_OWNER_KEY = 'associated_file' def post_process(self, obj, context, value, owner=None): value = value.strip() - base_dct = {self.PARCEL_OWNER_KEY:obj, 'history_modifier':owner} + base_dct = {self.PARCEL_OWNER_KEY: obj, 'history_modifier': owner} if 'parcels' in context: for key in context['parcels']: if context['parcels'][key]: @@ -49,8 +49,9 @@ class ImportParcelFormater(ImportFormater): try: models.Parcel.objects.get_or_create(**parcel_dct) except IntegrityError: - raise ImporterError("Erreur d'import parcelle, contexte : %s" \ - % unicode(parcel_dct)) + raise ImporterError("Erreur d'import parcelle, contexte : %s" + % unicode(parcel_dct)) + class ImportYearFormater(ImportFormater): def post_process(self, obj, context, value, owner=None): @@ -60,6 +61,7 @@ class ImportYearFormater(ImportFormater): obj.year = value.year obj.save() + class TownFormater(Formater): def __init__(self, town_full_dct={}, town_dct={}): self._town_full_dct = town_full_dct @@ -70,13 +72,13 @@ class TownFormater(Formater): for town in Town.objects.all(): key = (slugify(town.name.strip()), town.numero_insee[:2]) if key in self._town_full_dct: - print("Danger! %s is ambiguous with another town on the same "\ - "department."% town.name) + print("Danger! %s is ambiguous with another town on the same " + "department." % town.name) continue self._town_full_dct[key] = town key = slugify(town.name.strip()) if key in self._town_dct: - print("Warning %s is ambiguous with no department provided" %\ + print("Warning %s is ambiguous with no department provided" % town.name) continue self._town_dct[key] = town @@ -98,6 +100,7 @@ class TownFormater(Formater): if key in self._town_dct: return self._town_dct[key] + class TownINSEEFormater(Formater): def __init__(self): self._town_dct = {} @@ -114,6 +117,7 @@ class TownINSEEFormater(Formater): self._town_dct[value] = q.all()[0] return self._town_dct[value] + class SurfaceFormater(Formater): def test(self): assert self.format(u"352 123") == 352123 @@ -133,14 +137,13 @@ class SurfaceFormater(Formater): try: return int(value.replace(' ', '')) * factor except ValueError: - raise ImporterError("Erreur import surface : %s" \ - % unicode(value)) + raise ImporterError("Erreur import surface : %s" % unicode(value)) -#RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)[, ](\d{5}) (.*?) *(?: "\ -# "*CEDEX|cedex|Cedex *\d*)*") +# RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)[, ](\d{5}) (.*?) *(?: "\ +# "*CEDEX|cedex|Cedex *\d*)*") -RE_NAME_ADD_CD_POSTAL_TOWN = re.compile("(.+)?[, ]*" + NEW_LINE_BREAK \ - + "(.+)?[, ]*(\d{2} *\d{3})[, ]*(.+)") +RE_NAME_ADD_CD_POSTAL_TOWN = re.compile( + "(.+)?[, ]*" + NEW_LINE_BREAK + "(.+)?[, ]*(\d{2} *\d{3})[, ]*(.+)") RE_ADD_CD_POSTAL_TOWN = re.compile("(.+)?[, ]*(\d{2} *\d{3})[, ]*(.+)") @@ -148,14 +151,15 @@ RE_CD_POSTAL_FILTER = re.compile("(\d*) (\d*)") RE_ORGA = re.compile("([^,]*)") + class OperationImporterBibracte(Importer): OBJECT_CLS = models.Operation DESC = u"Exports Bibracte : importeur pour l'onglet opération" DEFAULTS = { - ('operator',):{ - 'organization_type':OPERATOR - }, - } + ('operator',): { + 'organization_type': OPERATOR + }, + } LINE_FORMAT = [ # CODE OPE ImportFormater('operation_code', IntegerFormater(),), @@ -165,9 +169,9 @@ class OperationImporterBibracte(Importer): ImportFormater('operation_type', TypeFormater(models.OperationType),), # NOM ImportFormater('common_name', UnicodeFormater(120),), - # OPERATEUR + # OPERATEUR ImportFormater('operator__name', UnicodeFormater(120),), - # resp. lien IMPORT avec personne + # resp. lien IMPORT avec personne ImportFormater('in_charge__raw_name', UnicodeFormater(300),), # début ImportFormater('start_date', DateFormater(['%Y/%m/%d']),), @@ -181,23 +185,25 @@ class OperationImporterBibracte(Importer): RE_PARCEL_SECT_NUM = re.compile("([A-Za-z]*)([0-9]*)") RE_NUM_INSEE = re.compile("([0-9]*)") + class ParcelImporterBibracte(Importer): OBJECT_CLS = models.Parcel DESC = u"Exports Bibracte : importeur pour l'onglet parcelles" DEFAULTS = { - ('operator',):{ - 'organization_type':OrganizationType.objects.get( - txt_idx="operator")}, - } + ('operator',): { + 'organization_type': OrganizationType.objects.get( + txt_idx="operator")}, + } LINE_FORMAT = [ # code OA ImportFormater('operation__operation_code', IntegerFormater(),), # identifiant parcelle - ImportFormater(['section', 'parcel_number'], - [UnicodeFormater(4), UnicodeFormater(6),], - regexp=RE_PARCEL_SECT_NUM, - regexp_formater_args=[[0], [1]], required=False, - duplicate_fields=['external_id'],), + ImportFormater( + ['section', 'parcel_number'], + [UnicodeFormater(4), UnicodeFormater(6), ], + regexp=RE_PARCEL_SECT_NUM, + regexp_formater_args=[[0], [1]], required=False, + duplicate_fields=['external_id'],), # numero parcelle ImportFormater('parcel_number', UnicodeFormater(6), required=False,), @@ -211,7 +217,7 @@ class ParcelImporterBibracte(Importer): # numero INSEE commune ImportFormater('town__numero_insee', UnicodeFormater(6), regexp=RE_NUM_INSEE, required=False,), - # nom departement + # nom departement None, # lieu dit adresse ImportFormater('address', UnicodeFormater(500), @@ -224,8 +230,7 @@ MAIN_AUTHOR, created = AuthorType.objects.get_or_create(txt_idx='main_author') class DocImporterBibracte(Importer): OBJECT_CLS = models.OperationSource DEFAULTS = { - ('authors',):{ - 'author_type':MAIN_AUTHOR}, + ('authors',): {'author_type': MAIN_AUTHOR}, } DESC = u"Exports Bibracte : importeur pour l'onglet documentation" LINE_FORMAT = [ @@ -233,17 +238,20 @@ class DocImporterBibracte(Importer): ImportFormater('operation__operation_code', IntegerFormater(),), # identifiant documentation ImportFormater('external_id', UnicodeFormater(12),), - # type - ImportFormater('source_type', TypeFormater(SourceType), required=False), + # type + ImportFormater('source_type', TypeFormater(SourceType), + required=False), # nature support - ImportFormater('support_type', TypeFormater(SupportType), required=False), + ImportFormater('support_type', TypeFormater(SupportType), + required=False), # nombre element ImportFormater('item_number', IntegerFormater(), required=False), # auteur - ImportFormater('authors__person__raw_name', UnicodeFormater(300), required=False), + ImportFormater('authors__person__raw_name', UnicodeFormater(300), + required=False), # annee ImportFormater('creation_date', DateFormater(['%Y']),), - # format + # format ImportFormater('format_type', TypeFormater(Format), required=False), # description legende ImportFormater('description', UnicodeFormater(1000), required=False), |