diff options
Diffstat (limited to 'archaeological_operations/utils.py')
| -rw-r--r-- | archaeological_operations/utils.py | 186 |
1 files changed, 112 insertions, 74 deletions
diff --git a/archaeological_operations/utils.py b/archaeological_operations/utils.py index 60b1585ef..53bfedc62 100644 --- a/archaeological_operations/utils.py +++ b/archaeological_operations/utils.py @@ -21,23 +21,24 @@ DELIMITER = ";" QUOTECHAR = '"' import datetime -import csv, codecs import re from django.conf import settings -from django.db import transaction +from django.core.exceptions import ObjectDoesNotExist from django.template.defaultfilters import slugify from django.contrib.auth.models import User from ishtar_common.models import Town, Person, PersonType, OrganizationType, \ Organization, SourceType -from archaeological_files.models import PermitType, File, FileType -from archaeological_operations.models import Operation, OperationType, Period, \ - AdministrativeAct, ActType, OperationSource, Parcel +from archaeological_files.models import PermitType +from archaeological_operations.models import OperationType, Period, \ + ActType + def get_default_person(): return User.objects.order_by('pk').all()[0] + def _get_parse_string(trunc_number=None): def parse_string(value): value = value.strip() @@ -51,20 +52,24 @@ def _get_parse_string(trunc_number=None): parse_string = _get_parse_string() + def parse_multivalue(value): - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', name) + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', value) s1 = re.sub('([a-z0-9])([A-Z])', r'\1 \2', s1) return re.sub('([0-9])([a-z])', r'\1 \2', s1) ope_types = {} + + def _init_ope_types(): for k in settings.ISHTAR_OPE_TYPES.keys(): ot, created = OperationType.objects.get_or_create( - txt_idx=settings.ISHTAR_OPE_TYPES[k][0], - defaults={'label':settings.ISHTAR_OPE_TYPES[k][1], - 'preventive':k[0]==u'préventive'}) + txt_idx=settings.ISHTAR_OPE_TYPES[k][0], + defaults={'label': settings.ISHTAR_OPE_TYPES[k][1], + 'preventive': k[0] == u'préventive'}) ope_types[k] = ot + def parse_operationtype(value, preventive, owner): value = (preventive.strip(), value.strip()) if not ope_types: @@ -75,6 +80,8 @@ def parse_operationtype(value, preventive, owner): periods = {} periods_keys = [] + + def _init_period(): for k in settings.ISHTAR_PERIODS.keys(): periods[k] = Period.objects.get(txt_idx=settings.ISHTAR_PERIODS[k]) @@ -92,6 +99,7 @@ def _init_period(): period_names_keys.sort(key=len) period_names_keys.reverse() + def parse_period(value): value = parse_string(value) value = value[3:] if value.startswith('EUR') else value @@ -119,6 +127,7 @@ REPLACED_PERIOD_DCT = dict(_REPLACED_PERIOD) period_names = {} period_names_keys = {} + def parse_period_name(value): if not period_names: _init_period() @@ -138,36 +147,41 @@ def parse_period_name(value): _CACHED_PERMIT_TYPES = {} + def _init_permit_type(): for k in settings.ISHTAR_PERMIT_TYPES: txt_idx, label = settings.ISHTAR_PERMIT_TYPES[k] - permit_type, created = PermitType.objects.get_or_create(txt_idx=txt_idx, - defaults={'label':label, - 'available':True}) + permit_type, created = PermitType.objects.get_or_create( + txt_idx=txt_idx, defaults={'label': label, + 'available': True}) _CACHED_PERMIT_TYPES[k] = permit_type + def parse_permittype(value): value = parse_string(value).lower() if not _CACHED_PERMIT_TYPES: _init_permit_type() if value not in _CACHED_PERMIT_TYPES: - if not "" in _CACHED_PERMIT_TYPES: + if "" not in _CACHED_PERMIT_TYPES: return value = "" return _CACHED_PERMIT_TYPES[value] _CACHED_ADMIN_ACT_TYPES = {} + + def parse_admin_act_typ(value, code, owner): value = parse_string(value).lower() code = parse_string(code).lower() if not value or not code: return if code not in _CACHED_ADMIN_ACT_TYPES: - act_type, created = ActType.objects.get_or_create(txt_idx=code, - defaults={'label':value}) - _CACHED_ADMIN_ACT_TYPES[code] = act_type + act_type, created = ActType.objects.get_or_create( + txt_idx=code, defaults={'label': value}) + _CACHED_ADMIN_ACT_TYPES[code] = act_type return _CACHED_ADMIN_ACT_TYPES[code] + def parse_fileref(value): value = parse_string(value).split('/')[0] value = value.split('.')[0] @@ -176,6 +190,7 @@ def parse_fileref(value): return None return int(match.group()) + def parse_orga(value, alternate_value, owner): value = parse_string(value) if not value: @@ -194,6 +209,7 @@ def parse_orga(value, alternate_value, owner): history_modifier=owner) return orga + def parse_bool(value): value = parse_string(value) if value.lower() in ('yes', 'oui'): @@ -204,6 +220,7 @@ def parse_bool(value): value = None return value + def parse_date(value): value = parse_string(value).split(' ')[0] try: @@ -211,6 +228,7 @@ def parse_date(value): except: return None + def parse_yearref(value): value = parse_string(value).split('.')[0] match = re.search('[0-9].[0-9]*', value) @@ -218,6 +236,7 @@ def parse_yearref(value): return None return int(match.group()) + def parse_surface(value): value = parse_string(value) value = value.replace(',', '.') @@ -230,6 +249,7 @@ def parse_surface(value): except: return None + def parse_year(value): value = parse_string(value) try: @@ -240,6 +260,7 @@ def parse_year(value): return None return yr + def parse_trunc_patriarche(value): value = parse_string(value) if not value: @@ -251,6 +272,7 @@ def parse_trunc_patriarche(value): return return '18' + unicode(value) + def parse_operation_code(value): value = parse_string(value) code = value.split('.')[-1] @@ -259,12 +281,14 @@ def parse_operation_code(value): except: return + def parse_title(value): value = parse_string(value) if not value: return return value.title() + def parse_name_surname(value): value = parse_string(value) items = value.split(' ') @@ -273,40 +297,41 @@ def parse_name_surname(value): if len(items) > 1: name = " ".join(items[:-1]) surname = items[-1] - values = {"surname":parse_title(surname)[:30], - "name":parse_title(name)[:30]} + values = {"surname": parse_title(surname)[:30], + "name": parse_title(name)[:30]} if not values['surname'] and not values['name']: return q = Person.objects.filter(**values) if q.count(): return q.all()[0] else: - defaults = {'history_modifier':get_default_user(), - 'title':''} + defaults = {'history_modifier': get_default_user(), + 'title': ''} defaults.update(values) p = Person.objects.create(**defaults) p.person_types.add(PersonType.objects.get( - txt_idx='head_scientist')) + txt_idx='head_scientist')) return p + def parse_person(surname, name, old_ref, owner): - value = parse_string(value) - values = {"surname":parse_title(surname), - "name":parse_title(name)} + values = {"surname": parse_title(surname), + "name": parse_title(name)} if not values['surname'] and not values['name']: return q = Person.objects.filter(**values) if q.count(): return q.all()[0] else: - defaults = {'history_modifier':owner, - 'title':''} + defaults = {'history_modifier': owner, + 'title': ''} defaults.update(values) p = Person.objects.create(**defaults) p.person_types.add(PersonType.objects.get( - txt_idx='head_scientist')) + txt_idx='head_scientist')) return p + def parse_comment_addr_nature(nature, addr, owner): addr = parse_string(addr) nature = parse_string(nature) @@ -322,57 +347,60 @@ def parse_comment_addr_nature(nature, addr, owner): # si pas de start date : premier janvier de year ope_types = { - 'AET':('other_study', - 'Autre étude', True), - 'APP':('assistance_preparation_help', - 'Aide à la préparation de publication', True), - 'DOC':('documents_study', - 'Étude documentaire', True), - 'EV':('evaluation', - "Fouille d'évaluation", True), - 'FOU':('ancient_excavation', - "Fouille ancienne", True), - 'FP':('prog_excavation', - "Fouille programmée", False), - 'MH':('building_study', "Fouille avant MH", True), - 'OPD':('arch_diagnostic', - "Diagnostic archéologique", True), - 'PAN':('analysis_program', - "Programme d'analyses", False), - 'PCR':('collective_research_project', - "Projet collectif de recherche", False), - 'PMS':('specialized_eqp_prospection', - "Prospection avec matériel spécialisé", False), - 'PRD':('diachronic_prospection', + 'AET': ('other_study', + 'Autre étude', True), + 'APP': ('assistance_preparation_help', + 'Aide à la préparation de publication', True), + 'DOC': ('documents_study', + 'Étude documentaire', True), + 'EV': ('evaluation', + "Fouille d'évaluation", True), + 'FOU': ('ancient_excavation', + "Fouille ancienne", True), + 'FP': ('prog_excavation', + "Fouille programmée", False), + 'MH': ('building_study', "Fouille avant MH", True), + 'OPD': ('arch_diagnostic', + "Diagnostic archéologique", True), + 'PAN': ('analysis_program', + "Programme d'analyses", False), + 'PCR': ('collective_research_project', + "Projet collectif de recherche", False), + 'PMS': ('specialized_eqp_prospection', + "Prospection avec matériel spécialisé", False), + 'PRD': ('diachronic_prospection', + "Prospection diachronique", False), + 'PI': ('diachronic_prospection', "Prospection diachronique", False), - 'PI':('diachronic_prospection', - "Prospection diachronique", False), - 'PRM':('metal_detector_prospection', - "Prospection détecteur de métaux", False), - 'PRT':('thematic_prospection', - "Prospection thématique", False), - 'PT':('thematic_prospection', + 'PRM': ('metal_detector_prospection', + "Prospection détecteur de métaux", False), + 'PRT': ('thematic_prospection', + "Prospection thématique", False), + 'PT': ('thematic_prospection', "Prospection thématique", False), - 'RAR':('cave_art_record', - "Relevé d'art rupestre", False), - 'SD':('sampling_research', - "Sondage", False), - 'SP':('prev_excavation', - "Fouille préventive", True), - 'SU':('emergency_excavation', - "Fouille préventive d'urgence", True), + 'RAR': ('cave_art_record', + "Relevé d'art rupestre", False), + 'SD': ('sampling_research', + "Sondage", False), + 'SP': ('prev_excavation', + "Fouille préventive", True), + 'SU': ('emergency_excavation', + "Fouille préventive d'urgence", True), } _CACHED_OPE_TYPES = {} + def _prepare_ope_types(): for k in ope_types.keys(): txt_idx, label, preventive = ope_types[k] - ot, created = OperationType.objects.get_or_create(txt_idx=txt_idx, - defaults={'label':label, 'preventive':preventive}) + ot, created = OperationType.objects.get_or_create( + txt_idx=txt_idx, defaults={'label': label, + 'preventive': preventive}) if k not in _CACHED_OPE_TYPES.keys(): _CACHED_OPE_TYPES[k] = ot + def parse_patriarche_operationtype(value): if value not in _CACHED_OPE_TYPES.keys(): return None @@ -380,6 +408,7 @@ def parse_patriarche_operationtype(value): _dpt_re_filter = re.compile('^\([0-9]*\) ') + def parse_ope_name(value): if not value: return '' @@ -389,6 +418,7 @@ def parse_ope_name(value): value = _dpt_re_filter.sub('', value) return value + def parse_ha(value): value = parse_string(value) try: @@ -397,6 +427,7 @@ def parse_ha(value): value = None return value + def parse_rapp_index(value): value = parse_string(value) items = re.findall(r'[0-9]+$', value) @@ -405,16 +436,18 @@ def parse_rapp_index(value): _CACHED_DOC_TYPES = {} + def parse_doc_types(value): value = parse_string(value) if value not in _CACHED_DOC_TYPES: if value not in settings.ISHTAR_DOC_TYPES: return _CACHED_DOC_TYPES[value], created = SourceType.objects.get_or_create( - txt_idx=value, - defaults={"label":settings.ISHTAR_DOC_TYPES[value]}) + txt_idx=value, + defaults={"label": settings.ISHTAR_DOC_TYPES[value]}) return _CACHED_DOC_TYPES[value] + def parse_insee(value): value = parse_string(value) values = [] @@ -427,15 +460,20 @@ def parse_insee(value): town = Town.objects.get(numero_insee=value) towns.append(town) except: - #sys.stderr.write('Numero INSEE : %s non existant en base' % value) + # sys.stderr.write('Numero INSEE : %s non existant en base' + # % value) continue return towns PARCEL_YEAR_REGEXP = re.compile(r"^([0-9]{4})[ :]+") -PARCEL_SECTION_REGEXP = re.compile(ur"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*[0-9]+[p]?)+)") +PARCEL_SECTION_REGEXP = re.compile( + ur"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*" + ur"((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*[0-9]+[p]?)+)") PARCEL_NB_RANGE_REGEXP = re.compile(ur'([0-9]+[p]?) (?:à|to) ([0-9]+[p]?)') -PARCEL_NB_REGEXP = re.compile(ur'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*([0-9]+[p]?)') +PARCEL_NB_REGEXP = re.compile( + ur'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*([0-9]+[p]?)') + def parse_parcels(parcel_str, insee_code=None, owner=None): parcels, town = [], None @@ -456,7 +494,7 @@ def parse_parcels(parcel_str, insee_code=None, owner=None): for num in PARCEL_NB_REGEXP.findall(nums): if len(unicode(num)) > 6: continue - dct = {'year':year, 'section':sector, 'parcel_number':num} + dct = {'year': year, 'section': sector, 'parcel_number': num} if town: dct['town'] = town if owner: @@ -471,8 +509,8 @@ def parse_parcels(parcel_str, insee_code=None, owner=None): except ValueError: continue for num in xrange(lower_range, higher_range): - dct = {'year':year, 'section':sector, - 'parcel_number':unicode(num)} + dct = {'year': year, 'section': sector, + 'parcel_number': unicode(num)} if town: dct['town'] = town if owner: |