diff options
Diffstat (limited to 'archaeological_operations/utils.py')
| -rw-r--r-- | archaeological_operations/utils.py | 241 |
1 files changed, 121 insertions, 120 deletions
diff --git a/archaeological_operations/utils.py b/archaeological_operations/utils.py index bf38a675a..ba4f17358 100644 --- a/archaeological_operations/utils.py +++ b/archaeological_operations/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (C) 2013 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> @@ -28,35 +28,43 @@ from django.core.exceptions import ObjectDoesNotExist from django.template.defaultfilters import slugify from django.contrib.auth.models import User -from ishtar_common.models import Town, Person, PersonType, OrganizationType, \ - Organization, SourceType +from ishtar_common.models import ( + Town, + Person, + PersonType, + OrganizationType, + Organization, + SourceType, +) from archaeological_files.models import PermitType -from archaeological_operations.models import OperationType, Period, \ - ActType +from archaeological_operations.models import OperationType, Period, ActType def get_default_person(): - return User.objects.order_by('pk').all()[0] + return User.objects.order_by("pk").all()[0] def _get_parse_string(trunc_number=None): def parse_string(value): value = value.strip() - if value == '#EMPTY': - value = '' - value = value.replace(' ', ' ') + if value == "#EMPTY": + value = "" + value = value.replace(" ", " ") if trunc_number: value = value[:trunc_number] return value + return parse_string + parse_string = _get_parse_string() def parse_multivalue(value): - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', value) - s1 = re.sub('([a-z0-9])([A-Z])', r'\1 \2', s1) - return re.sub('([0-9])([a-z])', r'\1 \2', s1) + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1 \2", value) + s1 = re.sub("([a-z0-9])([A-Z])", r"\1 \2", s1) + return re.sub("([0-9])([a-z])", r"\1 \2", s1) + ope_types = {} @@ -65,8 +73,11 @@ def _init_ope_types(): for k in settings.ISHTAR_OPE_TYPES.keys(): ot, created = OperationType.objects.get_or_create( txt_idx=settings.ISHTAR_OPE_TYPES[k][0], - defaults={'label': settings.ISHTAR_OPE_TYPES[k][1], - 'preventive': k[0] == 'préventive'}) + defaults={ + "label": settings.ISHTAR_OPE_TYPES[k][1], + "preventive": k[0] == "préventive", + }, + ) ope_types[k] = ot @@ -78,6 +89,7 @@ def parse_operationtype(value, preventive, owner): return None return ope_types[value] + periods = {} periods_keys = [] @@ -102,25 +114,26 @@ def _init_period(): def parse_period(value): value = parse_string(value) - value = value[3:] if value.startswith('EUR') else value - while value.endswith('-'): + value = value[3:] if value.startswith("EUR") else value + while value.endswith("-"): value = value[:-1] - value = value[3:] if value.startswith('EUR') else value + value = value[3:] if value.startswith("EUR") else value if not periods: _init_period() if not value: - return [periods['']] - period, old_val = [], '' + return [periods[""]] + period, old_val = [], "" while value and old_val != value: old_val = value for k in periods_keys: if value.startswith(k): period.append(periods[k]) - value = value[len(k):] + value = value[len(k) :] break return period -_REPLACED_PERIOD = [('deuxieme', 'second')] + +_REPLACED_PERIOD = [("deuxieme", "second")] _REPLACED_PERIOD += [(y, x) for x, y in _REPLACED_PERIOD] REPLACED_PERIOD_DCT = dict(_REPLACED_PERIOD) @@ -133,18 +146,19 @@ def parse_period_name(value): _init_period() value = parse_string(value) if not value: - return [period_names['']] - period, old_val = [], '' + return [period_names[""]] + period, old_val = [], "" value = slugify(value) while value and old_val != value: old_val = value for k in period_names_keys: if value.startswith(k): period.append(period_names[k]) - value = value[len(k):] + value = value[len(k) :] break return period + _CACHED_PERMIT_TYPES = {} @@ -152,8 +166,8 @@ def _init_permit_type(): for k in settings.ISHTAR_PERMIT_TYPES: txt_idx, label = settings.ISHTAR_PERMIT_TYPES[k] permit_type, created = PermitType.objects.get_or_create( - txt_idx=txt_idx, defaults={'label': label, - 'available': True}) + txt_idx=txt_idx, defaults={"label": label, "available": True} + ) _CACHED_PERMIT_TYPES[k] = permit_type @@ -167,6 +181,7 @@ def parse_permittype(value): value = "" return _CACHED_PERMIT_TYPES[value] + _CACHED_ADMIN_ACT_TYPES = {} @@ -177,15 +192,16 @@ def parse_admin_act_typ(value, code, owner): return if code not in _CACHED_ADMIN_ACT_TYPES: act_type, created = ActType.objects.get_or_create( - txt_idx=code, defaults={'label': value}) + txt_idx=code, defaults={"label": value} + ) _CACHED_ADMIN_ACT_TYPES[code] = act_type return _CACHED_ADMIN_ACT_TYPES[code] def parse_fileref(value): - value = parse_string(value).split('/')[0] - value = value.split('.')[0] - match = re.search('[0-9].[0-9]*', value) + value = parse_string(value).split("/")[0] + value = value.split(".")[0] + match = re.search("[0-9].[0-9]*", value) if not match: return None return int(match.group()) @@ -203,18 +219,18 @@ def parse_orga(value, alternate_value, owner): try: organization_type = OrganizationType.objects.get(label__iexact=value) except ObjectDoesNotExist: - organization_type = OrganizationType.objects.get(txt_idx='undefined') - orga = Organization.objects.create(name=value, - organization_type=organization_type, - history_modifier=owner) + organization_type = OrganizationType.objects.get(txt_idx="undefined") + orga = Organization.objects.create( + name=value, organization_type=organization_type, history_modifier=owner + ) return orga def parse_bool(value): value = parse_string(value) - if value.lower() in ('yes', 'oui'): + if value.lower() in ("yes", "oui"): value = True - elif value.lower() in ('no', 'non'): + elif value.lower() in ("no", "non"): value = False else: value = None @@ -222,16 +238,16 @@ def parse_bool(value): def parse_date(value): - value = parse_string(value).split(' ')[0] + value = parse_string(value).split(" ")[0] try: - return datetime.datetime.strptime(value, '%d/%m/%Y') + return datetime.datetime.strptime(value, "%d/%m/%Y") except: return None def parse_yearref(value): - value = parse_string(value).split('.')[0] - match = re.search('[0-9].[0-9]*', value) + value = parse_string(value).split(".")[0] + match = re.search("[0-9].[0-9]*", value) if not match: return None return int(match.group()) @@ -239,7 +255,7 @@ def parse_yearref(value): def parse_surface(value): value = parse_string(value) - value = value.replace(',', '.') + value = value.replace(",", ".") try: # hectare en metre carrés value = float(value) * 10000 @@ -265,17 +281,17 @@ def parse_trunc_patriarche(value): value = parse_string(value) if not value: return - value = value.replace(' ', '') + value = value.replace(" ", "") try: int(value) except ValueError: return - return '18' + str(value) + return "18" + str(value) def parse_operation_code(value): value = parse_string(value) - code = value.split('.')[-1] + code = value.split(".")[-1] try: return int(code) except: @@ -291,44 +307,38 @@ def parse_title(value): def parse_name_surname(value, owner): value = parse_string(value) - items = value.split(' ') + items = value.split(" ") name = items[0] surname = "" if len(items) > 1: name = " ".join(items[:-1]) surname = items[-1] - values = {"surname": parse_title(surname)[:30], - "name": parse_title(name)[:30]} - if not values['surname'] and not values['name']: + values = {"surname": parse_title(surname)[:30], "name": parse_title(name)[:30]} + if not values["surname"] and not values["name"]: return q = Person.objects.filter(**values) if q.count(): return q.all()[0] else: - defaults = {'history_modifier': owner, - 'title': ''} + defaults = {"history_modifier": owner, "title": ""} defaults.update(values) p = Person.objects.create(**defaults) - p.person_types.add(PersonType.objects.get( - txt_idx='head_scientist')) + p.person_types.add(PersonType.objects.get(txt_idx="head_scientist")) return p def parse_person(surname, name, old_ref, owner): - values = {"surname": parse_title(surname), - "name": parse_title(name)} - if not values['surname'] and not values['name']: + values = {"surname": parse_title(surname), "name": parse_title(name)} + if not values["surname"] and not values["name"]: return q = Person.objects.filter(**values) if q.count(): return q.all()[0] else: - defaults = {'history_modifier': owner, - 'title': ''} + defaults = {"history_modifier": owner, "title": ""} defaults.update(values) p = Person.objects.create(**defaults) - p.person_types.add(PersonType.objects.get( - txt_idx='head_scientist')) + p.person_types.add(PersonType.objects.get(txt_idx="head_scientist")) return p @@ -344,48 +354,38 @@ def parse_comment_addr_nature(nature, addr, owner): return "" return "\n".join(comments) + # si pas de start date : premier janvier de year ope_types = { - 'AET': ('other_study', - 'Autre étude', True), - 'APP': ('assistance_preparation_help', - 'Aide à la préparation de publication', True), - 'DOC': ('documents_study', - 'Étude documentaire', True), - 'EV': ('evaluation', - "Fouille d'évaluation", True), - 'FOU': ('ancient_excavation', - "Fouille ancienne", True), - 'FP': ('prog_excavation', - "Fouille programmée", False), - 'MH': ('building_study', "Fouille avant MH", True), - 'OPD': ('arch_diagnostic', - "Diagnostic archéologique", True), - 'PAN': ('analysis_program', - "Programme d'analyses", False), - 'PCR': ('collective_research_project', - "Projet collectif de recherche", False), - 'PMS': ('specialized_eqp_prospection', - "Prospection avec matériel spécialisé", False), - 'PRD': ('diachronic_prospection', - "Prospection diachronique", False), - 'PI': ('diachronic_prospection', - "Prospection diachronique", False), - 'PRM': ('metal_detector_prospection', - "Prospection détecteur de métaux", False), - 'PRT': ('thematic_prospection', - "Prospection thématique", False), - 'PT': ('thematic_prospection', - "Prospection thématique", False), - 'RAR': ('cave_art_record', - "Relevé d'art rupestre", False), - 'SD': ('sampling_research', - "Sondage", False), - 'SP': ('prev_excavation', - "Fouille préventive", True), - 'SU': ('emergency_excavation', - "Fouille préventive d'urgence", True), + "AET": ("other_study", "Autre étude", True), + "APP": ( + "assistance_preparation_help", + "Aide à la préparation de publication", + True, + ), + "DOC": ("documents_study", "Étude documentaire", True), + "EV": ("evaluation", "Fouille d'évaluation", True), + "FOU": ("ancient_excavation", "Fouille ancienne", True), + "FP": ("prog_excavation", "Fouille programmée", False), + "MH": ("building_study", "Fouille avant MH", True), + "OPD": ("arch_diagnostic", "Diagnostic archéologique", True), + "PAN": ("analysis_program", "Programme d'analyses", False), + "PCR": ("collective_research_project", "Projet collectif de recherche", False), + "PMS": ( + "specialized_eqp_prospection", + "Prospection avec matériel spécialisé", + False, + ), + "PRD": ("diachronic_prospection", "Prospection diachronique", False), + "PI": ("diachronic_prospection", "Prospection diachronique", False), + "PRM": ("metal_detector_prospection", "Prospection détecteur de métaux", False), + "PRT": ("thematic_prospection", "Prospection thématique", False), + "PT": ("thematic_prospection", "Prospection thématique", False), + "RAR": ("cave_art_record", "Relevé d'art rupestre", False), + "SD": ("sampling_research", "Sondage", False), + "SP": ("prev_excavation", "Fouille préventive", True), + "SU": ("emergency_excavation", "Fouille préventive d'urgence", True), } _CACHED_OPE_TYPES = {} @@ -395,8 +395,8 @@ def _prepare_ope_types(): for k in ope_types.keys(): txt_idx, label, preventive = ope_types[k] ot, created = OperationType.objects.get_or_create( - txt_idx=txt_idx, defaults={'label': label, - 'preventive': preventive}) + txt_idx=txt_idx, defaults={"label": label, "preventive": preventive} + ) if k not in _CACHED_OPE_TYPES.keys(): _CACHED_OPE_TYPES[k] = ot @@ -406,16 +406,17 @@ def parse_patriarche_operationtype(value): return None return _CACHED_OPE_TYPES[value] -_dpt_re_filter = re.compile('^\([0-9]*\) ') + +_dpt_re_filter = re.compile("^\([0-9]*\) ") def parse_ope_name(value): if not value: - return '' + return "" value = value.strip() - if value.lower() == 'null': - return '' - value = _dpt_re_filter.sub('', value) + if value.lower() == "null": + return "" + value = _dpt_re_filter.sub("", value) return value @@ -430,10 +431,11 @@ def parse_ha(value): def parse_rapp_index(value): value = parse_string(value) - items = re.findall(r'[0-9]+$', value) + items = re.findall(r"[0-9]+$", value) if items: return int(items[-1]) + _CACHED_DOC_TYPES = {} @@ -443,8 +445,8 @@ def parse_doc_types(value): if value not in settings.ISHTAR_DOC_TYPES: return _CACHED_DOC_TYPES[value], created = SourceType.objects.get_or_create( - txt_idx=value, - defaults={"label": settings.ISHTAR_DOC_TYPES[value]}) + txt_idx=value, defaults={"label": settings.ISHTAR_DOC_TYPES[value]} + ) return _CACHED_DOC_TYPES[value] @@ -469,10 +471,10 @@ def parse_insee(value): PARCEL_YEAR_REGEXP = re.compile(r"^([0-9]{4})[ :]+") PARCEL_SECTION_REGEXP = re.compile( r"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*" - r"((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*[0-9]+[p]?)+)") -PARCEL_NB_RANGE_REGEXP = re.compile(r'([0-9]+[p]?) (?:à|to) ([0-9]+[p]?)') -PARCEL_NB_REGEXP = re.compile( - r'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*([0-9]+[p]?)') + r"((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*[0-9]+[p]?)+)" +) +PARCEL_NB_RANGE_REGEXP = re.compile(r"([0-9]+[p]?) (?:à|to) ([0-9]+[p]?)") +PARCEL_NB_REGEXP = re.compile(r"(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à|to)*([0-9]+[p]?)") def parse_parcels(parcel_str, insee_code=None, owner=None): @@ -488,17 +490,17 @@ def parse_parcels(parcel_str, insee_code=None, owner=None): year = None if m: year = m.groups()[0] - parcel_str = parcel_str[m.span()[1]:] + parcel_str = parcel_str[m.span()[1] :] for parcel in PARCEL_SECTION_REGEXP.findall(parcel_str): sector, nums = parcel[0], parcel[1] for num in PARCEL_NB_REGEXP.findall(nums): if len(str(num)) > 6: continue - dct = {'year': year, 'section': sector, 'parcel_number': num} + dct = {"year": year, "section": sector, "parcel_number": num} if town: - dct['town'] = town + dct["town"] = town if owner: - dct['history_modifier'] = owner + dct["history_modifier"] = owner parcels.append(dct) for parcel_ranges in PARCEL_NB_RANGE_REGEXP.findall(nums): lower_range, higher_range = parcel_ranges @@ -509,11 +511,10 @@ def parse_parcels(parcel_str, insee_code=None, owner=None): except ValueError: continue for num in range(lower_range, higher_range): - dct = {'year': year, 'section': sector, - 'parcel_number': str(num)} + dct = {"year": year, "section": sector, "parcel_number": str(num)} if town: - dct['town'] = town + dct["town"] = town if owner: - dct['history_modifier'] = owner + dct["history_modifier"] = owner parcels.append(dct) return parcels |