diff options
Diffstat (limited to 'archaeological_operations/import_from_csv.py')
| -rw-r--r-- | archaeological_operations/import_from_csv.py | 146 |
1 files changed, 112 insertions, 34 deletions
diff --git a/archaeological_operations/import_from_csv.py b/archaeological_operations/import_from_csv.py index 9d3a58ffd..d00327cef 100644 --- a/archaeological_operations/import_from_csv.py +++ b/archaeological_operations/import_from_csv.py @@ -36,7 +36,7 @@ from ishtar_common.models import Town, Person, PersonType, OrganizationType, \ Organization, SourceType from archaeological_files.models import PermitType, File, FileType from archaeological_operations.models import Operation, OperationType, Period, \ - AdministrativeAct, ActType, OperationSource + AdministrativeAct, ActType, OperationSource, Parcel DEFAULT_PERSON = User.objects.order_by('pk').all()[0] @@ -410,6 +410,44 @@ def parse_rapp_index(value): if items: return int(items[-1]) +PARCEL_YEAR_REGEXP = re.compile(r"^([0-9]{4})[ :]+") +PARCEL_SECTION_REGEXP = re.compile(ur"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*[0-9]+[p]?)+)") +PARCEL_NB_RANGE_REGEXP = re.compile(ur'([0-9]+[p]?) à ([0-9]+[p]?)') +PARCEL_NB_REGEXP = re.compile(ur'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*([0-9]+[p]?)') + +def parse_parcels(parcel_str, insee_code, owner): + parcels = [] + town = parse_insee(insee_code) + # manage only one town at a time + assert len(town) < 2 + if not town: + return parcels + town = town[0] + m = PARCEL_YEAR_REGEXP.match(parcel_str) + year = None + if m: + year = m.groups()[0] + parcel_str = parcel_str[m.span()[1]:] + for parcel in PARCEL_SECTION_REGEXP.findall(parcel_str): + sector, nums = parcel[0], parcel[1] + print PARCEL_NB_REGEXP.findall(nums) + for num in PARCEL_NB_REGEXP.findall(nums): + parcels.append({'year':year, 'town':town, 'section':sector, + 'parcel_number':num, 'history_modifier':owner}) + for parcel_ranges in PARCEL_NB_RANGE_REGEXP.findall(nums): + lower_range, higher_range = parcel_ranges + try: + # the lower range itself has been already kept + lower_range = int(lower_range) + 1 + higher_range = int(higher_range) + except ValueError: + continue + for num in xrange(lower_range, higher_range): + parcels.append({'year':year, 'town':town, + 'section':sector, 'parcel_number':unicode(num), + 'history_modifier':owner}) + return parcels + _CACHED_DOC_TYPES = {} def parse_doc_types(value): @@ -462,7 +500,6 @@ for cols in _OPE_COLS: else: OPE_COLS.append(None) - def ope_postimportfix(ope, dct): changed = False if not ope.year: @@ -478,10 +515,15 @@ def ope_postimportfix(ope, dct): ope.save() return ope +class BreakIt(Exception): + pass + class RelatedClass: - def __init__(self, key, cls, default_data={}, reverse_key=False, unique_keys=[]): + def __init__(self, key, cls, default_data={}, reverse_key=False, + unique_keys=[], extra_data=[], multi=None): self.key, self.cls, self.default_data = key, cls, default_data self.reverse_key, self.unique_keys = reverse_key, unique_keys + self.extra_data, self.multi = extra_data, multi def create_object(self, data): if self.unique_keys: @@ -490,25 +532,39 @@ class RelatedClass: unique_data[k] = data.pop(k) unique_data['defaults'] = data obj, created = self.cls.objects.get_or_create(**unique_data) + if not created: + for k in unique_data['defaults']: + setattr(obj, k, unique_data['defaults'][k]) + obj.save() else: obj = self.cls.objects.create(**data) return obj def create(self, item, data, attr=None): - if self.reverse_key: - data[self.reverse_key] = item - obj = self.create_object(data) - else: - obj = getattr(item, attr) - if not obj: + datas = data + if not self.multi: + datas = [data] + objs = [] + for data in datas: + if self.reverse_key: + data[self.reverse_key] = item + if self.reverse_key not in self.unique_keys: + self.unique_keys.append(self.reverse_key) obj = self.create_object(data) - setattr(item, attr, obj) - item.save() else: - for k in data: - setattr(obj, k, data[k]) - obj.save() - return obj + obj = getattr(item, attr) + if not obj: + obj = self.create_object(data) + setattr(item, attr, obj) + item.save() + else: + for k in data: + setattr(obj, k, data[k]) + obj.save() + objs.append(obj) + if not self.multi: + return objs[0] + return objs #@transaction.commit_manually def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, @@ -523,11 +579,17 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, reverse_key='operation', unique_keys=['ref_sra']), RelatedClass('associated_file', File, + extra_data=['year'], default_data={'history_modifier':default_person, 'file_type':FileType.objects.get( - txt_idx='undefined')}), + txt_idx='undefined')}, + unique_keys=['internal_reference']), RelatedClass('source', OperationSource, reverse_key='operation', unique_keys=['index']), + RelatedClass('parcels', Parcel, reverse_key='operation', + unique_keys=['operation', 'town', 'section', + 'parcel_number'], + multi=True), ] RELATED_CLASSES_KEYS = dict([(rel_cls.key, rel_cls) for rel_cls in RELATED_CLASSES]) @@ -575,11 +637,11 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, if not extra_cols: v = typ(val) else: - arguments = [vals[col_number] for col_number in extra_cols] - if not [arg for arg in arguments if arg]: - continue - arguments += [default_person] - v = typ(val, *arguments) + arguments = [vals[col_number] for col_number in extra_cols] + if not [arg for arg in arguments if arg]: + continue + arguments += [default_person] + v = typ(val, *arguments) except: v = None if len(attrs) == 1: @@ -605,19 +667,35 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, multis = [] attached_models = {} for k in args.keys(): - if k in related_classes: - rel_cls = related_classes[k] - cls, default = rel_cls.cls, rel_cls.default_data - reverse_key = rel_cls.reverse_key - default.update(args[k]) - args.pop(k) - related_items.append((rel_cls, default.copy(), k)) - elif type(args[k]) == list or k in multi_keys: - multis.append((k, args[k])) - args.pop(k) - elif '__' in k: - mod, value = k.split('__') - attached_models[(mod, value)] = args.pop(k) + try: + if k in related_classes: + rel_cls = related_classes[k] + cls, default = rel_cls.cls, rel_cls.default_data + reverse_key = rel_cls.reverse_key + values = None + if rel_cls.multi: + values = [] + for v in args[k]: + v.update(default) + values.append(v) + else: + values = default.copy() + values.update(args[k]) + exited = False + for extra in rel_cls.extra_data: + if not args.get(extra): + raise BreakIt + values[extra] = args[extra] + args.pop(k) + related_items.append((rel_cls, values, k)) + elif k in multi_keys: + multis.append((k, args[k])) + args.pop(k) + elif '__' in k: + mod, value = k.split('__') + attached_models[(mod, value)] = args.pop(k) + except BreakIt: + continue op = None if not update and not args.get('operation_type'): #print "Pas d'operation_type" |