diff options
Diffstat (limited to 'archaeological_operations/import_from_csv.py')
| -rw-r--r-- | archaeological_operations/import_from_csv.py | 150 |
1 files changed, 95 insertions, 55 deletions
diff --git a/archaeological_operations/import_from_csv.py b/archaeological_operations/import_from_csv.py index e61596b00..d00327cef 100644 --- a/archaeological_operations/import_from_csv.py +++ b/archaeological_operations/import_from_csv.py @@ -36,7 +36,7 @@ from ishtar_common.models import Town, Person, PersonType, OrganizationType, \ Organization, SourceType from archaeological_files.models import PermitType, File, FileType from archaeological_operations.models import Operation, OperationType, Period, \ - AdministrativeAct, ActType, OperationSource + AdministrativeAct, ActType, OperationSource, Parcel DEFAULT_PERSON = User.objects.order_by('pk').all()[0] @@ -411,11 +411,11 @@ def parse_rapp_index(value): return int(items[-1]) PARCEL_YEAR_REGEXP = re.compile(r"^([0-9]{4})[ :]+") +PARCEL_SECTION_REGEXP = re.compile(ur"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*[0-9]+[p]?)+)") +PARCEL_NB_RANGE_REGEXP = re.compile(ur'([0-9]+[p]?) à ([0-9]+[p]?)') +PARCEL_NB_REGEXP = re.compile(ur'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*([0-9]+[p]?)') -PARCEL_SECTION_REGEXP = re.compile(r"([A-Z0-9]{1,3})[ :]+ *(([0-9]+[a-z]?[ ,à]*[et]*)+)") -PARCEL_NB_REGEXP = re.compile(r'([0-9]+[a-z]?)[ ,à]*[et]*') - -def parse_parcels(insee_code, parcel_str, owner): +def parse_parcels(parcel_str, insee_code, owner): parcels = [] town = parse_insee(insee_code) # manage only one town at a time @@ -423,28 +423,29 @@ def parse_parcels(insee_code, parcel_str, owner): if not town: return parcels town = town[0] - parcel_values = {} m = PARCEL_YEAR_REGEXP.match(parcel_str) year = None if m: year = m.groups()[0] parcel_str = parcel_str[m.span()[1]:] - parcel_values[year] = [] - parcels_str = [parcel_str] - if ';' in parcel_str: - parcels_str = parcel_str.split(';') - if '/' in parcel_str: - parcels_str = parcel_str.split('/') - for parcel_str in parcels_str: - m = PARCEL_SECTION_REGEXP.match(parcel_str) - while m: - gp = m.groups() - sector, nums = gp[0], gp[1] - for num in PARCEL_NB_REGEXP.findall(nums): - parcel_values[year].append((sector, num)) - parcel_str = parcel_str[m.span()[1]:] - m = PARCEL_SECTION_REGEXP.match(parcel_str) - print parcel_values + for parcel in PARCEL_SECTION_REGEXP.findall(parcel_str): + sector, nums = parcel[0], parcel[1] + print PARCEL_NB_REGEXP.findall(nums) + for num in PARCEL_NB_REGEXP.findall(nums): + parcels.append({'year':year, 'town':town, 'section':sector, + 'parcel_number':num, 'history_modifier':owner}) + for parcel_ranges in PARCEL_NB_RANGE_REGEXP.findall(nums): + lower_range, higher_range = parcel_ranges + try: + # the lower range itself has been already kept + lower_range = int(lower_range) + 1 + higher_range = int(higher_range) + except ValueError: + continue + for num in xrange(lower_range, higher_range): + parcels.append({'year':year, 'town':town, + 'section':sector, 'parcel_number':unicode(num), + 'history_modifier':owner}) return parcels _CACHED_DOC_TYPES = {} @@ -499,7 +500,6 @@ for cols in _OPE_COLS: else: OPE_COLS.append(None) - def ope_postimportfix(ope, dct): changed = False if not ope.year: @@ -515,11 +515,15 @@ def ope_postimportfix(ope, dct): ope.save() return ope +class BreakIt(Exception): + pass + class RelatedClass: def __init__(self, key, cls, default_data={}, reverse_key=False, - unique_keys=[]): + unique_keys=[], extra_data=[], multi=None): self.key, self.cls, self.default_data = key, cls, default_data self.reverse_key, self.unique_keys = reverse_key, unique_keys + self.extra_data, self.multi = extra_data, multi def create_object(self, data): if self.unique_keys: @@ -528,25 +532,39 @@ class RelatedClass: unique_data[k] = data.pop(k) unique_data['defaults'] = data obj, created = self.cls.objects.get_or_create(**unique_data) + if not created: + for k in unique_data['defaults']: + setattr(obj, k, unique_data['defaults'][k]) + obj.save() else: obj = self.cls.objects.create(**data) return obj def create(self, item, data, attr=None): - if self.reverse_key: - data[self.reverse_key] = item - obj = self.create_object(data) - else: - obj = getattr(item, attr) - if not obj: + datas = data + if not self.multi: + datas = [data] + objs = [] + for data in datas: + if self.reverse_key: + data[self.reverse_key] = item + if self.reverse_key not in self.unique_keys: + self.unique_keys.append(self.reverse_key) obj = self.create_object(data) - setattr(item, attr, obj) - item.save() else: - for k in data: - setattr(obj, k, data[k]) - obj.save() - return obj + obj = getattr(item, attr) + if not obj: + obj = self.create_object(data) + setattr(item, attr, obj) + item.save() + else: + for k in data: + setattr(obj, k, data[k]) + obj.save() + objs.append(obj) + if not self.multi: + return objs[0] + return objs #@transaction.commit_manually def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, @@ -561,11 +579,17 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, reverse_key='operation', unique_keys=['ref_sra']), RelatedClass('associated_file', File, + extra_data=['year'], default_data={'history_modifier':default_person, 'file_type':FileType.objects.get( - txt_idx='undefined')}), + txt_idx='undefined')}, + unique_keys=['internal_reference']), RelatedClass('source', OperationSource, reverse_key='operation', unique_keys=['index']), + RelatedClass('parcels', Parcel, reverse_key='operation', + unique_keys=['operation', 'town', 'section', + 'parcel_number'], + multi=True), ] RELATED_CLASSES_KEYS = dict([(rel_cls.key, rel_cls) for rel_cls in RELATED_CLASSES]) @@ -613,11 +637,11 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, if not extra_cols: v = typ(val) else: - arguments = [vals[col_number] for col_number in extra_cols] - if not [arg for arg in arguments if arg]: - continue - arguments += [default_person] - v = typ(val, *arguments) + arguments = [vals[col_number] for col_number in extra_cols] + if not [arg for arg in arguments if arg]: + continue + arguments += [default_person] + v = typ(val, *arguments) except: v = None if len(attrs) == 1: @@ -643,19 +667,35 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, multis = [] attached_models = {} for k in args.keys(): - if k in related_classes: - rel_cls = related_classes[k] - cls, default = rel_cls.cls, rel_cls.default_data - reverse_key = rel_cls.reverse_key - default.update(args[k]) - args.pop(k) - related_items.append((rel_cls, default.copy(), k)) - elif type(args[k]) == list or k in multi_keys: - multis.append((k, args[k])) - args.pop(k) - elif '__' in k: - mod, value = k.split('__') - attached_models[(mod, value)] = args.pop(k) + try: + if k in related_classes: + rel_cls = related_classes[k] + cls, default = rel_cls.cls, rel_cls.default_data + reverse_key = rel_cls.reverse_key + values = None + if rel_cls.multi: + values = [] + for v in args[k]: + v.update(default) + values.append(v) + else: + values = default.copy() + values.update(args[k]) + exited = False + for extra in rel_cls.extra_data: + if not args.get(extra): + raise BreakIt + values[extra] = args[extra] + args.pop(k) + related_items.append((rel_cls, values, k)) + elif k in multi_keys: + multis.append((k, args[k])) + args.pop(k) + elif '__' in k: + mod, value = k.split('__') + attached_models[(mod, value)] = args.pop(k) + except BreakIt: + continue op = None if not update and not args.get('operation_type'): #print "Pas d'operation_type" |