diff options
| -rw-r--r-- | archaeological_operations/import_from_csv.py | 146 | ||||
| -rw-r--r-- | archaeological_operations/models.py | 15 | ||||
| -rw-r--r-- | archaeological_operations/tests.py | 247 | ||||
| -rw-r--r-- | example_project/local_settings_nantes.py | 4 | ||||
| -rw-r--r-- | example_project/settings.py | 3 |
5 files changed, 376 insertions, 39 deletions
diff --git a/archaeological_operations/import_from_csv.py b/archaeological_operations/import_from_csv.py index 9d3a58ffd..d00327cef 100644 --- a/archaeological_operations/import_from_csv.py +++ b/archaeological_operations/import_from_csv.py @@ -36,7 +36,7 @@ from ishtar_common.models import Town, Person, PersonType, OrganizationType, \ Organization, SourceType from archaeological_files.models import PermitType, File, FileType from archaeological_operations.models import Operation, OperationType, Period, \ - AdministrativeAct, ActType, OperationSource + AdministrativeAct, ActType, OperationSource, Parcel DEFAULT_PERSON = User.objects.order_by('pk').all()[0] @@ -410,6 +410,44 @@ def parse_rapp_index(value): if items: return int(items[-1]) +PARCEL_YEAR_REGEXP = re.compile(r"^([0-9]{4})[ :]+") +PARCEL_SECTION_REGEXP = re.compile(ur"(?: )*(?:[Ss]ection(?:s)?)?(?: )*([A-Z][A-Z0-9]{0,3})[ :]*((?:(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*[0-9]+[p]?)+)") +PARCEL_NB_RANGE_REGEXP = re.compile(ur'([0-9]+[p]?) à ([0-9]+[p]?)') +PARCEL_NB_REGEXP = re.compile(ur'(?: |;|,|[Pp]arcelle(?:s)?|n°|et|à)*([0-9]+[p]?)') + +def parse_parcels(parcel_str, insee_code, owner): + parcels = [] + town = parse_insee(insee_code) + # manage only one town at a time + assert len(town) < 2 + if not town: + return parcels + town = town[0] + m = PARCEL_YEAR_REGEXP.match(parcel_str) + year = None + if m: + year = m.groups()[0] + parcel_str = parcel_str[m.span()[1]:] + for parcel in PARCEL_SECTION_REGEXP.findall(parcel_str): + sector, nums = parcel[0], parcel[1] + print PARCEL_NB_REGEXP.findall(nums) + for num in PARCEL_NB_REGEXP.findall(nums): + parcels.append({'year':year, 'town':town, 'section':sector, + 'parcel_number':num, 'history_modifier':owner}) + for parcel_ranges in PARCEL_NB_RANGE_REGEXP.findall(nums): + lower_range, higher_range = parcel_ranges + try: + # the lower range itself has been already kept + lower_range = int(lower_range) + 1 + higher_range = int(higher_range) + except ValueError: + continue + for num in xrange(lower_range, higher_range): + parcels.append({'year':year, 'town':town, + 'section':sector, 'parcel_number':unicode(num), + 'history_modifier':owner}) + return parcels + _CACHED_DOC_TYPES = {} def parse_doc_types(value): @@ -462,7 +500,6 @@ for cols in _OPE_COLS: else: OPE_COLS.append(None) - def ope_postimportfix(ope, dct): changed = False if not ope.year: @@ -478,10 +515,15 @@ def ope_postimportfix(ope, dct): ope.save() return ope +class BreakIt(Exception): + pass + class RelatedClass: - def __init__(self, key, cls, default_data={}, reverse_key=False, unique_keys=[]): + def __init__(self, key, cls, default_data={}, reverse_key=False, + unique_keys=[], extra_data=[], multi=None): self.key, self.cls, self.default_data = key, cls, default_data self.reverse_key, self.unique_keys = reverse_key, unique_keys + self.extra_data, self.multi = extra_data, multi def create_object(self, data): if self.unique_keys: @@ -490,25 +532,39 @@ class RelatedClass: unique_data[k] = data.pop(k) unique_data['defaults'] = data obj, created = self.cls.objects.get_or_create(**unique_data) + if not created: + for k in unique_data['defaults']: + setattr(obj, k, unique_data['defaults'][k]) + obj.save() else: obj = self.cls.objects.create(**data) return obj def create(self, item, data, attr=None): - if self.reverse_key: - data[self.reverse_key] = item - obj = self.create_object(data) - else: - obj = getattr(item, attr) - if not obj: + datas = data + if not self.multi: + datas = [data] + objs = [] + for data in datas: + if self.reverse_key: + data[self.reverse_key] = item + if self.reverse_key not in self.unique_keys: + self.unique_keys.append(self.reverse_key) obj = self.create_object(data) - setattr(item, attr, obj) - item.save() else: - for k in data: - setattr(obj, k, data[k]) - obj.save() - return obj + obj = getattr(item, attr) + if not obj: + obj = self.create_object(data) + setattr(item, attr, obj) + item.save() + else: + for k in data: + setattr(obj, k, data[k]) + obj.save() + objs.append(obj) + if not self.multi: + return objs[0] + return objs #@transaction.commit_manually def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, @@ -523,11 +579,17 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, reverse_key='operation', unique_keys=['ref_sra']), RelatedClass('associated_file', File, + extra_data=['year'], default_data={'history_modifier':default_person, 'file_type':FileType.objects.get( - txt_idx='undefined')}), + txt_idx='undefined')}, + unique_keys=['internal_reference']), RelatedClass('source', OperationSource, reverse_key='operation', unique_keys=['index']), + RelatedClass('parcels', Parcel, reverse_key='operation', + unique_keys=['operation', 'town', 'section', + 'parcel_number'], + multi=True), ] RELATED_CLASSES_KEYS = dict([(rel_cls.key, rel_cls) for rel_cls in RELATED_CLASSES]) @@ -575,11 +637,11 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, if not extra_cols: v = typ(val) else: - arguments = [vals[col_number] for col_number in extra_cols] - if not [arg for arg in arguments if arg]: - continue - arguments += [default_person] - v = typ(val, *arguments) + arguments = [vals[col_number] for col_number in extra_cols] + if not [arg for arg in arguments if arg]: + continue + arguments += [default_person] + v = typ(val, *arguments) except: v = None if len(attrs) == 1: @@ -605,19 +667,35 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None, multis = [] attached_models = {} for k in args.keys(): - if k in related_classes: - rel_cls = related_classes[k] - cls, default = rel_cls.cls, rel_cls.default_data - reverse_key = rel_cls.reverse_key - default.update(args[k]) - args.pop(k) - related_items.append((rel_cls, default.copy(), k)) - elif type(args[k]) == list or k in multi_keys: - multis.append((k, args[k])) - args.pop(k) - elif '__' in k: - mod, value = k.split('__') - attached_models[(mod, value)] = args.pop(k) + try: + if k in related_classes: + rel_cls = related_classes[k] + cls, default = rel_cls.cls, rel_cls.default_data + reverse_key = rel_cls.reverse_key + values = None + if rel_cls.multi: + values = [] + for v in args[k]: + v.update(default) + values.append(v) + else: + values = default.copy() + values.update(args[k]) + exited = False + for extra in rel_cls.extra_data: + if not args.get(extra): + raise BreakIt + values[extra] = args[extra] + args.pop(k) + related_items.append((rel_cls, values, k)) + elif k in multi_keys: + multis.append((k, args[k])) + args.pop(k) + elif '__' in k: + mod, value = k.split('__') + attached_models[(mod, value)] = args.pop(k) + except BreakIt: + continue op = None if not update and not args.get('operation_type'): #print "Pas d'operation_type" diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py index 60ee06a9c..9826fbfad 100644 --- a/archaeological_operations/models.py +++ b/archaeological_operations/models.py @@ -282,7 +282,8 @@ class AdministrativeAct(BaseHistorizedItem, OwnPerms): operator = models.ForeignKey(Organization, blank=True, null=True, verbose_name=_(u"Archaeological preventive operator")) scientific = models.ForeignKey(Person, blank=True, null=True, -related_name='+', verbose_name=_(u"Person in charge of the scientific part")) + related_name='+', + verbose_name=_(u"Person in charge of the scientific part")) signatory = models.ForeignKey(Person, blank=True, null=True, related_name='+', verbose_name=_(u"Signatory")) operation = models.ForeignKey(Operation, blank=True, null=True, @@ -362,6 +363,18 @@ class Parcel(LightHistorizedItem): if item] return settings.JOINT.join(items) +def parcel_post_save(sender, **kwargs): + if not kwargs['instance'] or not FILES_AVAILABLE: + return + parcel = kwargs['instance'] + if parcel.operation and parcel.associated_file: + return + if parcel.operation and parcel.operation.associated_file: + parcel.associated_file = parcel.operation.associated_file + parcel.save() + return +post_save.connect(parcel_post_save, sender=Parcel) + class ParcelOwner(LightHistorizedItem): owner = models.ForeignKey(Person, verbose_name=_(u"Owner")) parcel = models.ForeignKey(Parcel, verbose_name=_(u"Parcel")) diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 61e7695f9..983165968 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -28,7 +28,7 @@ from django.test import TestCase from django.contrib.auth.models import User import models -from ishtar_common.models import OrganizationType, Organization +from ishtar_common.models import OrganizationType, Organization, Town class ImportOperationTest(TestCase): fixtures = ['../ishtar_common/fixtures/initial_data.json', @@ -51,3 +51,248 @@ class ImportOperationTest(TestCase): """ call_command('import_operations', os.sep.join([os.getcwd(), '..', 'archaeological_operations', 'tests', 'sample.csv'])) + + def testParseParcels(self): + # the database needs to be initialised before importing + from archaeological_operations.import_from_csv import parse_parcels + default_town = Town.objects.create(numero_insee="12345", + name="default_town") + test_values = ( + (u"1996 : XT:53,54,56,57,59,60,61,62", + {1996:[ + ("XT", "53"), ("XT", "54"), ("XT", "56"), ("XT", "57"), + ("XT", "59"), ("XT", "60"), ("XT", "61"), ("XT", "62"), + ]} + ), + (u"AD:23", + {None:[ + ("AD", "23") + ]}), + (u"1961 :B1:227;", + {1961:[ + ("B1", '227') + ]} + ), + (u"1982 CV:35;CV:36", + {1982:[ + ("CV", "35"), ("CV", "36"), + ]} + ), + (u"E:24;E:25", + {None:[ + ("E", "24"), ("E", "25"), + ]} + ), + (u"B : 375, 376, 386, 387, 645, 646 / C : 412 à 415, 432 à 435, "\ + u"622 / F : 120, 149, 150, 284, 287, 321 à 323", + {None:[ + ("B", "375"), ("B", "376"), ("B", "386"), ("B", "387"), + ("B", "645"), ("B", "646"), + ("C", "412"), ("C", "413"), ("C", "414"), ("C", "415"), + ("C", "432"), ("C", "433"), ("C", "434"), ("C", "435"), + ("C", "622"), + ("F", "120"), ("F", "149"), ("F", "150"), ("F", "284"), + ("F", "287"), ("F", "321"), ("F", "322"), ("F", "323"), + ]} + ), + (u"AD : 95, 96, 86, 87, 81, 252, AE : 58, AD : 115 à 132", + {None:[ + ("AD", "95"), ("AD", "96"), ("AD", "86"), ("AD", "87"), + ("AD", "81"), ("AD", "252"), ("AD", "115"), ("AD", "116"), + ("AD", "117"), ("AD", "118"), ("AD", "119"), ("AD", "120"), + ("AD", "121"), ("AD", "122"), ("AD", "123"), ("AD", "124"), + ("AD", "125"), ("AD", "126"), ("AD", "127"), ("AD", "128"), + ("AD", "129"), ("AD", "130"), ("AD", "131"), ("AD", "132"), + ("AE", "58"), + ]} + ), + (u"XD:1 à 13, 24 à 28, 33 à 39, 50 à 52, 80, 83, 84 à 86, 259 à "\ + u"261, 182, 225 ; XH:5 ; P:1640, 1888, 1889, 1890 ; R:1311, "\ + u"1312, 1314,1342, 1343, 1559 à 1569", + {None:[ + ('XD', "1"), ('XD', "2"), ('XD', "3"), ('XD', "4"), ('XD', "5"), + ('XD', "6"), ('XD', "7"), ('XD', "8"), ('XD', "9"), + ('XD', "10"), ('XD', "11"), ('XD', "12"), ('XD', "13"), + ("XD", "24"), ("XD", "25"), ("XD", "26"), ("XD", "27"), + ("XD", "28"), ("XD", "33"), ("XD", "34"), ("XD", "35"), + ("XD", "36"), ("XD", "37"), ("XD", "38"), ("XD", "39"), + ("XD", "50"), ("XD", "51"), ("XD", "52"), ("XD", "80"), + ("XD", "83"), ("XD", "84"), ("XD", "85"), ("XD", "86"), + ("XD", "259"), ("XD", "260"), ("XD", "261"), ("XD", "182"), + ("XD", "225"), ("XH", "5"), + ("P", "1640"), ("P", "1888"), ("P", "1889"), ("P", "1890"), + ("R", "1311"), ("R", "1312"), ("R", "1314"), ("R", "1342"), + ("R", "1343"), ("R", "1559"), ("R", "1560"), ("R", "1561"), + ("R", "1562"), ("R", "1563"), ("R", "1564"), ("R", "1565"), + ("R", "1566"), ("R", "1567"), ("R", "1568"), ("R", "1569"), + ]} + ), + (u"BZ:2 à 5, 365 ; CD:88 à 104, 106, 108, 326", + {None:[ + ('BZ', '2'), ('BZ', '3'), ('BZ', '4'), ('BZ', '5'), ('BZ', '365'), + ('CD', '88'), ('CD', '89'), ('CD', '90'), ('CD', '91'), + ('CD', '92'), ('CD', '93'), ('CD', '94'), ('CD', '95'), + ('CD', '96'), ('CD', '97'), ('CD', '98'), ('CD', '99'), + ('CD', '100'), ('CD', '101'), ('CD', '102'), ('CD', '103'), + ('CD', '104'), ('CD', '106'), ('CD', '326'), ('CD', '108') + ]} + ), + (u"AV 118 à 125, 127, 132 à 137, 153, 398p, 399, 402; BI 27, 30, "\ + u"32, 33, 188, 255, 256 à 258, 260, 284p, 294; BL 297", + {None:[ + ('AV','118'), ('AV','119'), ('AV','120'), ('AV','121'), + ('AV','122'), ('AV','123'), ('AV','124'), ('AV','125'), + ('AV','127'), ('AV','132'), ('AV','133'), ('AV','134'), + ('AV','135'), ('AV','136'), ('AV','137'), ('AV','153'), + ('AV','398p'), ('AV','399'), ('AV','402'), + ('BI','27'), ('BI','30'), ('BI','32'), ('BI','33'), ('BI','188'), + ('BI','255'), ('BI','256'), ('BI','257'), ('BI','258'), + ('BI','260'), ('BI','284p'), ('BI','294'), + ('BL','297'), + ]} + ), + (u"A : 904 à 906, 911 ; E:40, 41", + {None:[ + ("A",'904'), ("A",'905'), ("A",'906'), ("A",'911'), + ("E", '40'), ("E", "41") + ]} + ), + (u"1991 : BE:8, 12", + {"1991":[ + ('BE', '8'), ('BE', '12'), + ]} + ), + (u"1979 : EM:1", + {"1979":[ + ('EM', '1') + ]}, + ), + (u"B:448;B:449;B:450;B:451;B:452;B:455;B:456;B:457;B:458;B:459;"\ + u"B:1486;", + {None:[ + ("B", "448"), ("B", "449"), ("B", "450"), ("B", "451"), + ("B", "452"), ("B", "455"), ("B", "456"), ("B", "457"), + ("B", "458"), ("B", "459"), ("B", "1486"), + ]} + ), + (u"AC : 72 à 81, 91 à 100, 197 / ZC:180 à 189", + {None:[ + ('AC', '72'), ('AC', '73'), ('AC', '74'), ('AC', '75'), + ('AC', '76'), ('AC', '77'), ('AC', '78'), ('AC', '79'), ('AC', + '80'), ('AC', '81'), ('AC', '91'), ('AC', '92'), ('AC', '93'), + ('AC', '94'), ('AC', '95'), ('AC', '96'), ('AC', '97'), ('AC', + '98'), ('AC', '99'), ('AC', '100'), ('AC', '197'), ('ZC', '180'), + ('ZC', '181'), ('ZC', '182'), ('ZC', '183'), ('ZC', '184'), + ('ZC', '185'), ('ZC', '186'), ('ZC', '187'), ('ZC', '188'), + ('ZC', '189'), + ]} + ), + (u"AB 37 et 308", + {None:[ + ('AB', '37'), ('AB', '308'), + ]} + ), + (u"1983 D2 n° 458 et 459", + {"1983":[ + ('D2', '458'), ('D2', '459'), + ]} + ), + (u"ZS : 21p, 66", + {None:[ + ('ZS', '21p'), ('ZS', '66'), + ]} + ), + (u"VV:166, 167, domaine public", + {None:[ + ('VV', '166'), ('VV', '167'), + ]} + ), + (u" AS:13 à 15, 17 à 19, 21 à 32, 34 à 45, 47 à 53, 69, 70, 82, "\ + u"84 / CK:1, 24, 25, 29, 30, 37 à 43", + {None:[ + ("AS", "13"), ("AS", "14"), ("AS", "15"), ("AS", "17"), ("AS", + "18"), ("AS", "19"), ("AS", "21"), ("AS", "22"), ("AS", "23"), + ("AS", "24"), ("AS", "25"), ("AS", "26"), ("AS", "27"), ("AS", + "28"), ("AS", "29"), ("AS", "30"), ("AS", "31"), ("AS", "32"), + ("AS", "34"), ("AS", "35"), ("AS", "36"), ("AS", "37"), ("AS", + "38"), ("AS", "39"), ("AS", "40"), ("AS", "41"), ("AS", "42"), + ("AS", "43"), ("AS", "44"), ("AS", "45"), ("AS", "47"), ("AS", + "48"), ("AS", "49"), ("AS", "50"), ("AS", "51"), ("AS", "52"), + ("AS", "53"), ('AS', "69"), ('AS', "70"), ('AS', "82"), ('AS', + "84"), + ('CK', "1"), ('CK', "24"), ('CK', "25"), ('CK', "29"), ('CK', + "30"), ('CK', "37"), ('CK', "38"), ('CK', "39"), ('CK', "40"), + ('CK', "41"), ('CK', "42"), ('CK', "43"), + ]} + ), + (u" ZN:37, 15, 35, 28, 29 / ZM:9, 73", + {None:[ + ("ZN", "37"), ("ZN", "15"), ("ZN", "35"), ("ZN", "28"), + ("ZN", "29"), ("ZM", "9"), ("ZM", "73"), + ]} + ), + (u" Tranche n°1 : YP:243, 12, 14 à 16, 18 à 26, DP / Tranche n°2 :"\ + u"YP:17, 307, 27, 308, 44 à 46, 683, BM:1, 250, 488 à 492", + {None:[ + ('YP', '243'), ('YP', '12'), ('YP', '14'), ('YP', '15'), ('YP', + '16'), ('YP', '18'), ('YP', '19'), ('YP', '20'), ('YP', '21'), + ('YP', '22'), ('YP', '23'), ('YP', '24'), ('YP', '25'), ('YP', + '26'), ('YP', '17'), ('YP', '27'), ('YP', '308'), ('YP', '44'), + ('YP', '45'), ('YP', '46'), ('YP', '683'), ('YP', '307'), + ('BM', '1'), ('BM', '250'), ('BM', '488'), ('BM', '489'), + ('BM', '490'), ('BM', '491'), ('BM', '492'), + ]} + ), + (u" H : 106, 156, 158", + {None:[ + ('H','106'), ('H','156'), ('H','158'), + ]} + ), + (u"Section YO : parcelles n° 19; 20", + {None:[ + ('YO','19'), ('YO','20'), + ]} + ), + (u"1991 :AI:23;19;20;21;22;181;AM:116;214;215;233;235", + {u"1991":[ + (u"AI", "19"), (u"AI", "20"), (u"AI", "21"), (u"AI", "22"), + (u"AI", "23"), (u"AI", "181"), + (u"AM", "116"), (u"AM", "214"), (u"AM", "215"), (u"AM", "233"), + (u"AM", "235"), + ] + }) + ) + #),(u"Domaine public", {} + #),(u"Tranche 1 : AV:4 à 6, 18, 80, 104 / partiellement : 5 et 18", {} + #),(u" 1987 : ZD: ?", {} + #),(u"A:26a, 26b, 27 / AB:95 / AK:4, 12, 20", {} + for value, result in test_values: + parcels = parse_parcels(value, "12345", None) + if not parcels and not result: + continue + self.assertTrue(parcels != [], + msg="No parcel parsed for \"%s\"" % value) + parcels_copy = parcels[:] + for year in result.keys(): + for values in parcels_copy: + if values['year'] != year and \ + values['year'] != unicode(year): + continue + self.assertTrue((values['section'], values['parcel_number']) + in result[year], + msg="Section - Parcel number: \"%s - %s\" is not "\ + "in \"%s\"" % (values['section'], + values['parcel_number'], unicode(result[year]))) + parcels.pop(parcels.index(values)) + result[year].pop(result[year].index((values['section'], + values['parcel_number']))) + # all parcels have been imported + self.assertEqual(parcels, [], msg="Parcel(s): \"%s\" haven't be "\ + "recognized in \"%s\"" % (str(parcels), + value)) + not_imported = [data for data in result.values() if data] + self.assertEqual(not_imported, [], msg="Parcel(s): \"%s\" haven't be "\ + "recognized in \"%s\"" % (str(not_imported), + value) + ) + diff --git a/example_project/local_settings_nantes.py b/example_project/local_settings_nantes.py index 1b9b9a537..ac839e00d 100644 --- a/example_project/local_settings_nantes.py +++ b/example_project/local_settings_nantes.py @@ -197,9 +197,9 @@ ISHTAR_OPE_COL_FORMAT = [ None, # pass None, # pass None, # pass - (('towns',), 'parse_insee'), + (('towns',), 'parse_insee', None, True), None, # pass - None, # (('parcels',), 'parse_parcels'), + (('parcels',), 'parse_parcels', [79]), None, # pass None, # pass None, # pass diff --git a/example_project/settings.py b/example_project/settings.py index bfb59fb5d..bb7a59d74 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -102,7 +102,7 @@ INSTALLED_APPS = [ 'django.contrib.formtools', 'south', 'registration', - 'geodjangofla', + #'geodjangofla', 'ishtar_common', 'archaeological_operations', # mandatory app to run ishtar #'django_extensions', @@ -196,3 +196,4 @@ DEBUG_TOOLBAR_CONFIG = {'INTERCEPT_REDIRECTS':False} import sys if 'test' in sys.argv or 'test_coverage' in sys.argv: DATABASES['default']['engine'] = 'sqlite3' + |