summaryrefslogtreecommitdiff
path: root/archaeological_files/data_importer.py
diff options
context:
space:
mode:
Diffstat (limited to 'archaeological_files/data_importer.py')
-rw-r--r--archaeological_files/data_importer.py421
1 files changed, 219 insertions, 202 deletions
diff --git a/archaeological_files/data_importer.py b/archaeological_files/data_importer.py
index 2b715cb42..712bcbc51 100644
--- a/archaeological_files/data_importer.py
+++ b/archaeological_files/data_importer.py
@@ -20,16 +20,15 @@
import unicodecsv
from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
from ishtar_common.data_importer import *
from ishtar_common.models import Person, OrganizationType
-from ishtar_common.unicode_csv import unicode_csv_reader
from archaeological_operations.data_importer import *
from archaeological_files import models
+
class ImportClosingFormater(ImportFormater):
def post_process(self, obj, context, value, owner=None):
value = self.formater.format(value)
@@ -41,11 +40,12 @@ class ImportClosingFormater(ImportFormater):
obj.end_date = open_date + datetime.timedelta(30)
obj.save()
+
class ImportMayorFormater(ImportFormater):
def post_process(self, obj, context, value, owner=None):
value = self.formater.format(value)
if type(self.field_name) in (list, tuple):
- return #not managed
+ return # not managed
associated_obj = get_object_from_path(obj, self.field_name)
if not value or not obj.main_town or not associated_obj:
return
@@ -55,10 +55,11 @@ class ImportMayorFormater(ImportFormater):
setattr(associated_obj, self.field_name.split('__')[-1], value)
associated_obj.save()
+
class FilePostProcessing(object):
# erroneaous date to be easily identified on post-processing
DEFAULT_YEAR = 1756
- EXTRA_DEFAULTS = {tuple():{'year':DEFAULT_YEAR}}
+ EXTRA_DEFAULTS = {tuple(): {'year': DEFAULT_YEAR}}
def post_processing(self, item, data):
if item.year == self.DEFAULT_YEAR:
@@ -69,237 +70,253 @@ class FilePostProcessing(object):
item.save()
return item
+
class FileImporterSraPdL(FilePostProcessing, Importer):
DESC = u"Exports dossiers SRA PdL : importeur Filemaker dossiers"
LINE_FORMAT = []
OBJECT_CLS = models.File
- DEFAULTS = {('responsible_town_planning_service', 'attached_to'):{
- 'organization_type':OrganizationType.objects.get(
- txt_idx="planning_service")},
- ('general_contractor', 'attached_to'):{
- 'organization_type':OrganizationType.objects.get(
- txt_idx="general_contractor")},
- tuple():{
- 'file_type': models.FileType.objects.get(
- txt_idx='preventive'),
- },
- ('in_charge',):{'attached_to':None}, # initialized in __init__
- }
- STR_CUT = {tuple():{'comment':2000}}
+ DEFAULTS = {
+ ('responsible_town_planning_service', 'attached_to'): {
+ 'organization_type': OrganizationType.objects.get(
+ txt_idx="planning_service")},
+ ('general_contractor', 'attached_to'): {
+ 'organization_type': OrganizationType.objects.get(
+ txt_idx="general_contractor")},
+ tuple(): {
+ 'file_type': models.FileType.objects.get(
+ txt_idx='preventive'),
+ },
+ ('in_charge',): {'attached_to': None}, # initialized in __init__
+ }
+ STR_CUT = {tuple(): {'comment': 2000}}
def _init_line_format(self):
tf = TownFormater()
tf.town_dct_init()
self.line_format = [
- None, # A, 1
- ImportFormater(['address', 'postal_code', ['main_town', 'parcels__town']], # B, 2
- [UnicodeFormater(500, clean=True),
- UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
- tf],
- regexp=RE_ADD_CD_POSTAL_TOWN,
- regexp_formater_args=[[0], [1], [2, 1]], required=False,
- comment=u"Dossier - adresse"),
- ImportMayorFormater('general_contractor__raw_name', # C, 3 TODO - extraire nom_prenom_titre
- UnicodeFormater(200),
- comment=u"Aménageur - nom brut",
- post_processing=True,
- required=False),
- ImportFormater(['general_contractor__address', # D, 4
- 'general_contractor__postal_code',
- 'general_contractor__town'],
- [UnicodeFormater(500, clean=True),
- UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
- TownFormater(town_full_dct=tf._town_full_dct,
- town_dct=tf._town_dct)],
- regexp=RE_ADD_CD_POSTAL_TOWN,
- regexp_formater_args=[[0], [1], [2, 1]], required=False,
- comment=u"Aménageur - adresse"),
- ImportFormater("general_contractor__title", # E, 5
- StrChoiceFormater(Person.TYPE, cli=True),
- required=False,
- comment=u"Aménageur - titre"),
- None, # F, 6
- None, # G, 7
- None, # H, 8
- ImportFormater("parcels__year", # I, 9
- YearNoFuturFormater(),
- required=False),
- ImportParcelFormater('', required=False, post_processing=True), # J, 10
- None, # K, 11
- ImportFormater([['main_town', 'parcels__town']], # L, 12
- tf,
- required=False,
- comment=u"Commune (si non définie avant)"),
- ImportFormater([['main_town', 'parcels__town']], # M, 13
- tf,
- required=False,
- comment=u"Commune (si non définie avant)"),
- ImportFormater('saisine_type', # N, 14
- TypeFormater(models.SaisineType),
- required=False,
- comment=u"Type de saisine"),
- None, # O, 15
- ImportFormater('comment', # P, 16
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- None, # Q, 17
- ImportFormater([
- 'responsible_town_planning_service__raw_name', # R, 18 service instructeur
+ None, # A, 1
+ ImportFormater(
+ ['address', 'postal_code',
+ ['main_town', 'parcels__town']], # B, 2
+ [UnicodeFormater(500, clean=True),
+ UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER), tf],
+ regexp=RE_ADD_CD_POSTAL_TOWN,
+ regexp_formater_args=[[0], [1], [2, 1]], required=False,
+ comment=u"Dossier - adresse"),
+ ImportMayorFormater(
+ # C, 3 TODO - extraire nom_prenom_titre
+ 'general_contractor__raw_name',
+ UnicodeFormater(200),
+ comment=u"Aménageur - nom brut",
+ post_processing=True,
+ required=False),
+ ImportFormater(
+ ['general_contractor__address', # D, 4
+ 'general_contractor__postal_code',
+ 'general_contractor__town'],
+ [UnicodeFormater(500, clean=True),
+ UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
+ TownFormater(town_full_dct=tf._town_full_dct,
+ town_dct=tf._town_dct)],
+ regexp=RE_ADD_CD_POSTAL_TOWN,
+ regexp_formater_args=[[0], [1], [2, 1]], required=False,
+ comment=u"Aménageur - adresse"),
+ ImportFormater(
+ "general_contractor__title", # E, 5
+ StrChoiceFormater(Person.TYPE, cli=True),
+ required=False,
+ comment=u"Aménageur - titre"),
+ None, # F, 6
+ None, # G, 7
+ None, # H, 8
+ ImportFormater("parcels__year", # I, 9
+ YearNoFuturFormater(),
+ required=False),
+ # J, 10
+ ImportParcelFormater('', required=False, post_processing=True),
+ None, # K, 11
+ ImportFormater([['main_town', 'parcels__town']], # L, 12
+ tf,
+ required=False,
+ comment=u"Commune (si non définie avant)"),
+ ImportFormater([['main_town', 'parcels__town']], # M, 13
+ tf,
+ required=False,
+ comment=u"Commune (si non définie avant)"),
+ ImportFormater('saisine_type', # N, 14
+ TypeFormater(models.SaisineType),
+ required=False,
+ comment=u"Type de saisine"),
+ None, # O, 15
+ ImportFormater('comment', # P, 16
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ None, # Q, 17
+ ImportFormater(
+ [
+ 'responsible_town_planning_service__raw_name',
+ # R, 18 service instructeur
'responsible_town_planning_service__address',
'responsible_town_planning_service__postal_code',
- 'responsible_town_planning_service__town',],
- [UnicodeFormater(300, clean=True),
- UnicodeFormater(300, clean=True),
- UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
- TownFormater(town_full_dct=tf._town_full_dct,
- town_dct=tf._town_dct)],
- regexp=RE_NAME_ADD_CD_POSTAL_TOWN,
- regexp_formater_args=[[0], [1], [2], [3, 2]],
- comment=u"Aménageur - adresse",
- required=False),
- ImportFormater('comment', # S, 19
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- ImportYearFormater('reception_date', # T, 20
- DateFormater(['%d/%m/%Y', '%d/%m/%Y']),
- comment=u"Date de création",
- required=False,
- duplicate_fields=['creation_date']),
- None, # U, 21
- None, # V, 22
- None, # W, 23
- None, # X, 24
- None, # Y, 25
- None, # Z, 26
- None, # AA, 27
- None, # AB, 28
- None, # AC, 29
- None, # AD, 30
- None, # AE, 31
- None, # AF, 32
- None, # AG, 33
- None, # AH, 34
- ImportFormater('creation_date', # AI, 35
- DateFormater(['%d/%m/%Y', '%d/%m/%y']),
- force_value=True,
- comment=u"Date de création",
- required=False,),
- None, # AJ, 36
- ImportFormater('comment', # AK, 37
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- None, # AL, 38
- None, # AM, 39
- None, # AN, 40
- None, # AO, 41
- ImportFormater('comment', # AP, 42
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- None, # AQ, 43
- None, # AR, 44
- None, # AS, 45
- None, # AT, 46
- ImportFormater('comment', # AU, 47
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- None, # AV, 48
- ImportFormater('permit_reference', # AW, 49
- UnicodeFormater(300, clean=True),
- regexp=RE_PERMIT_REFERENCE,
- comment=u"Réf. du permis de construire",
- required=False),
- None, # AX, 50
- None, # AY, 51
- None, # AZ, 52
- None, # BA, 53
- None, # BB, 54
- None, # BC, 55
- None, # BD, 56
- ImportFormater([['main_town', 'parcels__town']], # BE, 57
- TownINSEEFormater(),
- required=False,
- comment=u"Commune (si non définie avant)"),
- ImportFormater('comment', # BF, 58
- UnicodeFormater(2000),
- comment=u"Commentaire",
- concat=True, required=False),
- None, # BG, 59
- None, # BH, 60
- None, # BI, 61
- None, # BJ, 62
- None, # BK, 63
- None, # BL, 64
- None, # BM, 65
- None, # BN, 66
- None, # BO, 67
- None, # BP, 68
- None, # BQ, 69
- None, # BR, 70
- None, # BS, 71
- None, # BT, 72
- None, # BU, 73
- ImportClosingFormater('', StrToBoolean(cli=True),
- post_processing=True, required=False,
- comment=u'Dossier clos'), # BV, 74, end date
- ImportClosingFormater('in_charge__raw_name', # BW, 75 responsable
- UnicodeFormater(200),
- comment=u"Responsable - nom brut",
- required=False),
- ImportFormater('total_surface', # BX, 76 surface totale
- SurfaceFormater(),
- comment=u"Surface totale",
- required=False),
- ImportFormater('total_developed_surface', # BY, 77 surface totale aménagée
- SurfaceFormater(),
- comment=u"Surface totale aménagée",
- required=False),
- None, # BZ, 78
- None, # CA, 79
- None, # CB, 80
- None, # CC, 81
- None, # CD, 82
- None, # CE, 83
- None, # CF, 84
- ImportFormater('permit_type',
- TypeFormater(models.PermitType),
- required=False,
- comment=u"Type de permis"), # CG, 85
- None, # CH, 85
+ 'responsible_town_planning_service__town'],
+ [UnicodeFormater(300, clean=True),
+ UnicodeFormater(300, clean=True),
+ UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
+ TownFormater(town_full_dct=tf._town_full_dct,
+ town_dct=tf._town_dct)],
+ regexp=RE_NAME_ADD_CD_POSTAL_TOWN,
+ regexp_formater_args=[[0], [1], [2], [3, 2]],
+ comment=u"Aménageur - adresse",
+ required=False),
+ ImportFormater('comment', # S, 19
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ ImportYearFormater('reception_date', # T, 20
+ DateFormater(['%d/%m/%Y', '%d/%m/%Y']),
+ comment=u"Date de création",
+ required=False,
+ duplicate_fields=['creation_date']),
+ None, # U, 21
+ None, # V, 22
+ None, # W, 23
+ None, # X, 24
+ None, # Y, 25
+ None, # Z, 26
+ None, # AA, 27
+ None, # AB, 28
+ None, # AC, 29
+ None, # AD, 30
+ None, # AE, 31
+ None, # AF, 32
+ None, # AG, 33
+ None, # AH, 34
+ ImportFormater('creation_date', # AI, 35
+ DateFormater(['%d/%m/%Y', '%d/%m/%y']),
+ force_value=True,
+ comment=u"Date de création",
+ required=False,),
+ None, # AJ, 36
+ ImportFormater('comment', # AK, 37
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ None, # AL, 38
+ None, # AM, 39
+ None, # AN, 40
+ None, # AO, 41
+ ImportFormater('comment', # AP, 42
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ None, # AQ, 43
+ None, # AR, 44
+ None, # AS, 45
+ None, # AT, 46
+ ImportFormater('comment', # AU, 47
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ None, # AV, 48
+ ImportFormater('permit_reference', # AW, 49
+ UnicodeFormater(300, clean=True),
+ regexp=RE_PERMIT_REFERENCE,
+ comment=u"Réf. du permis de construire",
+ required=False),
+ None, # AX, 50
+ None, # AY, 51
+ None, # AZ, 52
+ None, # BA, 53
+ None, # BB, 54
+ None, # BC, 55
+ None, # BD, 56
+ ImportFormater([['main_town', 'parcels__town']], # BE, 57
+ TownINSEEFormater(),
+ required=False,
+ comment=u"Commune (si non définie avant)"),
+ ImportFormater('comment', # BF, 58
+ UnicodeFormater(2000),
+ comment=u"Commentaire",
+ concat=True, required=False),
+ None, # BG, 59
+ None, # BH, 60
+ None, # BI, 61
+ None, # BJ, 62
+ None, # BK, 63
+ None, # BL, 64
+ None, # BM, 65
+ None, # BN, 66
+ None, # BO, 67
+ None, # BP, 68
+ None, # BQ, 69
+ None, # BR, 70
+ None, # BS, 71
+ None, # BT, 72
+ None, # BU, 73
+ ImportClosingFormater('', StrToBoolean(cli=True),
+ post_processing=True, required=False,
+ comment=u'Dossier clos'), # BV, 74, end date
+ ImportClosingFormater(
+ 'in_charge__raw_name', # BW, 75 responsable
+ UnicodeFormater(200),
+ comment=u"Responsable - nom brut",
+ required=False),
+ ImportFormater('total_surface', # BX, 76 surface totale
+ SurfaceFormater(),
+ comment=u"Surface totale",
+ required=False),
+ ImportFormater('total_developed_surface',
+ # BY, 77 surface totale aménagée
+ SurfaceFormater(),
+ comment=u"Surface totale aménagée",
+ required=False),
+ None, # BZ, 78
+ None, # CA, 79
+ None, # CB, 80
+ None, # CC, 81
+ None, # CD, 82
+ None, # CE, 83
+ None, # CF, 84
+ ImportFormater('permit_type',
+ TypeFormater(models.PermitType),
+ required=False,
+ comment=u"Type de permis"), # CG, 85
+ None, # CH, 85
]
def __init__(self, *args, **kwargs):
super(FileImporterSraPdL, self).__init__(*args, **kwargs)
self.DEFAULTS[('in_charge',)]['attached_to'], created = \
- models.Organization.objects.get_or_create(name='SRA Pays de la Loire',
- defaults={
- 'organization_type':OrganizationType.objects.get(
- txt_idx='sra')})
+ models.Organization.objects.get_or_create(
+ name='SRA Pays de la Loire',
+ defaults={
+ 'organization_type':
+ OrganizationType.objects.get(txt_idx='sra')})
self._init_line_format()
if tuple() not in self._defaults:
self._defaults[tuple()] = {}
self._defaults[tuple()]['history_modifier'] = self.history_modifier
+
def test(filename):
importer = FileImporterSraPdL(skip_first_line=True, output=True)
with open(filename) as csv_file:
encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
for encoding in encodings:
try:
- importer.importation([line for line in
- unicodecsv.reader(csv_file, encoding='utf-8')])
- #importer.importation(unicode_csv_reader(
+ importer.importation(
+ [line for line in
+ unicodecsv.reader(csv_file, encoding='utf-8')])
+ # importer.importation(unicode_csv_reader(
# [line.decode(encoding)
# for line in csv_file.readlines()])
print importer.get_csv_errors()
break
except ImporterError, e:
print(unicode(e))
- if e.type == ImporterError.HEADER and encoding != encodings[-1]:
+ if e.type == ImporterError.HEADER \
+ and encoding != encodings[-1]:
csv_file.seek(0)
continue
except UnicodeDecodeError:
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-31 22:34:11 +0000