Fix SRA Pdl file import

1 files changed, 100 insertions, 52 deletions

diff --git a/archaeological_files/data_importer.py b/archaeological_files/data_importer.py
index a1dc0fc3a..c6dbf4034 100644
--- a/archaeological_files/data_importer.py
+++ b/archaeological_files/data_importer.py
@@ -59,15 +59,37 @@ class ImportMayorFormater(ImportFormater):
 
 class FilePostProcessing(object):
     # erroneaous date to be easily identified on post-processing
-    DEFAULT_YEAR = 1756
+    DEFAULT_YEAR = 1871
     EXTRA_DEFAULTS = {tuple(): {'year': DEFAULT_YEAR}}
 
     def post_processing(self, item, data):
-        if item.year == self.DEFAULT_YEAR:
-            if item.creation_date:
-                item.year = item.creation_date.year
-            elif item.reception_date:
-                item.year = item.reception_date.year
+        if item.year == self.DEFAULT_YEAR:  # the default year has been applied
+            if item.external_id:
+                idx = None
+                if '-' in item.external_id:
+                    year, y_idx = item.external_id.split('-')
+                    if len(year) == 4:  # 2007-XXXX
+                        try:
+                            item.year = int(year)
+                            idx = y_idx
+                        except ValueError:
+                            pass
+                if '.' in item.external_id:
+                    year, y_idx = item.external_id.split('.')
+                    if len(year) == 4:  # 2011.XXXX
+                        try:
+                            item.year = int(year)
+                            idx = y_idx
+                        except ValueError:
+                            pass
+            if not item.year:
+                if item.creation_date:
+                    item.year = item.creation_date.year
+                elif item.reception_date:
+                    item.year = item.reception_date.year
+        if not item.end_date:  # auto-close
+            open_date = item.reception_date or item.creation_date
+            item.end_date = open_date + datetime.timedelta(30)
         item.save()
         return item
 
@@ -91,7 +113,6 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
         },
         ('in_charge',): {'attached_to': None},  # initialized in __init__
     }
-    STR_CUT = {tuple(): {'comment': 2000}}
 
     def _init_line_format(self):
         tf = TownFormater()
@@ -99,13 +120,15 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
         self.line_format = [
             None,  # A, 1
             ImportFormater(
-                ['address', 'postal_code',
-                 ['main_town', 'parcels__town']],  # B, 2
+                ['general_contractor__attached_to__address',  # B, 2
+                 'general_contractor__attached_to__postal_code',
+                 'general_contractor__attached_to__town'],
                 [UnicodeFormater(500, clean=True),
-                 UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER), tf],
+                 UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
+                 UnicodeFormater(70, clean=True), ],
                 regexp=RE_ADD_CD_POSTAL_TOWN,
-                regexp_formater_args=[[0], [1], [2, 1]], required=False,
-                comment=u"Dossier - adresse"),
+                regexp_formater_args=[[0], [1], [2]], required=False,
+                comment=u"Aménageur - adresse"),
             ImportMayorFormater(
                 # C, 3 TODO - extraire nom_prenom_titre
                 'general_contractor__raw_name',
@@ -113,16 +136,7 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
                 comment=u"Aménageur - nom brut",
                 post_processing=True,
                 required=False),
-            ImportFormater(
-                ['general_contractor__address',  # D, 4
-                 'general_contractor__postal_code',
-                 'general_contractor__town'],
-                [UnicodeFormater(500, clean=True),
-                 UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
-                 UnicodeFormater(70, clean=True),],
-                regexp=RE_ADD_CD_POSTAL_TOWN,
-                regexp_formater_args=[[0], [1], [2]], required=False,
-                comment=u"Aménageur - adresse"),
+            None,  # D, 4
             ImportFormater(
                 "general_contractor__title",  # E, 5
                 StrChoiceFormater(Person.TYPE, cli=True),
@@ -150,33 +164,36 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
                            required=False,
                            comment=u"Type de saisine"),
             None,  # O, 15
-            ImportFormater('comment',  # P, 16
-                           UnicodeFormater(2000),
-                           comment=u"Commentaire",
-                           concat=True, required=False),
+            ImportFormater('name',  # P, 16
+                           UnicodeFormater(),
+                           comment=u"Nom du dossier",
+                           required=False),
             None,  # Q, 17
             ImportFormater(
                 [
                     'responsible_town_planning_service__raw_name',
                     # R, 18 service instructeur
-                    'responsible_town_planning_service__address',
-                    'responsible_town_planning_service__postal_code',
-                    'responsible_town_planning_service__town'],
+                    'responsible_town_planning_service__attached_to__address',
+                    'responsible_town_planning_service__'
+                    'attached_to__postal_code',
+                    'responsible_town_planning_service__attached_to__town'],
                 [UnicodeFormater(300, clean=True),
                  UnicodeFormater(300, clean=True),
                  UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER),
-                 UnicodeFormater(70, clean=True),],
+                 UnicodeFormater(70, clean=True), ],
                 regexp=RE_NAME_ADD_CD_POSTAL_TOWN,
                 regexp_formater_args=[[0], [1], [2], [3]],
-                comment=u"Aménageur - adresse",
+                comment=u"Service instructeur - adresse",
+                required=False),
+            ImportFormater(
+                'comment',  # S, 19
+                UnicodeFormater(prefix=u'* Considérants : '),
+                comment=u"Commentaire",
+                concat=True,
                 required=False),
-            ImportFormater('comment',  # S, 19
-                           UnicodeFormater(2000),
-                           comment=u"Commentaire",
-                           concat=True, required=False),
             ImportYearFormater('reception_date',  # T, 20
                                DateFormater(['%d/%m/%Y', '%d/%m/%Y']),
-                               comment=u"Date de création",
+                               comment=u"Date de réception",
                                required=False,
                                duplicate_fields=[['creation_date', False]]),
             None,  # U, 21
@@ -200,24 +217,34 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
                            required=False,),
             None,  # AJ, 36
             ImportFormater('comment',  # AK, 37
-                           UnicodeFormater(2000),
+                           UnicodeFormater(prefix=u"* Historique : "),
                            comment=u"Commentaire",
                            concat=True, required=False),
-            None,  # AL, 38
+            ImportFormater('internal_reference',  # AL, 38
+                           UnicodeFormater(60),
+                           comment=u"Autre référence",
+                           required=False,
+                           duplicate_fields=[['external_id', False]]),
             None,  # AM, 39
             None,  # AN, 40
-            None,  # AO, 41
+            ImportFormater('comment',  # AO, 41
+                           UnicodeFormater(
+                               prefix=u"* Justificatif de prescription : "),
+                           comment=u"Justificatif de prescription",
+                           concat=True, required=False),
             ImportFormater('comment',  # AP, 42
-                           UnicodeFormater(2000),
-                           comment=u"Commentaire",
+                           UnicodeFormater(
+                               prefix=u"* Justificatif d'intervention : "),
+                           comment=u"Justificatif d'intervention",
                            concat=True, required=False),
             None,  # AQ, 43
             None,  # AR, 44
             None,  # AS, 45
             None,  # AT, 46
             ImportFormater('comment',  # AU, 47
-                           UnicodeFormater(2000),
-                           comment=u"Commentaire",
+                           UnicodeFormater(
+                               prefix=u"* Méthodologie de l'opération : "),
+                           comment=u"Méthodologie de l'opération",
                            concat=True, required=False),
             None,  # AV, 48
             ImportFormater('permit_reference',  # AW, 49
@@ -225,12 +252,29 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
                            regexp=RE_PERMIT_REFERENCE,
                            comment=u"Réf. du permis de construire",
                            required=False),
-            None,  # AX, 50
+            ImportFormater('comment',  # AX, 50
+                           UnicodeFormater(
+                               prefix=u"* Référence de dossier aménageur : "),
+                           comment=u"Référence de dossier aménageur",
+                           concat=True, required=False),
             None,  # AY, 51
             None,  # AZ, 52
-            None,  # BA, 53
-            None,  # BB, 54
-            None,  # BC, 55
+            ImportFormater('comment',  # BA, 53
+                           UnicodeFormater(
+                               prefix=u"* Numéro d'arrêté préfectoral : "),
+                           comment=u"Numéro d'arrêté préfectoral",
+                           concat=True, required=False),
+            ImportFormater('comment',  # BB, 54
+                           UnicodeFormater(
+                               prefix=u"* Numéro d'arrêté SRA : "),
+                           comment=u"Numéro d'arrêté SRA",
+                           concat=True, required=False),
+            ImportFormater('comment',  # BC, 55
+                           UnicodeFormater(
+                               prefix=u"* Numéro d'arrêté de "
+                               u"post-diagnostic : "),
+                           comment=u"Numéro d'arrêté de post-diagnostic",
+                           concat=True, required=False),
             None,  # BD, 56
             ImportFormater([['main_town', 'parcels__town']],  # BE, 57
                            TownINSEEFormater(),
@@ -253,12 +297,16 @@ class FileImporterSraPdL(FilePostProcessing, Importer):
             None,  # BQ, 69
             None,  # BR, 70
             None,  # BS, 71
-            None,  # BT, 72
+            ImportFormater(  # BT, 72 nom service instructeur
+                ['responsible_town_planning_service__attached_to__name', ],
+                [UnicodeFormater(300, clean=True), ],
+                regexp=RE_NAME_ADD_CD_POSTAL_TOWN,
+                regexp_formater_args=[[0], ],
+                comment=u"Service instructeur - nom",
+                required=False),
             None,  # BU, 73
-            ImportClosingFormater('', StrToBoolean(cli=True),
-                                  post_processing=True, required=False,
-                                  comment=u'Dossier clos'),  # BV, 74, end date
-            ImportClosingFormater(
+            None,  # BV, 74
+            ImportFormater(
                 'in_charge__raw_name',  # BW, 75 responsable
                 UnicodeFormater(200),
                 comment=u"Responsable - nom brut",