Add a generic relation model to list general types keys. Reorganize imports.

1 files changed, 171 insertions, 0 deletions

diff --git a/archaeological_operations/data_importer.py b/archaeological_operations/data_importer.py
new file mode 100644
index 000000000..4402c8b17
--- /dev/null
+++ b/archaeological_operations/data_importer.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2015  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import copy, datetime, re
+
+from django.db import IntegrityError
+from django.template.defaultfilters import slugify
+from django.utils.translation import ugettext_lazy as _
+
+from ishtar_common.data_importer import *
+from ishtar_common.models import Town
+
+from archaeological_operations.models import Parcel
+from archaeological_operations.utils import parse_parcels
+
+RE_PERMIT_REFERENCE = re.compile('[A-Za-z]*(.*)')
+
+class ImportParcelFormater(ImportFormater):
+    NEED = ['town',]
+    PARCEL_OWNER_KEY = 'associated_file'
+
+    def post_process(self, obj, context, value, owner=None):
+        value = value.strip()
+        base_dct = {self.PARCEL_OWNER_KEY:obj, 'history_modifier':owner}
+        if 'parcels' in context:
+            for key in context['parcels']:
+                if context['parcels'][key]:
+                    base_dct[key] = context['parcels'][key]
+        for parcel_dct in parse_parcels(value, owner=owner):
+            parcel_dct.update(base_dct)
+            try:
+                Parcel.objects.get_or_create(**parcel_dct)
+            except IntegrityError:
+                raise ImporterError("Erreur d'import parcelle, contexte : %s" \
+                                                    % unicode(parcel_dct))
+
+class ImportYearFormater(ImportFormater):
+    def post_process(self, obj, context, value, owner=None):
+        value = self.formater.format(value)
+        if not value:
+            return
+        obj.year = value.year
+        obj.save()
+
+class TownFormater(Formater):
+    def __init__(self, town_full_dct={}, town_dct={}):
+        self._town_full_dct = town_full_dct
+        self._town_dct = town_dct
+        self._initialized = False if not self._town_full_dct else True
+
+    def town_dct_init(self):
+        for town in Town.objects.all():
+            key = (slugify(town.name.strip()), town.numero_insee[:2])
+            if key in self._town_full_dct:
+                print("Danger! %s is ambiguous with another town on the same "\
+                      "department."% town.name)
+                continue
+            self._town_full_dct[key] = town
+            key = slugify(town.name.strip())
+            if key in self._town_dct:
+                print("Warning %s is ambiguous with no department provided" %\
+                      town.name)
+                continue
+            self._town_dct[key] = town
+            self._initialized = True
+
+    def format(self, value, extra=None):
+        if not self._initialized:
+            self.town_dct_init()
+        m = RE_FILTER_CEDEX.match(value)
+        if m:
+            value = m.groups()[0]
+        if not value:
+            return None
+        if extra:
+            key = (slugify(value), extra)
+            if key in self._town_full_dct:
+                return self._town_full_dct[key]
+        key = slugify(value)
+        if key in self._town_dct:
+            return self._town_dct[key]
+
+class TownINSEEFormater(Formater):
+    def __init__(self):
+        self._town_dct = {}
+
+    def format(self, value, extra=None):
+        value = value.strip()
+        if not value:
+            return None
+        if value in self._town_dct:
+            return self._town_dct[value]
+        q = Town.objects.filter(insee_code=value)
+        if not q.count():
+            return
+        self._town_dct[value] = q.all()[0]
+        return self._town_dct[value]
+
+class SurfaceFormater(Formater):
+    def test(self):
+        assert self.format(u"352 123") == 352123
+        assert self.format(u"456 789 m²") == 456789
+        assert self.format(u"78ha") == 780000
+
+    def format(self, value, extra=None):
+        value = value.strip()
+        if not value:
+            return None
+        factor = 1
+        if value.endswith(u"m2") or value.endswith(u"m²"):
+            value = value[:-2]
+        if value.endswith(u"ha"):
+            value = value[:-2]
+            factor = 10000
+        try:
+            return int(value.replace(' ', '')) * factor
+        except ValueError:
+            raise ImporterError("Erreur import surface : %s" \
+                                                    % unicode(value))
+
+#RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)[, ](\d{5}) (.*?) *(?: "\
+#                                   "*CEDEX|cedex|Cedex *\d*)*")
+
+RE_NAME_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*" + NEW_LINE_BREAK \
+                                      + "(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)")
+
+RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)")
+
+RE_CD_POSTAL_FILTER = re.compile("(\d*) (\d*)")
+
+RE_ORGA = re.compile("([^,]*)")
+
+
+class OperationImporterBibracte(Importer):
+    LINE_FORMAT = [
+        # CODE OPE
+        ImportFormater('operation_code',
+                IntegerFormater(),),
+        # REGION
+        None,
+        # TYPE operation
+        ImportFormater('operation_type',
+                TypeFormater(models.OperationType),),
+
+        # NOM
+        # OPERATEUR
+        # resp. lien IMPORT avec personne
+        # début
+        # fin
+        # Chronos
+    ]
+
+    OBJECT_CLS = models.Operation
+    DEFAULTS = {}
+