Merge branch 'develop' into develop-bootstrap

2 files changed, 193 insertions, 0 deletions

diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py
new file mode 100644
index 000000000..7ef0e0a38
--- /dev/null
+++ b/ishtar_common/management/commands/import_geofla_csv.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+from django.contrib.gis.geos import GEOSGeometry, Point
+
+from ishtar_common.models import Town
+
+
+class Command(BaseCommand):
+    help = 'Import GEOFLA csv'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--year', type=int, default=2014, dest='year',
+            help='Default year to affect to the town')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        default_year = options['year']
+        sys.stdout.write('* using year {} as a default\n'.format(default_year))
+        sys.stdout.write('* Opening file {}\n'.format(csv_file))
+        nb_created, nb_changed = 0, 0
+        with open(csv_file, 'rb') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for idx, row in enumerate(reader):
+                sys.stdout.write('Processing town %d.\r' % (idx + 1))
+                sys.stdout.flush()
+                num_insee = row['INSEE_COM']
+                if len(num_insee) < 5:
+                    num_insee = '0' + num_insee
+                q = Town.objects.filter(numero_insee=num_insee)
+                created = False
+                if q.count():
+                    if q.filter(year=default_year).count():
+                        town = q.filter(year=default_year).all()[0]
+                    else:
+                        town = q.order_by('-year').all()[0]
+                else:
+                    created = True
+                    nb_created += 1
+                    town = Town(name=row['NOM_COM'],
+                                numero_insee=num_insee)
+                geom = row['wkt_geom'].upper()
+                if 'MULTI' not in geom:
+                    geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')'
+                town.limit = GEOSGeometry(geom, srid=2154)
+                if 'X_CENTROID' in row:
+                    town.center = Point(float(row['X_CENTROID']),
+                                        float(row['Y_CENTROID']), srid=2154)
+                else:
+                    town.center = None
+                if not town.year and default_year:
+                    town.year = default_year
+                if 'SUPERFICIE' in row:
+                    town.surface = row['SUPERFICIE']
+                else:
+                    town.surface = None
+                if not created:
+                    nb_changed += 1
+                town.save()
+        sys.stdout.write('\n* {} town created'.format(nb_created))
+        sys.stdout.write('\n* {} town changed\n'.format(nb_changed))
+        sys.stdout.flush()
+
+
+
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
new file mode 100644
index 000000000..24eb2013e
--- /dev/null
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import re
+import sys
+
+from django.core.management.base import BaseCommand
+
+from ishtar_common.models import Town
+
+
+class Command(BaseCommand):
+    help = 'Import INSEE csv'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--year', type=int, default=2015, dest='year',
+            help='Year to affect to new towns')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        default_year = options['year']
+        sys.stdout.write('* using year {} for new towns\n'.format(default_year))
+        sys.stdout.write('* opening file {}\n'.format(csv_file))
+        r = re.compile(r"(.*)\((.*)\)")
+        nb_created = 0
+        nb_link = 0
+        missing = []
+        strange = []
+        linked = set()
+        with open(csv_file, 'rb') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for idx, row in enumerate(reader):
+                sys.stdout.write('Processing town %d.\r' % (idx + 1))
+                sys.stdout.flush()
+
+                old_insee = row['DepComA']
+                if len(old_insee) < 5:
+                    old_insee = '0' + old_insee
+                q = Town.objects.filter(numero_insee=old_insee)
+
+                if not q.count():
+                    missing.append((old_insee, row['NomCA']))
+                    continue
+                if q.count() > 1:
+                    q = q.filter(year__lt=default_year).order_by('-year')
+                    if not q.count():
+                        strange.append((old_insee, row['NomCA']))
+                        continue
+                old_town = q.all()[0]
+
+                new_insee = row['DepComN']
+                if len(new_insee) < 5:
+                    new_insee = '0' + new_insee
+                q = Town.objects.filter(numero_insee=new_insee,
+                                        year=default_year)
+                if not q.count():
+                    nb_created += 1
+                    name = row['NomCN'].decode('utf-8').strip()
+                    name = r.sub(r"\2 \1", name).strip()
+                    new_town = Town.objects.create(name=name, year=default_year,
+                                                   numero_insee=new_insee)
+                else:
+                    new_town = q.all()[0]
+                if new_town in old_town.children.all():
+                    continue  # link already created
+                nb_link += 1
+                old_town.children.add(new_town)
+                linked.add(new_town)
+        nb_limit = 0
+        sys.stdout.write('\nGenerate limits...'.format(nb_created))
+        for town in linked:
+            if town.generate_geo():
+                nb_limit += 1
+        sys.stdout.write('\n* {} town created\n'.format(nb_created))
+        sys.stdout.write('* {} link created\n'.format(nb_link))
+        sys.stdout.write('* {} limit generated\n'.format(nb_limit))
+        if missing:
+            sys.stdout.write('* theses towns are missing:\n')
+            for insee, name in missing:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        if strange:
+            sys.stdout.write('* theses towns have newer version:\n')
+            for insee, name in strange:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        sys.stdout.flush()
+
+