Merge branch 'develop' into develop-bootstrap

1 files changed, 106 insertions, 0 deletions

diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
new file mode 100644
index 000000000..24eb2013e
--- /dev/null
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import re
+import sys
+
+from django.core.management.base import BaseCommand
+
+from ishtar_common.models import Town
+
+
+class Command(BaseCommand):
+    help = 'Import INSEE csv'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--year', type=int, default=2015, dest='year',
+            help='Year to affect to new towns')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        default_year = options['year']
+        sys.stdout.write('* using year {} for new towns\n'.format(default_year))
+        sys.stdout.write('* opening file {}\n'.format(csv_file))
+        r = re.compile(r"(.*)\((.*)\)")
+        nb_created = 0
+        nb_link = 0
+        missing = []
+        strange = []
+        linked = set()
+        with open(csv_file, 'rb') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for idx, row in enumerate(reader):
+                sys.stdout.write('Processing town %d.\r' % (idx + 1))
+                sys.stdout.flush()
+
+                old_insee = row['DepComA']
+                if len(old_insee) < 5:
+                    old_insee = '0' + old_insee
+                q = Town.objects.filter(numero_insee=old_insee)
+
+                if not q.count():
+                    missing.append((old_insee, row['NomCA']))
+                    continue
+                if q.count() > 1:
+                    q = q.filter(year__lt=default_year).order_by('-year')
+                    if not q.count():
+                        strange.append((old_insee, row['NomCA']))
+                        continue
+                old_town = q.all()[0]
+
+                new_insee = row['DepComN']
+                if len(new_insee) < 5:
+                    new_insee = '0' + new_insee
+                q = Town.objects.filter(numero_insee=new_insee,
+                                        year=default_year)
+                if not q.count():
+                    nb_created += 1
+                    name = row['NomCN'].decode('utf-8').strip()
+                    name = r.sub(r"\2 \1", name).strip()
+                    new_town = Town.objects.create(name=name, year=default_year,
+                                                   numero_insee=new_insee)
+                else:
+                    new_town = q.all()[0]
+                if new_town in old_town.children.all():
+                    continue  # link already created
+                nb_link += 1
+                old_town.children.add(new_town)
+                linked.add(new_town)
+        nb_limit = 0
+        sys.stdout.write('\nGenerate limits...'.format(nb_created))
+        for town in linked:
+            if town.generate_geo():
+                nb_limit += 1
+        sys.stdout.write('\n* {} town created\n'.format(nb_created))
+        sys.stdout.write('* {} link created\n'.format(nb_link))
+        sys.stdout.write('* {} limit generated\n'.format(nb_limit))
+        if missing:
+            sys.stdout.write('* theses towns are missing:\n')
+            for insee, name in missing:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        if strange:
+            sys.stdout.write('* theses towns have newer version:\n')
+            for insee, name in strange:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        sys.stdout.flush()
+
+