Command import_insee_comm_csv: import town relations

5 files changed, 170 insertions, 2 deletions

diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py
index 28c2dabf5..1657006f6 100644
--- a/ishtar_common/management/commands/import_geofla_csv.py
+++ b/ishtar_common/management/commands/import_geofla_csv.py
@@ -37,8 +37,9 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         csv_file = options['csv_file']
-        sys.stdout.write('* Opening file {}\n'.format(csv_file))
         default_year = options['year']
+        sys.stdout.write('* using year {} as a default\n'.format(default_year))
+        sys.stdout.write('* Opening file {}\n'.format(csv_file))
         nb_created, nb_changed = 0, 0
         with open(csv_file, 'rb') as csvfile:
             reader = csv.DictReader(csvfile)
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
new file mode 100644
index 000000000..e64fe42bb
--- /dev/null
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import re
+import sys
+
+from django.core.management.base import BaseCommand
+
+from ishtar_common.models import Town
+
+
+class Command(BaseCommand):
+    help = 'Import INSEE csv'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--year', type=int, default=2015, dest='year',
+            help='Year to affect to new towns')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        default_year = options['year']
+        sys.stdout.write('* using year {} for new towns\n'.format(default_year))
+        sys.stdout.write('* opening file {}\n'.format(csv_file))
+        r = re.compile(r"(.*)\((.*)\)")
+        nb_created = 0
+        nb_link = 0
+        missing = []
+        strange = []
+        linked = set()
+        with open(csv_file, 'rb') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for idx, row in enumerate(reader):
+                sys.stdout.write('Processing town %d.\r' % (idx + 1))
+                sys.stdout.flush()
+
+                old_insee = row['DepComA']
+                if len(old_insee) < 5:
+                    old_insee = '0' + old_insee
+                q = Town.objects.filter(numero_insee=old_insee)
+
+                if not q.count():
+                    missing.append((old_insee, row['NomCA']))
+                    continue
+                if q.count() > 1:
+                    q = q.filter(year_lt=default_year).order_by('-year')
+                    if not q.count():
+                        strange.append((old_insee, row['NomCA']))
+                        continue
+                old_town = q.all()[0]
+
+                new_insee = row['DepComN']
+                if len(new_insee) < 5:
+                    new_insee = '0' + new_insee
+                q = Town.objects.filter(numero_insee=new_insee,
+                                        year=default_year)
+                if not q.count():
+                    nb_created += 1
+                    name = row['NomCN'].upper().strip()
+                    name = r.sub(r"\2 \1", name).strip()
+                    new_town = Town.objects.create(name=name, year=default_year,
+                                                   numero_insee=new_insee)
+                else:
+                    new_town = q.all()[0]
+                if new_town in old_town.children.all():
+                    continue  # link already created
+                nb_link += 1
+                old_town.children.add(new_town)
+                linked.add(new_town)
+        nb_limit = 0
+        for town in linked:
+            if town.generate_geo():
+                nb_limit += 1
+                town.save()
+        sys.stdout.write('\n* {} town created\n'.format(nb_created))
+        sys.stdout.write('* {} link created\n'.format(nb_link))
+        sys.stdout.write('* {} limit generated\n'.format(nb_limit))
+        if missing:
+            sys.stdout.write('* theses towns are missing:\n')
+            for insee, name in missing:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        if strange:
+            sys.stdout.write('* theses towns have newer version:\n')
+            for insee, name in strange:
+                sys.stdout.write('* {} ({})\n'.format(name, insee))
+        sys.stdout.flush()
+
+
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a06568d99..36da56cad 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -2688,6 +2688,25 @@ class Town(Imported, models.Model):
         self.save()
         return self.cached_label
 
+    def generate_geo(self):
+        if self.limit:
+            return
+        parents = None
+        if not self.parents.count():
+            return
+        for parent in self.parents.all():
+            if not parent.limit:
+                return
+            if not parents:
+                parents = parent.limit
+            else:
+                parents = parents.union(parent.limit)
+        # if union is a simple polygon make it a multi
+        if 'MULTI' not in parents.wkt:
+            parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")"
+        self.limit = parents
+        return self.limit
+
     def _generate_cached_label(self):
         cached_label = self.name
         if settings.COUNTRY == "fr":
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 2b0a87386..4aa290ed1 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -92,6 +92,9 @@ class TestCase(BaseTestCase):
 
 
 class CommandsTestCase(TestCase):
+    fixtures = [settings.ROOT_PATH +
+                '../ishtar_common/fixtures/test_towns.json']
+
     def test_clean_ishtar(self):
         """
         Clean ishtar db
@@ -108,11 +111,47 @@ class CommandsTestCase(TestCase):
         self.assertEqual(Parcel.objects.filter(pk=p.pk).count(), 0)
 
     def test_import_geofla(self):
-        town_nb = models.Town.objects.count()
+        q = models.Town.objects
+        town_nb = q.count()
         out = StringIO()
         call_command('import_geofla_csv',
                      '../ishtar_common/tests/geofla-test.csv', stdout=out)
         self.assertEqual(town_nb + 9, models.Town.objects.count())
+        call_command('import_geofla_csv',
+                     '../ishtar_common/tests/geofla-test.csv', stdout=out)
+        # no new town
+        self.assertEqual(town_nb + 9, models.Town.objects.count())
+
+    def test_import_insee(self):
+        q = models.Town.objects
+        town_nb = q.count()
+        first, union_start, union_end = '', '', []
+        for idx, town in enumerate(q.all()):
+            l = 'MULTIPOLYGON((({x1} 1,{x2} 1,{x2} 0,{x1} 0,{x1} 1)))'.format(
+                x1=idx, x2=idx + 1)
+            if union_start:
+                union_start += ", "
+            else:
+                first = '{x1} 1'.format(x1=idx)
+            union_start += '{x2} 1'.format(x1=idx, x2=idx + 1)
+            union_end.append('{x2} 0'.format(x1=idx, x2=idx + 1))
+            town.limit = l
+            town.save()
+        union = 'MULTIPOLYGON (((' + first + ", " + union_start + \
+            ", " + ", ".join(reversed(union_end)) + ", 0 0, " + first + ")))"
+        out = StringIO()
+        call_command('import_insee_comm_csv',
+                     '../ishtar_common/tests/insee-test.csv', stdout=out)
+        self.assertEqual(town_nb + 1, models.Town.objects.count())
+        new = models.Town.objects.order_by('-pk').all()[0]
+        self.assertEqual(new.parents.count(), 2)
+
+        self.assertEqual(new.limit.wkt, union)
+
+        call_command('import_insee_comm_csv',
+                     '../ishtar_common/tests/insee-test.csv', stdout=out)
+        # no new town
+        self.assertEqual(town_nb + 1, models.Town.objects.count())
 
 
 class WizardTestFormData(object):
diff --git a/ishtar_common/tests/insee-test.csv b/ishtar_common/tests/insee-test.csv
new file mode 100644
index 000000000..8845908ba
--- /dev/null
+++ b/ishtar_common/tests/insee-test.csv
@@ -0,0 +1,3 @@
+DepComN,NomCN,DepComA,NomCA,ChefLieu,ComDLG,Date1,Date2,Date3
+01015,Arboys en Bugey,75101,Arbignieu,O,O,29/09/2015,,24/12/2015
+01015,Arboys en Bugey,59350,Saint-Bois,N,O,,,