summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ishtar_common/management/commands/import_geofla_csv.py3
-rw-r--r--ishtar_common/management/commands/import_insee_comm_csv.py106
-rw-r--r--ishtar_common/models.py19
-rw-r--r--ishtar_common/tests.py41
-rw-r--r--ishtar_common/tests/insee-test.csv3
5 files changed, 170 insertions, 2 deletions
diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py
index 28c2dabf5..1657006f6 100644
--- a/ishtar_common/management/commands/import_geofla_csv.py
+++ b/ishtar_common/management/commands/import_geofla_csv.py
@@ -37,8 +37,9 @@ class Command(BaseCommand):
def handle(self, *args, **options):
csv_file = options['csv_file']
- sys.stdout.write('* Opening file {}\n'.format(csv_file))
default_year = options['year']
+ sys.stdout.write('* using year {} as a default\n'.format(default_year))
+ sys.stdout.write('* Opening file {}\n'.format(csv_file))
nb_created, nb_changed = 0, 0
with open(csv_file, 'rb') as csvfile:
reader = csv.DictReader(csvfile)
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
new file mode 100644
index 000000000..e64fe42bb
--- /dev/null
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import re
+import sys
+
+from django.core.management.base import BaseCommand
+
+from ishtar_common.models import Town
+
+
+class Command(BaseCommand):
+ help = 'Import INSEE csv'
+
+ def add_arguments(self, parser):
+ parser.add_argument('csv_file')
+ parser.add_argument(
+ '--year', type=int, default=2015, dest='year',
+ help='Year to affect to new towns')
+
+ def handle(self, *args, **options):
+ csv_file = options['csv_file']
+ default_year = options['year']
+ sys.stdout.write('* using year {} for new towns\n'.format(default_year))
+ sys.stdout.write('* opening file {}\n'.format(csv_file))
+ r = re.compile(r"(.*)\((.*)\)")
+ nb_created = 0
+ nb_link = 0
+ missing = []
+ strange = []
+ linked = set()
+ with open(csv_file, 'rb') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for idx, row in enumerate(reader):
+ sys.stdout.write('Processing town %d.\r' % (idx + 1))
+ sys.stdout.flush()
+
+ old_insee = row['DepComA']
+ if len(old_insee) < 5:
+ old_insee = '0' + old_insee
+ q = Town.objects.filter(numero_insee=old_insee)
+
+ if not q.count():
+ missing.append((old_insee, row['NomCA']))
+ continue
+ if q.count() > 1:
+ q = q.filter(year_lt=default_year).order_by('-year')
+ if not q.count():
+ strange.append((old_insee, row['NomCA']))
+ continue
+ old_town = q.all()[0]
+
+ new_insee = row['DepComN']
+ if len(new_insee) < 5:
+ new_insee = '0' + new_insee
+ q = Town.objects.filter(numero_insee=new_insee,
+ year=default_year)
+ if not q.count():
+ nb_created += 1
+ name = row['NomCN'].upper().strip()
+ name = r.sub(r"\2 \1", name).strip()
+ new_town = Town.objects.create(name=name, year=default_year,
+ numero_insee=new_insee)
+ else:
+ new_town = q.all()[0]
+ if new_town in old_town.children.all():
+ continue # link already created
+ nb_link += 1
+ old_town.children.add(new_town)
+ linked.add(new_town)
+ nb_limit = 0
+ for town in linked:
+ if town.generate_geo():
+ nb_limit += 1
+ town.save()
+ sys.stdout.write('\n* {} town created\n'.format(nb_created))
+ sys.stdout.write('* {} link created\n'.format(nb_link))
+ sys.stdout.write('* {} limit generated\n'.format(nb_limit))
+ if missing:
+ sys.stdout.write('* theses towns are missing:\n')
+ for insee, name in missing:
+ sys.stdout.write('* {} ({})\n'.format(name, insee))
+ if strange:
+ sys.stdout.write('* theses towns have newer version:\n')
+ for insee, name in strange:
+ sys.stdout.write('* {} ({})\n'.format(name, insee))
+ sys.stdout.flush()
+
+
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a06568d99..36da56cad 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -2688,6 +2688,25 @@ class Town(Imported, models.Model):
self.save()
return self.cached_label
+ def generate_geo(self):
+ if self.limit:
+ return
+ parents = None
+ if not self.parents.count():
+ return
+ for parent in self.parents.all():
+ if not parent.limit:
+ return
+ if not parents:
+ parents = parent.limit
+ else:
+ parents = parents.union(parent.limit)
+ # if union is a simple polygon make it a multi
+ if 'MULTI' not in parents.wkt:
+ parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")"
+ self.limit = parents
+ return self.limit
+
def _generate_cached_label(self):
cached_label = self.name
if settings.COUNTRY == "fr":
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 2b0a87386..4aa290ed1 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -92,6 +92,9 @@ class TestCase(BaseTestCase):
class CommandsTestCase(TestCase):
+ fixtures = [settings.ROOT_PATH +
+ '../ishtar_common/fixtures/test_towns.json']
+
def test_clean_ishtar(self):
"""
Clean ishtar db
@@ -108,11 +111,47 @@ class CommandsTestCase(TestCase):
self.assertEqual(Parcel.objects.filter(pk=p.pk).count(), 0)
def test_import_geofla(self):
- town_nb = models.Town.objects.count()
+ q = models.Town.objects
+ town_nb = q.count()
out = StringIO()
call_command('import_geofla_csv',
'../ishtar_common/tests/geofla-test.csv', stdout=out)
self.assertEqual(town_nb + 9, models.Town.objects.count())
+ call_command('import_geofla_csv',
+ '../ishtar_common/tests/geofla-test.csv', stdout=out)
+ # no new town
+ self.assertEqual(town_nb + 9, models.Town.objects.count())
+
+ def test_import_insee(self):
+ q = models.Town.objects
+ town_nb = q.count()
+ first, union_start, union_end = '', '', []
+ for idx, town in enumerate(q.all()):
+ l = 'MULTIPOLYGON((({x1} 1,{x2} 1,{x2} 0,{x1} 0,{x1} 1)))'.format(
+ x1=idx, x2=idx + 1)
+ if union_start:
+ union_start += ", "
+ else:
+ first = '{x1} 1'.format(x1=idx)
+ union_start += '{x2} 1'.format(x1=idx, x2=idx + 1)
+ union_end.append('{x2} 0'.format(x1=idx, x2=idx + 1))
+ town.limit = l
+ town.save()
+ union = 'MULTIPOLYGON (((' + first + ", " + union_start + \
+ ", " + ", ".join(reversed(union_end)) + ", 0 0, " + first + ")))"
+ out = StringIO()
+ call_command('import_insee_comm_csv',
+ '../ishtar_common/tests/insee-test.csv', stdout=out)
+ self.assertEqual(town_nb + 1, models.Town.objects.count())
+ new = models.Town.objects.order_by('-pk').all()[0]
+ self.assertEqual(new.parents.count(), 2)
+
+ self.assertEqual(new.limit.wkt, union)
+
+ call_command('import_insee_comm_csv',
+ '../ishtar_common/tests/insee-test.csv', stdout=out)
+ # no new town
+ self.assertEqual(town_nb + 1, models.Town.objects.count())
class WizardTestFormData(object):
diff --git a/ishtar_common/tests/insee-test.csv b/ishtar_common/tests/insee-test.csv
new file mode 100644
index 000000000..8845908ba
--- /dev/null
+++ b/ishtar_common/tests/insee-test.csv
@@ -0,0 +1,3 @@
+DepComN,NomCN,DepComA,NomCA,ChefLieu,ComDLG,Date1,Date2,Date3
+01015,Arboys en Bugey,75101,Arbignieu,O,O,29/09/2015,,24/12/2015
+01015,Arboys en Bugey,59350,Saint-Bois,N,O,,,
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-16 11:01:56 +0000