From b6676aa6ad94739743ac613e2f19a8b29b9705eb Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 26 Oct 2017 20:58:21 +0200 Subject: Command: import geofla csv --- .../management/commands/import_geofla_csv.py | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 ishtar_common/management/commands/import_geofla_csv.py (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py new file mode 100644 index 000000000..b7cb2b604 --- /dev/null +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import csv + +from django.core.management.base import BaseCommand +from django.contrib.gis.geos import GEOSGeometry, Point + +from ishtar_common.models import Town + + +class Command(BaseCommand): + help = 'Import GEOFLA csv' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--year', type=int, default=2014, dest='year', + help='Default year to affect to the town') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + self.stdout.write('* Opening file {}\n'.format(csv_file)) + default_year = options['year'] + nb_created, nb_changed = 0, 0 + with open(csv_file, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + self.stdout.write('Processing town %d.\r' % (idx + 1)) + self.stdout.flush() + num_insee = row['INSEE_COM'] + if len(num_insee) < 5: + num_insee = '0' + num_insee + q = Town.objects.filter(numero_insee=num_insee) + changed, created = False, False + if q.count(): + town = q.all()[0] + else: + changed = True + created = True + nb_created += 1 + town = Town(name=row['NOM_COM'], + numero_insee=num_insee) + if not town.limit: + changed = True + geom = row['wkt_geom'].upper() + if 'MULTI' not in geom: + geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' + town.limit = GEOSGeometry(geom, srid=2154) + if not town.center: + changed = True + town.center = Point(float(row['X_CENTROID']), + float(row['Y_CENTROID']), srid=2154) + if not town.year and default_year: + changed = True + town.year = default_year + if not town.surface: + changed = True + town.surface = row['SUPERFICIE'] + if changed: + if not created: + nb_changed += 1 + town.save() + self.stdout.write('\n* {} town created'.format(nb_created)) + self.stdout.write('\n* {} town changed\n'.format(nb_changed)) + self.stdout.flush() + + + -- cgit v1.2.3 From 4be73e1e4cbcab26e71abbd973dd982055aa20fa Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 26 Oct 2017 22:52:23 +0200 Subject: Fix tests for towns --- ishtar_common/fixtures/test_towns.json | 2 -- ishtar_common/management/commands/import_geofla_csv.py | 13 +++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/fixtures/test_towns.json b/ishtar_common/fixtures/test_towns.json index a88ca6476..d34c10411 100644 --- a/ishtar_common/fixtures/test_towns.json +++ b/ishtar_common/fixtures/test_towns.json @@ -7,7 +7,6 @@ "name": "PARIS-1ER-ARRONDISSEMENT", "center": "POINT (599976.9837326267734170 2429351.2226647692732513)", "surface": 1810000, - "canton": null, "numero_insee": "75101" } }, @@ -19,7 +18,6 @@ "name": "LILLE", "center": "POINT (650348.5204579939600080 2626592.6267738011665642)", "surface": 34990000, - "canton": null, "numero_insee": "59350" } } diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index b7cb2b604..80e10bc81 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -18,6 +18,7 @@ # See the file COPYING for details. import csv +import sys from django.core.management.base import BaseCommand from django.contrib.gis.geos import GEOSGeometry, Point @@ -36,14 +37,14 @@ class Command(BaseCommand): def handle(self, *args, **options): csv_file = options['csv_file'] - self.stdout.write('* Opening file {}\n'.format(csv_file)) + sys.stdout.write('* Opening file {}\n'.format(csv_file)) default_year = options['year'] nb_created, nb_changed = 0, 0 with open(csv_file, 'rb') as csvfile: reader = csv.DictReader(csvfile) for idx, row in enumerate(reader): - self.stdout.write('Processing town %d.\r' % (idx + 1)) - self.stdout.flush() + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() num_insee = row['INSEE_COM'] if len(num_insee) < 5: num_insee = '0' + num_insee @@ -77,9 +78,9 @@ class Command(BaseCommand): if not created: nb_changed += 1 town.save() - self.stdout.write('\n* {} town created'.format(nb_created)) - self.stdout.write('\n* {} town changed\n'.format(nb_changed)) - self.stdout.flush() + sys.stdout.write('\n* {} town created'.format(nb_created)) + sys.stdout.write('\n* {} town changed\n'.format(nb_changed)) + sys.stdout.flush() -- cgit v1.2.3 From 72a1783b7617ce501f598b7d6cc5868a15ff5fe5 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 27 Oct 2017 09:50:31 +0200 Subject: Command import_geofla_csv: manage town request with year --- ishtar_common/management/commands/import_geofla_csv.py | 5 ++++- ishtar_common/tests.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 80e10bc81..28c2dabf5 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -51,7 +51,10 @@ class Command(BaseCommand): q = Town.objects.filter(numero_insee=num_insee) changed, created = False, False if q.count(): - town = q.all()[0] + if q.filter(year=default_year).count(): + town = q.filter(year=default_year).all()[0] + else: + town = q.order_by('-year').all()[0] else: changed = True created = True diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index d882cd85b..2b0a87386 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -115,7 +115,6 @@ class CommandsTestCase(TestCase): self.assertEqual(town_nb + 9, models.Town.objects.count()) - class WizardTestFormData(object): """ Test set to simulate wizard steps -- cgit v1.2.3 From c8d3021cee3397e200df30eac12984b5770adb6c Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 27 Oct 2017 17:51:52 +0200 Subject: Command import_insee_comm_csv: import town relations --- .../management/commands/import_geofla_csv.py | 3 +- .../management/commands/import_insee_comm_csv.py | 106 +++++++++++++++++++++ ishtar_common/models.py | 19 ++++ ishtar_common/tests.py | 41 +++++++- ishtar_common/tests/insee-test.csv | 3 + 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 ishtar_common/management/commands/import_insee_comm_csv.py create mode 100644 ishtar_common/tests/insee-test.csv (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 28c2dabf5..1657006f6 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -37,8 +37,9 @@ class Command(BaseCommand): def handle(self, *args, **options): csv_file = options['csv_file'] - sys.stdout.write('* Opening file {}\n'.format(csv_file)) default_year = options['year'] + sys.stdout.write('* using year {} as a default\n'.format(default_year)) + sys.stdout.write('* Opening file {}\n'.format(csv_file)) nb_created, nb_changed = 0, 0 with open(csv_file, 'rb') as csvfile: reader = csv.DictReader(csvfile) diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py new file mode 100644 index 000000000..e64fe42bb --- /dev/null +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import csv +import re +import sys + +from django.core.management.base import BaseCommand + +from ishtar_common.models import Town + + +class Command(BaseCommand): + help = 'Import INSEE csv' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--year', type=int, default=2015, dest='year', + help='Year to affect to new towns') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + default_year = options['year'] + sys.stdout.write('* using year {} for new towns\n'.format(default_year)) + sys.stdout.write('* opening file {}\n'.format(csv_file)) + r = re.compile(r"(.*)\((.*)\)") + nb_created = 0 + nb_link = 0 + missing = [] + strange = [] + linked = set() + with open(csv_file, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() + + old_insee = row['DepComA'] + if len(old_insee) < 5: + old_insee = '0' + old_insee + q = Town.objects.filter(numero_insee=old_insee) + + if not q.count(): + missing.append((old_insee, row['NomCA'])) + continue + if q.count() > 1: + q = q.filter(year_lt=default_year).order_by('-year') + if not q.count(): + strange.append((old_insee, row['NomCA'])) + continue + old_town = q.all()[0] + + new_insee = row['DepComN'] + if len(new_insee) < 5: + new_insee = '0' + new_insee + q = Town.objects.filter(numero_insee=new_insee, + year=default_year) + if not q.count(): + nb_created += 1 + name = row['NomCN'].upper().strip() + name = r.sub(r"\2 \1", name).strip() + new_town = Town.objects.create(name=name, year=default_year, + numero_insee=new_insee) + else: + new_town = q.all()[0] + if new_town in old_town.children.all(): + continue # link already created + nb_link += 1 + old_town.children.add(new_town) + linked.add(new_town) + nb_limit = 0 + for town in linked: + if town.generate_geo(): + nb_limit += 1 + town.save() + sys.stdout.write('\n* {} town created\n'.format(nb_created)) + sys.stdout.write('* {} link created\n'.format(nb_link)) + sys.stdout.write('* {} limit generated\n'.format(nb_limit)) + if missing: + sys.stdout.write('* theses towns are missing:\n') + for insee, name in missing: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + if strange: + sys.stdout.write('* theses towns have newer version:\n') + for insee, name in strange: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + sys.stdout.flush() + + diff --git a/ishtar_common/models.py b/ishtar_common/models.py index a06568d99..36da56cad 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2688,6 +2688,25 @@ class Town(Imported, models.Model): self.save() return self.cached_label + def generate_geo(self): + if self.limit: + return + parents = None + if not self.parents.count(): + return + for parent in self.parents.all(): + if not parent.limit: + return + if not parents: + parents = parent.limit + else: + parents = parents.union(parent.limit) + # if union is a simple polygon make it a multi + if 'MULTI' not in parents.wkt: + parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")" + self.limit = parents + return self.limit + def _generate_cached_label(self): cached_label = self.name if settings.COUNTRY == "fr": diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 2b0a87386..4aa290ed1 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -92,6 +92,9 @@ class TestCase(BaseTestCase): class CommandsTestCase(TestCase): + fixtures = [settings.ROOT_PATH + + '../ishtar_common/fixtures/test_towns.json'] + def test_clean_ishtar(self): """ Clean ishtar db @@ -108,11 +111,47 @@ class CommandsTestCase(TestCase): self.assertEqual(Parcel.objects.filter(pk=p.pk).count(), 0) def test_import_geofla(self): - town_nb = models.Town.objects.count() + q = models.Town.objects + town_nb = q.count() out = StringIO() call_command('import_geofla_csv', '../ishtar_common/tests/geofla-test.csv', stdout=out) self.assertEqual(town_nb + 9, models.Town.objects.count()) + call_command('import_geofla_csv', + '../ishtar_common/tests/geofla-test.csv', stdout=out) + # no new town + self.assertEqual(town_nb + 9, models.Town.objects.count()) + + def test_import_insee(self): + q = models.Town.objects + town_nb = q.count() + first, union_start, union_end = '', '', [] + for idx, town in enumerate(q.all()): + l = 'MULTIPOLYGON((({x1} 1,{x2} 1,{x2} 0,{x1} 0,{x1} 1)))'.format( + x1=idx, x2=idx + 1) + if union_start: + union_start += ", " + else: + first = '{x1} 1'.format(x1=idx) + union_start += '{x2} 1'.format(x1=idx, x2=idx + 1) + union_end.append('{x2} 0'.format(x1=idx, x2=idx + 1)) + town.limit = l + town.save() + union = 'MULTIPOLYGON (((' + first + ", " + union_start + \ + ", " + ", ".join(reversed(union_end)) + ", 0 0, " + first + ")))" + out = StringIO() + call_command('import_insee_comm_csv', + '../ishtar_common/tests/insee-test.csv', stdout=out) + self.assertEqual(town_nb + 1, models.Town.objects.count()) + new = models.Town.objects.order_by('-pk').all()[0] + self.assertEqual(new.parents.count(), 2) + + self.assertEqual(new.limit.wkt, union) + + call_command('import_insee_comm_csv', + '../ishtar_common/tests/insee-test.csv', stdout=out) + # no new town + self.assertEqual(town_nb + 1, models.Town.objects.count()) class WizardTestFormData(object): diff --git a/ishtar_common/tests/insee-test.csv b/ishtar_common/tests/insee-test.csv new file mode 100644 index 000000000..8845908ba --- /dev/null +++ b/ishtar_common/tests/insee-test.csv @@ -0,0 +1,3 @@ +DepComN,NomCN,DepComA,NomCA,ChefLieu,ComDLG,Date1,Date2,Date3 +01015,Arboys en Bugey,75101,Arbignieu,O,O,29/09/2015,,24/12/2015 +01015,Arboys en Bugey,59350,Saint-Bois,N,O,,, -- cgit v1.2.3 From e78845fcf64aacb72fb0edd350d9b05bdd28fdef Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 16:03:35 +0200 Subject: Geofla import: always update geometry --- .../management/commands/import_geofla_csv.py | 31 ++++++++-------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 1657006f6..294219e9c 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -50,38 +50,29 @@ class Command(BaseCommand): if len(num_insee) < 5: num_insee = '0' + num_insee q = Town.objects.filter(numero_insee=num_insee) - changed, created = False, False + created = False if q.count(): if q.filter(year=default_year).count(): town = q.filter(year=default_year).all()[0] else: town = q.order_by('-year').all()[0] else: - changed = True created = True nb_created += 1 town = Town(name=row['NOM_COM'], numero_insee=num_insee) - if not town.limit: - changed = True - geom = row['wkt_geom'].upper() - if 'MULTI' not in geom: - geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' - town.limit = GEOSGeometry(geom, srid=2154) - if not town.center: - changed = True - town.center = Point(float(row['X_CENTROID']), - float(row['Y_CENTROID']), srid=2154) + geom = row['wkt_geom'].upper() + if 'MULTI' not in geom: + geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' + town.limit = GEOSGeometry(geom, srid=2154) + town.center = Point(float(row['X_CENTROID']), + float(row['Y_CENTROID']), srid=2154) if not town.year and default_year: - changed = True town.year = default_year - if not town.surface: - changed = True - town.surface = row['SUPERFICIE'] - if changed: - if not created: - nb_changed += 1 - town.save() + town.surface = row['SUPERFICIE'] + if not created: + nb_changed += 1 + town.save() sys.stdout.write('\n* {} town created'.format(nb_created)) sys.stdout.write('\n* {} town changed\n'.format(nb_changed)) sys.stdout.flush() -- cgit v1.2.3 From 4dbceff7ef3c52cdcebc4a0c0547d538a0a0ffd6 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 17:39:46 +0200 Subject: Geofla import: manage new ADMIN EXPRESS format --- ishtar_common/management/commands/import_geofla_csv.py | 12 +++++++++--- ishtar_common/management/commands/import_insee_comm_csv.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'ishtar_common/management/commands/import_geofla_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 294219e9c..7ef0e0a38 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -65,11 +65,17 @@ class Command(BaseCommand): if 'MULTI' not in geom: geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' town.limit = GEOSGeometry(geom, srid=2154) - town.center = Point(float(row['X_CENTROID']), - float(row['Y_CENTROID']), srid=2154) + if 'X_CENTROID' in row: + town.center = Point(float(row['X_CENTROID']), + float(row['Y_CENTROID']), srid=2154) + else: + town.center = None if not town.year and default_year: town.year = default_year - town.surface = row['SUPERFICIE'] + if 'SUPERFICIE' in row: + town.surface = row['SUPERFICIE'] + else: + town.surface = None if not created: nb_changed += 1 town.save() diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index 97b680267..24eb2013e 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -74,7 +74,7 @@ class Command(BaseCommand): year=default_year) if not q.count(): nb_created += 1 - name = row['NomCN'].decode('utf-8').upper().strip() + name = row['NomCN'].decode('utf-8').strip() name = r.sub(r"\2 \1", name).strip() new_town = Town.objects.create(name=name, year=default_year, numero_insee=new_insee) -- cgit v1.2.3