From 9c780920b2773ac92e46ec07e29a3e489c8357f7 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 26 Oct 2017 18:29:00 +0200 Subject: Towns: remove "canton" add limits - change meta for data --- ishtar_common/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/models.py b/ishtar_common/models.py index c3ba4fdd0..a06568d99 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2659,12 +2659,11 @@ class Town(Imported, models.Model): surface = models.IntegerField(_(u"Surface (m2)"), blank=True, null=True) center = models.PointField(_(u"Localisation"), srid=settings.SRID, blank=True, null=True) + limit = models.MultiPolygonField(_(u"Limit"), blank=True, null=True) if settings.COUNTRY == 'fr': numero_insee = models.CharField(u"Numéro INSEE", max_length=6) departement = models.ForeignKey( Department, verbose_name=u"Département", null=True, blank=True) - canton = models.ForeignKey(Canton, verbose_name=u"Canton", null=True, - blank=True) year = models.IntegerField( _("Year of creation"), null=True, blank=True, help_text=_(u"Filling this field is relevant to distinguish old towns " -- cgit v1.2.3 From a3aa47a3cc09cdc8d13a33f909aadc70cbbe3397 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 27 Oct 2017 17:51:52 +0200 Subject: Command import_insee_comm_csv: import town relations --- .../management/commands/import_geofla_csv.py | 3 +- .../management/commands/import_insee_comm_csv.py | 106 +++++++++++++++++++++ ishtar_common/models.py | 19 ++++ ishtar_common/tests.py | 41 +++++++- ishtar_common/tests/insee-test.csv | 3 + 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 ishtar_common/management/commands/import_insee_comm_csv.py create mode 100644 ishtar_common/tests/insee-test.csv (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 28c2dabf5..1657006f6 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -37,8 +37,9 @@ class Command(BaseCommand): def handle(self, *args, **options): csv_file = options['csv_file'] - sys.stdout.write('* Opening file {}\n'.format(csv_file)) default_year = options['year'] + sys.stdout.write('* using year {} as a default\n'.format(default_year)) + sys.stdout.write('* Opening file {}\n'.format(csv_file)) nb_created, nb_changed = 0, 0 with open(csv_file, 'rb') as csvfile: reader = csv.DictReader(csvfile) diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py new file mode 100644 index 000000000..e64fe42bb --- /dev/null +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import csv +import re +import sys + +from django.core.management.base import BaseCommand + +from ishtar_common.models import Town + + +class Command(BaseCommand): + help = 'Import INSEE csv' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--year', type=int, default=2015, dest='year', + help='Year to affect to new towns') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + default_year = options['year'] + sys.stdout.write('* using year {} for new towns\n'.format(default_year)) + sys.stdout.write('* opening file {}\n'.format(csv_file)) + r = re.compile(r"(.*)\((.*)\)") + nb_created = 0 + nb_link = 0 + missing = [] + strange = [] + linked = set() + with open(csv_file, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() + + old_insee = row['DepComA'] + if len(old_insee) < 5: + old_insee = '0' + old_insee + q = Town.objects.filter(numero_insee=old_insee) + + if not q.count(): + missing.append((old_insee, row['NomCA'])) + continue + if q.count() > 1: + q = q.filter(year_lt=default_year).order_by('-year') + if not q.count(): + strange.append((old_insee, row['NomCA'])) + continue + old_town = q.all()[0] + + new_insee = row['DepComN'] + if len(new_insee) < 5: + new_insee = '0' + new_insee + q = Town.objects.filter(numero_insee=new_insee, + year=default_year) + if not q.count(): + nb_created += 1 + name = row['NomCN'].upper().strip() + name = r.sub(r"\2 \1", name).strip() + new_town = Town.objects.create(name=name, year=default_year, + numero_insee=new_insee) + else: + new_town = q.all()[0] + if new_town in old_town.children.all(): + continue # link already created + nb_link += 1 + old_town.children.add(new_town) + linked.add(new_town) + nb_limit = 0 + for town in linked: + if town.generate_geo(): + nb_limit += 1 + town.save() + sys.stdout.write('\n* {} town created\n'.format(nb_created)) + sys.stdout.write('* {} link created\n'.format(nb_link)) + sys.stdout.write('* {} limit generated\n'.format(nb_limit)) + if missing: + sys.stdout.write('* theses towns are missing:\n') + for insee, name in missing: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + if strange: + sys.stdout.write('* theses towns have newer version:\n') + for insee, name in strange: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + sys.stdout.flush() + + diff --git a/ishtar_common/models.py b/ishtar_common/models.py index a06568d99..36da56cad 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2688,6 +2688,25 @@ class Town(Imported, models.Model): self.save() return self.cached_label + def generate_geo(self): + if self.limit: + return + parents = None + if not self.parents.count(): + return + for parent in self.parents.all(): + if not parent.limit: + return + if not parents: + parents = parent.limit + else: + parents = parents.union(parent.limit) + # if union is a simple polygon make it a multi + if 'MULTI' not in parents.wkt: + parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")" + self.limit = parents + return self.limit + def _generate_cached_label(self): cached_label = self.name if settings.COUNTRY == "fr": diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 2b0a87386..4aa290ed1 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -92,6 +92,9 @@ class TestCase(BaseTestCase): class CommandsTestCase(TestCase): + fixtures = [settings.ROOT_PATH + + '../ishtar_common/fixtures/test_towns.json'] + def test_clean_ishtar(self): """ Clean ishtar db @@ -108,11 +111,47 @@ class CommandsTestCase(TestCase): self.assertEqual(Parcel.objects.filter(pk=p.pk).count(), 0) def test_import_geofla(self): - town_nb = models.Town.objects.count() + q = models.Town.objects + town_nb = q.count() out = StringIO() call_command('import_geofla_csv', '../ishtar_common/tests/geofla-test.csv', stdout=out) self.assertEqual(town_nb + 9, models.Town.objects.count()) + call_command('import_geofla_csv', + '../ishtar_common/tests/geofla-test.csv', stdout=out) + # no new town + self.assertEqual(town_nb + 9, models.Town.objects.count()) + + def test_import_insee(self): + q = models.Town.objects + town_nb = q.count() + first, union_start, union_end = '', '', [] + for idx, town in enumerate(q.all()): + l = 'MULTIPOLYGON((({x1} 1,{x2} 1,{x2} 0,{x1} 0,{x1} 1)))'.format( + x1=idx, x2=idx + 1) + if union_start: + union_start += ", " + else: + first = '{x1} 1'.format(x1=idx) + union_start += '{x2} 1'.format(x1=idx, x2=idx + 1) + union_end.append('{x2} 0'.format(x1=idx, x2=idx + 1)) + town.limit = l + town.save() + union = 'MULTIPOLYGON (((' + first + ", " + union_start + \ + ", " + ", ".join(reversed(union_end)) + ", 0 0, " + first + ")))" + out = StringIO() + call_command('import_insee_comm_csv', + '../ishtar_common/tests/insee-test.csv', stdout=out) + self.assertEqual(town_nb + 1, models.Town.objects.count()) + new = models.Town.objects.order_by('-pk').all()[0] + self.assertEqual(new.parents.count(), 2) + + self.assertEqual(new.limit.wkt, union) + + call_command('import_insee_comm_csv', + '../ishtar_common/tests/insee-test.csv', stdout=out) + # no new town + self.assertEqual(town_nb + 1, models.Town.objects.count()) class WizardTestFormData(object): diff --git a/ishtar_common/tests/insee-test.csv b/ishtar_common/tests/insee-test.csv new file mode 100644 index 000000000..8845908ba --- /dev/null +++ b/ishtar_common/tests/insee-test.csv @@ -0,0 +1,3 @@ +DepComN,NomCN,DepComA,NomCA,ChefLieu,ComDLG,Date1,Date2,Date3 +01015,Arboys en Bugey,75101,Arbignieu,O,O,29/09/2015,,24/12/2015 +01015,Arboys en Bugey,59350,Saint-Bois,N,O,,, -- cgit v1.2.3 From f793507ea489b2bb27eefa92bb1ec49c7d157967 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 15:57:49 +0200 Subject: Town: generate geo field on save --- .../management/commands/import_insee_comm_csv.py | 1 - ishtar_common/models.py | 29 ++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index d0fccb2fc..97b680267 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -90,7 +90,6 @@ class Command(BaseCommand): for town in linked: if town.generate_geo(): nb_limit += 1 - town.save() sys.stdout.write('\n* {} town created\n'.format(nb_created)) sys.stdout.write('* {} link created\n'.format(nb_link)) sys.stdout.write('* {} limit generated\n'.format(nb_limit)) diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 36da56cad..5dd9ddee1 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -85,6 +85,8 @@ def post_save_user(sender, **kwargs): except DatabaseError: # manage when db is not synced pass IshtarUser.set_superuser(user) + + post_save.connect(post_save_user, sender=User) @@ -2636,6 +2638,7 @@ class Source(OwnPerms, ImageModel, models.Model): if getattr(self, attr)] return slugify(u"-".join(values)) + if settings.COUNTRY == 'fr': class Arrondissement(models.Model): name = models.CharField(u"Nom", max_length=30) @@ -2689,6 +2692,10 @@ class Town(Imported, models.Model): return self.cached_label def generate_geo(self): + self.generate_limit() + self.generate_center() + + def generate_limit(self): if self.limit: return parents = None @@ -2704,8 +2711,20 @@ class Town(Imported, models.Model): # if union is a simple polygon make it a multi if 'MULTI' not in parents.wkt: parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")" + if not parents: + return self.limit = parents - return self.limit + self.save() + return True + + def generate_center(self): + if self.center or not self.limit: + return + self.center = self.limit.centroid + if not self.center: + return False + self.save() + return True def _generate_cached_label(self): cached_label = self.name @@ -2716,7 +2735,13 @@ class Town(Imported, models.Model): return cached_label -post_save.connect(cached_label_changed, sender=Town) +def post_save_town(sender, **kwargs): + cached_label_changed(sender, **kwargs) + town = kwargs['instance'] + town.generate_geo() + + +post_save.connect(post_save_town, sender=Town) class OperationType(GeneralType): -- cgit v1.2.3 From 5383bbb4e17432929aeb4ee412d71a3d36297a9a Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 17:23:51 +0200 Subject: Towns: manage natural keys --- ishtar_common/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 5dd9ddee1..f0552d3eb 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2657,6 +2657,11 @@ if settings.COUNTRY == 'fr': (self.name, unicode(self.arrondissement))) +class TownManager(models.Manager): + def get_by_natural_key(self, numero_insee, year): + return self.get(numero_insee=numero_insee, year=year) + + class Town(Imported, models.Model): name = models.CharField(_(u"Name"), max_length=100) surface = models.IntegerField(_(u"Surface (m2)"), blank=True, null=True) @@ -2685,6 +2690,9 @@ class Town(Imported, models.Model): ordering = ['numero_insee'] unique_together = (('numero_insee', 'year'),) + def natural_key(self): + return (self.numero_insee, self.year) + def __unicode__(self): if self.cached_label: return self.cached_label -- cgit v1.2.3 From a31a572f0f6ad9401c8cef1a4a24672e248076e4 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 17:47:55 +0200 Subject: Town: allow to force geom regeneration --- ishtar_common/models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/models.py b/ishtar_common/models.py index f0552d3eb..c5d0750b9 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2699,12 +2699,12 @@ class Town(Imported, models.Model): self.save() return self.cached_label - def generate_geo(self): - self.generate_limit() - self.generate_center() + def generate_geo(self, force=False): + force = self.generate_limit(force=force) + self.generate_center(force=force) - def generate_limit(self): - if self.limit: + def generate_limit(self, force=False): + if not force and self.limit: return parents = None if not self.parents.count(): @@ -2725,8 +2725,8 @@ class Town(Imported, models.Model): self.save() return True - def generate_center(self): - if self.center or not self.limit: + def generate_center(self, force=False): + if not force and (self.center or not self.limit): return self.center = self.limit.centroid if not self.center: -- cgit v1.2.3 From e584b93f3da28f1f4df2017ead404855a8b22c92 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 18:47:27 +0200 Subject: Towns: fix custom manager --- ishtar_common/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ishtar_common/models.py') diff --git a/ishtar_common/models.py b/ishtar_common/models.py index c5d0750b9..703d4ccf7 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2657,7 +2657,7 @@ if settings.COUNTRY == 'fr': (self.name, unicode(self.arrondissement))) -class TownManager(models.Manager): +class TownManager(models.GeoManager): def get_by_natural_key(self, numero_insee, year): return self.get(numero_insee=numero_insee, year=year) @@ -2681,7 +2681,7 @@ class Town(Imported, models.Model): related_name='parents') cached_label = models.CharField(_(u"Cached name"), max_length=500, null=True, blank=True, db_index=True) - objects = models.GeoManager() + objects = TownManager() class Meta: verbose_name = _(u"Town") -- cgit v1.2.3 From d45ba08e3e94c1f4d56c6f46feac9442a1d2209f Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sun, 29 Oct 2017 15:39:40 +0100 Subject: Town: generate area --- example_project/settings.py | 1 + ishtar_common/models.py | 11 +++++++++++ 2 files changed, 12 insertions(+) (limited to 'ishtar_common/models.py') diff --git a/example_project/settings.py b/example_project/settings.py index f631047cb..84462b396 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -213,6 +213,7 @@ LOGGING = { # Ishtar custom SRID = 27572 +SURFACE_SRID = 2154 ENCODING = 'windows-1252' ALT_ENCODING = 'ISO-8859-15' APP_NAME = "SRA - Pays de la Loire" diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 703d4ccf7..14869e2a5 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -2702,6 +2702,7 @@ class Town(Imported, models.Model): def generate_geo(self, force=False): force = self.generate_limit(force=force) self.generate_center(force=force) + self.generate_area(force=force) def generate_limit(self, force=False): if not force and self.limit: @@ -2734,6 +2735,16 @@ class Town(Imported, models.Model): self.save() return True + def generate_area(self, force=False): + if not force and (self.surface or not self.limit): + return + self.surface = self.limit.transform(settings.SURFACE_SRID, + clone=True).area + if not self.surface: + return False + self.save() + return True + def _generate_cached_label(self): cached_label = self.name if settings.COUNTRY == "fr": -- cgit v1.2.3