From a3aa47a3cc09cdc8d13a33f909aadc70cbbe3397 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 27 Oct 2017 17:51:52 +0200 Subject: Command import_insee_comm_csv: import town relations --- .../management/commands/import_insee_comm_csv.py | 106 +++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 ishtar_common/management/commands/import_insee_comm_csv.py (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py') diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py new file mode 100644 index 000000000..e64fe42bb --- /dev/null +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import csv +import re +import sys + +from django.core.management.base import BaseCommand + +from ishtar_common.models import Town + + +class Command(BaseCommand): + help = 'Import INSEE csv' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--year', type=int, default=2015, dest='year', + help='Year to affect to new towns') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + default_year = options['year'] + sys.stdout.write('* using year {} for new towns\n'.format(default_year)) + sys.stdout.write('* opening file {}\n'.format(csv_file)) + r = re.compile(r"(.*)\((.*)\)") + nb_created = 0 + nb_link = 0 + missing = [] + strange = [] + linked = set() + with open(csv_file, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() + + old_insee = row['DepComA'] + if len(old_insee) < 5: + old_insee = '0' + old_insee + q = Town.objects.filter(numero_insee=old_insee) + + if not q.count(): + missing.append((old_insee, row['NomCA'])) + continue + if q.count() > 1: + q = q.filter(year_lt=default_year).order_by('-year') + if not q.count(): + strange.append((old_insee, row['NomCA'])) + continue + old_town = q.all()[0] + + new_insee = row['DepComN'] + if len(new_insee) < 5: + new_insee = '0' + new_insee + q = Town.objects.filter(numero_insee=new_insee, + year=default_year) + if not q.count(): + nb_created += 1 + name = row['NomCN'].upper().strip() + name = r.sub(r"\2 \1", name).strip() + new_town = Town.objects.create(name=name, year=default_year, + numero_insee=new_insee) + else: + new_town = q.all()[0] + if new_town in old_town.children.all(): + continue # link already created + nb_link += 1 + old_town.children.add(new_town) + linked.add(new_town) + nb_limit = 0 + for town in linked: + if town.generate_geo(): + nb_limit += 1 + town.save() + sys.stdout.write('\n* {} town created\n'.format(nb_created)) + sys.stdout.write('* {} link created\n'.format(nb_link)) + sys.stdout.write('* {} limit generated\n'.format(nb_limit)) + if missing: + sys.stdout.write('* theses towns are missing:\n') + for insee, name in missing: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + if strange: + sys.stdout.write('* theses towns have newer version:\n') + for insee, name in strange: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + sys.stdout.flush() + + -- cgit v1.2.3 From 70c5566cf88ef14280d51df9f9f280bd1a3bb073 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 27 Oct 2017 18:07:15 +0200 Subject: Command import_insee_comm_csv: fix old town request - fix unicode issue --- ishtar_common/management/commands/import_insee_comm_csv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py') diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index e64fe42bb..d0fccb2fc 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -61,7 +61,7 @@ class Command(BaseCommand): missing.append((old_insee, row['NomCA'])) continue if q.count() > 1: - q = q.filter(year_lt=default_year).order_by('-year') + q = q.filter(year__lt=default_year).order_by('-year') if not q.count(): strange.append((old_insee, row['NomCA'])) continue @@ -74,7 +74,7 @@ class Command(BaseCommand): year=default_year) if not q.count(): nb_created += 1 - name = row['NomCN'].upper().strip() + name = row['NomCN'].decode('utf-8').upper().strip() name = r.sub(r"\2 \1", name).strip() new_town = Town.objects.create(name=name, year=default_year, numero_insee=new_insee) @@ -86,6 +86,7 @@ class Command(BaseCommand): old_town.children.add(new_town) linked.add(new_town) nb_limit = 0 + sys.stdout.write('\nGenerate limits...'.format(nb_created)) for town in linked: if town.generate_geo(): nb_limit += 1 -- cgit v1.2.3 From f793507ea489b2bb27eefa92bb1ec49c7d157967 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 15:57:49 +0200 Subject: Town: generate geo field on save --- .../management/commands/import_insee_comm_csv.py | 1 - ishtar_common/models.py | 29 ++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py') diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index d0fccb2fc..97b680267 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -90,7 +90,6 @@ class Command(BaseCommand): for town in linked: if town.generate_geo(): nb_limit += 1 - town.save() sys.stdout.write('\n* {} town created\n'.format(nb_created)) sys.stdout.write('* {} link created\n'.format(nb_link)) sys.stdout.write('* {} limit generated\n'.format(nb_limit)) diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 36da56cad..5dd9ddee1 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -85,6 +85,8 @@ def post_save_user(sender, **kwargs): except DatabaseError: # manage when db is not synced pass IshtarUser.set_superuser(user) + + post_save.connect(post_save_user, sender=User) @@ -2636,6 +2638,7 @@ class Source(OwnPerms, ImageModel, models.Model): if getattr(self, attr)] return slugify(u"-".join(values)) + if settings.COUNTRY == 'fr': class Arrondissement(models.Model): name = models.CharField(u"Nom", max_length=30) @@ -2689,6 +2692,10 @@ class Town(Imported, models.Model): return self.cached_label def generate_geo(self): + self.generate_limit() + self.generate_center() + + def generate_limit(self): if self.limit: return parents = None @@ -2704,8 +2711,20 @@ class Town(Imported, models.Model): # if union is a simple polygon make it a multi if 'MULTI' not in parents.wkt: parents = parents.wkt.replace('POLYGON', 'MULTIPOLYGON(') + ")" + if not parents: + return self.limit = parents - return self.limit + self.save() + return True + + def generate_center(self): + if self.center or not self.limit: + return + self.center = self.limit.centroid + if not self.center: + return False + self.save() + return True def _generate_cached_label(self): cached_label = self.name @@ -2716,7 +2735,13 @@ class Town(Imported, models.Model): return cached_label -post_save.connect(cached_label_changed, sender=Town) +def post_save_town(sender, **kwargs): + cached_label_changed(sender, **kwargs) + town = kwargs['instance'] + town.generate_geo() + + +post_save.connect(post_save_town, sender=Town) class OperationType(GeneralType): -- cgit v1.2.3 From 127b1c19794cd631a1a373ee2aaa8ef729d61ecb Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sat, 28 Oct 2017 17:39:46 +0200 Subject: Geofla import: manage new ADMIN EXPRESS format --- ishtar_common/management/commands/import_geofla_csv.py | 12 +++++++++--- ishtar_common/management/commands/import_insee_comm_csv.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py') diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py index 294219e9c..7ef0e0a38 100644 --- a/ishtar_common/management/commands/import_geofla_csv.py +++ b/ishtar_common/management/commands/import_geofla_csv.py @@ -65,11 +65,17 @@ class Command(BaseCommand): if 'MULTI' not in geom: geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' town.limit = GEOSGeometry(geom, srid=2154) - town.center = Point(float(row['X_CENTROID']), - float(row['Y_CENTROID']), srid=2154) + if 'X_CENTROID' in row: + town.center = Point(float(row['X_CENTROID']), + float(row['Y_CENTROID']), srid=2154) + else: + town.center = None if not town.year and default_year: town.year = default_year - town.surface = row['SUPERFICIE'] + if 'SUPERFICIE' in row: + town.surface = row['SUPERFICIE'] + else: + town.surface = None if not created: nb_changed += 1 town.save() diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index 97b680267..24eb2013e 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -74,7 +74,7 @@ class Command(BaseCommand): year=default_year) if not q.count(): nb_created += 1 - name = row['NomCN'].decode('utf-8').upper().strip() + name = row['NomCN'].decode('utf-8').strip() name = r.sub(r"\2 \1", name).strip() new_town = Town.objects.create(name=name, year=default_year, numero_insee=new_insee) -- cgit v1.2.3