diff options
Diffstat (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py')
| -rw-r--r-- | ishtar_common/management/commands/import_insee_comm_csv.py | 106 |
1 files changed, 106 insertions, 0 deletions
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py new file mode 100644 index 000000000..e64fe42bb --- /dev/null +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import csv +import re +import sys + +from django.core.management.base import BaseCommand + +from ishtar_common.models import Town + + +class Command(BaseCommand): + help = 'Import INSEE csv' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--year', type=int, default=2015, dest='year', + help='Year to affect to new towns') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + default_year = options['year'] + sys.stdout.write('* using year {} for new towns\n'.format(default_year)) + sys.stdout.write('* opening file {}\n'.format(csv_file)) + r = re.compile(r"(.*)\((.*)\)") + nb_created = 0 + nb_link = 0 + missing = [] + strange = [] + linked = set() + with open(csv_file, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() + + old_insee = row['DepComA'] + if len(old_insee) < 5: + old_insee = '0' + old_insee + q = Town.objects.filter(numero_insee=old_insee) + + if not q.count(): + missing.append((old_insee, row['NomCA'])) + continue + if q.count() > 1: + q = q.filter(year_lt=default_year).order_by('-year') + if not q.count(): + strange.append((old_insee, row['NomCA'])) + continue + old_town = q.all()[0] + + new_insee = row['DepComN'] + if len(new_insee) < 5: + new_insee = '0' + new_insee + q = Town.objects.filter(numero_insee=new_insee, + year=default_year) + if not q.count(): + nb_created += 1 + name = row['NomCN'].upper().strip() + name = r.sub(r"\2 \1", name).strip() + new_town = Town.objects.create(name=name, year=default_year, + numero_insee=new_insee) + else: + new_town = q.all()[0] + if new_town in old_town.children.all(): + continue # link already created + nb_link += 1 + old_town.children.add(new_town) + linked.add(new_town) + nb_limit = 0 + for town in linked: + if town.generate_geo(): + nb_limit += 1 + town.save() + sys.stdout.write('\n* {} town created\n'.format(nb_created)) + sys.stdout.write('* {} link created\n'.format(nb_link)) + sys.stdout.write('* {} limit generated\n'.format(nb_limit)) + if missing: + sys.stdout.write('* theses towns are missing:\n') + for insee, name in missing: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + if strange: + sys.stdout.write('* theses towns have newer version:\n') + for insee, name in strange: + sys.stdout.write('* {} ({})\n'.format(name, insee)) + sys.stdout.flush() + + |