#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (C) 2017 Étienne Loks # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # See the file COPYING for details. import csv import re import sys from django.core.management.base import BaseCommand from ishtar_common.models import Town class Command(BaseCommand): help = 'Import INSEE csv' def add_arguments(self, parser): parser.add_argument('csv_file') parser.add_argument( '--year', type=int, default=2015, dest='year', help='Year to affect to new towns') parser.add_argument( '--quiet', dest='quiet', action='store_true', help='Quiet output') def handle(self, *args, **options): csv_file = options['csv_file'] default_year = options['year'] quiet = options['quiet'] if not quiet: sys.stdout.write('* using year {} for new towns\n'.format( default_year)) sys.stdout.write('* opening file {}\n'.format(csv_file)) r = re.compile(r"(.*)\((.*)\)") nb_created = 0 nb_link = 0 missing = [] strange = [] linked = set() with open(csv_file, 'rt') as csvfile: reader = csv.DictReader(csvfile) for idx, row in enumerate(reader): new_insee = row['DepComN'] if len(new_insee) < 5: new_insee = '0' + new_insee if not idx: # test if first do not exist q = Town.objects.filter(numero_insee=new_insee, year=default_year) if q.count(): print("First town already exists for this year....") return if not quiet: sys.stdout.write('Processing town %d.\r' % (idx + 1)) sys.stdout.flush() old_insee = row['DepComA'] if len(old_insee) < 5: old_insee = '0' + old_insee q = Town.objects.filter(numero_insee=old_insee) if not q.count(): missing.append((old_insee, row['NomCA'])) continue if q.count() > 1: q = q.filter(year__lt=default_year).order_by('-year') if not q.count(): strange.append((old_insee, row['NomCA'])) continue old_town = q.all()[0] q = Town.objects.filter(numero_insee=new_insee, year=default_year) if not q.count(): nb_created += 1 name = row['NomCN'].strip() name = r.sub(r"\2 \1", name).strip() new_town = Town.objects.create(name=name, year=default_year, numero_insee=new_insee) else: new_town = q.all()[0] if new_town in old_town.children.all(): continue # link already created nb_link += 1 old_town.children.add(new_town) linked.add(new_town) nb_limit = 0 if not quiet: sys.stdout.write('\nGenerate limits...'.format(nb_created)) for town in linked: if town.generate_geo(): nb_limit += 1 if quiet: return sys.stdout.write('\n* {} town created\n'.format(nb_created)) sys.stdout.write('* {} link created\n'.format(nb_link)) sys.stdout.write('* {} limit generated\n'.format(nb_limit)) if missing: sys.stdout.write('* these towns are missing:\n') for insee, name in missing: sys.stdout.write('* {} ({})\n'.format(name, insee)) if strange: sys.stdout.write('* these towns have newer version:\n') for insee, name in strange: sys.stdout.write('* {} ({})\n'.format(name, insee)) sys.stdout.flush()