#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (C) 2017 Étienne Loks # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # See the file COPYING for details. import csv import sys from django.core.management.base import BaseCommand from django.contrib.gis.geos import GEOSGeometry, Point from django.db import transaction from django.db.utils import DataError from django.contrib.contenttypes.models import ContentType from ishtar_common.models import Town, GeoVectorData, GeoDataType, GeoProviderType town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town") data_type, __ = GeoDataType.objects.get_or_create( txt_idx="town-limit", defaults={"label": "Limites commune"} ) provider, __ = GeoProviderType.objects.get_or_create( txt_idx="france-ign", defaults={"label": "IGN"} ) class Command(BaseCommand): help = 'Import GEOFLA csv' def add_arguments(self, parser): parser.add_argument('csv_file') parser.add_argument( '--year', type=int, default=2014, dest='year', help='Default year to affect to the town') parser.add_argument( '--quiet', dest='quiet', action='store_true', help='Quiet output') parser.add_argument( '--create-only', dest='create_only', action='store_true', help='Create only missing geo') parser.add_argument( '--srid', type=int, default=2154, dest='srid', help='SRID uses. Default: 2154.') def get_town(self, num_insee, name, default_year): q = Town.objects.filter(numero_insee=num_insee) created = False if q.count(): if q.filter(year=default_year).count(): town = q.filter(year=default_year).all()[0] else: town = q.order_by('-year').all()[0] else: created = True town = Town(name=name, numero_insee=num_insee) return town, created @transaction.atomic def handle(self, *args, **options): csv.field_size_limit(sys.maxsize) csv_file = options['csv_file'] default_year = options['year'] srid = options['srid'] quiet = options['quiet'] create_only = options["create_only"] if not quiet: sys.stdout.write('* using year {} as a default\n'.format( default_year)) sys.stdout.write('* Opening file {}\n'.format(csv_file)) nb_created, nb_changed, nb_error, nb_geo = 0, 0, 0, 0 with open(csv_file, 'rt') as csvfile: header = csvfile.readline() geom_colum = header.split(",")[0] csvfile.seek(0) reader = csv.DictReader(csvfile) for idx, row in enumerate(reader): if not quiet: sys.stdout.write('Processing town %d.\r' % (idx + 1)) sys.stdout.flush() num_insee = row['INSEE_COM'] if len(num_insee) < 5: num_insee = '0' + num_insee if 'NOM_COM_M' in row: name = row['NOM_COM_M'] elif 'NOM_M' in row: name = row['NOM_M'] elif 'NOM' in row: name = row['NOM'].upper() else: name = row['NOM_COM'].upper() town, created = self.get_town(num_insee, name, default_year) if created: nb_created += 1 else: nb_changed += 1 if create_only and town.main_geodata: continue geom = row[geom_colum].upper() if 'MULTI' not in geom: geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')' limit = GEOSGeometry(geom, srid=srid) if 'X_CENTROID' in row: center = Point( float(row['X_CENTROID']), float(row['Y_CENTROID']), srid=srid) else: center = None values = {} values['center'] = None if not town.year and default_year: values['year'] = default_year if 'SUPERFICIE' in row: values['surface'] = row['SUPERFICIE'] else: values['surface'] = None for k in values: setattr(town, k, values[k]) try: with transaction.atomic(): town.save() except DataError: nb_error += 1 town, created = self.get_town(num_insee, name, default_year) for k in values: setattr(town, k, values[k]) town.save() attrs = { "name": town._generate_cached_label(), "source_content_type": town_content_type, "source_id": town.pk, "data_type": data_type, "provider": provider, } if limit: attrs["multi_polygon"] = limit else: attrs["point_2d"] = center try: data, created = GeoVectorData.objects.get_or_create(**attrs) except DataError: if limit: print(f"\nError {town} polygon\n") attrs.pop("multi_polygon") attrs["point_2d"] = center data, created = GeoVectorData.objects.get_or_create(**attrs) else: print(f"\nError {town} center\n") town.main_geodata = data town._post_save_geo_ok = False town.save() if created: nb_geo += 1 if quiet: return sys.stdout.write('\n* {} town created'.format(nb_created)) sys.stdout.write('\n* {} geo created'.format(nb_geo)) sys.stdout.write('\n* {} town changed'.format(nb_changed)) sys.stdout.write('\n* {} town with geometry error\n'.format(nb_error)) sys.stdout.flush()