diff options
Diffstat (limited to 'ishtar_common/management/commands')
| -rw-r--r-- | ishtar_common/management/commands/dump_towns.py | 50 | ||||
| -rw-r--r-- | ishtar_common/management/commands/load_towns.py | 158 |
2 files changed, 18 insertions, 190 deletions
diff --git a/ishtar_common/management/commands/dump_towns.py b/ishtar_common/management/commands/dump_towns.py index 31465aa13..aa5270aef 100644 --- a/ishtar_common/management/commands/dump_towns.py +++ b/ishtar_common/management/commands/dump_towns.py @@ -44,7 +44,8 @@ class Command(BaseCommand): def handle(self, *args, **options): quiet = options['quiet'] query = options["query"] - q = Town.objects.filter(main_geodata__isnull=False, main_geodata__multi_polygon__isnull=False) + q = Town.objects.filter(main_geodata__isnull=False, + main_geodata__multi_polygon__isnull=False) if query: try: query = json.loads(query) @@ -61,52 +62,7 @@ class Command(BaseCommand): get_progress("processing town", idx, nb_lines, started) ) sys.stdout.flush() - geo = town.main_geodata - town_dct = { - "model": "ishtar_common.town", - "fields": { - "name": town.name, - "surface": town.surface, - "numero_insee": town.numero_insee, - "notice": town.notice, - "year": town.year, - "cached_label": town.cached_label, - "main_geodata": [ - "ishtar_common", - "town", - town.numero_insee - ], - "geodata": [ - ["ishtar_common", "town", town.numero_insee] - ], - "children": [ - t.numero_insee - for t in town.children.filter(numero_insee__isnull=False).all() - ] - } - } - geo_dct = { - "model": "ishtar_common.geovectordata", - "fields": { - "name": geo.name, - "source_content_type": [ - "ishtar_common", - "town" - ], - "source": town.numero_insee, - "data_type": [ - "town-limit" - ], - "provider": geo.provider.txt_idx, - "comment": geo.comment, - "cached_x": geo.cached_x, - "cached_y": geo.cached_y, - "spatial_reference_system": None, - "multi_polygon": geo.multi_polygon.wkt - } - } - result.append(geo_dct) - result.append(town_dct) + result += town.geodata_export today = datetime.date.today() result_file = f"ishtar-towns-{today.strftime('%Y-%m-%d')}.json" with open(result_file, "w") as r: diff --git a/ishtar_common/management/commands/load_towns.py b/ishtar_common/management/commands/load_towns.py index 1ab73d677..86066c4d8 100644 --- a/ishtar_common/management/commands/load_towns.py +++ b/ishtar_common/management/commands/load_towns.py @@ -17,24 +17,16 @@ # See the file COPYING for details. -import datetime import json import os import sys from django.conf import settings from django.core.management.base import BaseCommand -from django.contrib.contenttypes.models import ContentType from django.db import transaction -from ishtar_common.utils import BColors, get_log_time, get_progress -from ishtar_common.models import Town, GeoVectorData, GeoDataType, GeoProviderType - - -town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town") -town_data_type, __ = GeoDataType.objects.get_or_create( - txt_idx="town-limit", defaults={"label": "Limites commune"} -) +from ishtar_common.utils import BColors, get_log_time +from ishtar_common.models import Town class Command(BaseCommand): @@ -61,143 +53,23 @@ class Command(BaseCommand): with open(towns_file, "r") as t: src = json.loads(t.read()) - nb_lines = len(src) - started = datetime.datetime.now() - self.geo_updated, self.geo_created = 0, 0 - self.town_created = 0 log_filename = f"load_towns-{get_log_time().replace(':', '')}.csv" log_path = os.sep.join([log_path, log_filename]) - towns, geo_datas, children = {}, {}, {} - for idx, values in enumerate(src): - sys.stdout.write(get_progress("processing", idx, nb_lines, started)) - sys.stdout.flush() - fields = values["fields"] - if values["model"] == "ishtar_common.town": - if self.limit and not values["numero_insee"].startswith(self.limit): - continue - c_children = fields.pop("children") - if c_children: - children[fields["numero_insee"]] = c_children - towns[fields["numero_insee"]], created = self.update_town( - fields, geo_datas - ) - if values["model"] == "ishtar_common.geovectordata": - self.update_geodata(fields, geo_datas) - # manage geo sources - for insee in geo_datas: - if insee not in towns: - sys.stdout.write( - f"\n{BColors.FAIL}geodata source : INSEE manquant {insee}{BColors.ENDC}\n" - ) - else: - g = geo_datas[insee] - if g.source_id != towns[insee].pk: - g.source_id = towns[insee].pk - g.save() - nb_lines = len(children) - started = datetime.datetime.now() - self.nb_rels = 0 - print() - # management childrens - for idx, insee in enumerate(children): - sys.stdout.write(get_progress("update children", idx, nb_lines, started)) - sys.stdout.flush() - self.get_children(insee, towns, children) + town_created, geo_created, geo_updated, nb_rels = Town.geodata_import( + src, limit=self.limit, log_path=log_path, verbose=not quiet + ) + if quiet: + return sys.stdout.write(BColors.OKGREEN) - if self.town_created: - sys.stdout.write(f'\n* {self.town_created} town created') - if self.geo_created: - sys.stdout.write(f'\n* {self.geo_created} geo created') - if self.geo_updated: - sys.stdout.write(f'\n* {self.geo_updated} geo updated') - if self.nb_rels: - sys.stdout.write(f'\n* {self.nb_rels} relations updated') + if town_created: + sys.stdout.write(f'\n* {town_created} town created') + if geo_created: + sys.stdout.write(f'\n* {geo_created} geo created') + if geo_updated: + sys.stdout.write(f'\n* {geo_updated} geo updated') + if nb_rels: + sys.stdout.write(f'\n* {nb_rels} relations updated') sys.stdout.write(BColors.ENDC + "\n") sys.stdout.flush() - - def update_town(self, fields, geo_datas): - values = fields.copy() - geos = [] - for geo in values.pop("geodata"): - geo_id = geo[2] - if geo_id not in geo_datas: - sys.stdout.write(f"\n{BColors.FAIL}geodata : Geo INSEE manquant {geo_id}{BColors.ENDC}\n") - else: - geos.append(geo_datas[geo_id]) - main_geo = values["main_geodata"][2] - if main_geo not in geo_datas: - sys.stdout.write(f"\n{BColors.FAIL}main_geodata : Geo INSEE manquant {main_geo}{BColors.ENDC}\n") - values.pop(main_geo) - else: - values["main_geodata"] = geo_datas[main_geo] - - q = Town.objects.filter(numero_insee=values["numero_insee"]) - created = False - if q.count(): - q.update(**values) - town = q.all()[0] - else: - created = True - self.town_created += 1 - town = Town.objects.create(**values) - for geo in geos: - town.geodata.add(geo) - return town, created - - def update_geodata(self, fields, geo_datas): - numero_insee = fields.pop('source') - if self.limit and not numero_insee.startswith(self.limit): - return - q = Town.objects.filter(numero_insee=numero_insee) - values = { - "provider": GeoProviderType.objects.get(txt_idx=fields["provider"]), - "comment": fields["comment"], - 'multi_polygon': fields["multi_polygon"] - } - if q.count(): - source_id = q.all()[0].pk - q2 = GeoVectorData.objects.filter( - source_id=source_id, - source_content_type=town_content_type, - data_type=town_data_type - ) - if q2.count(): - geo = q2.all()[0] - changed = False - for k in values: - if k == "multi_polygon": - if geo.multi_polygon.wkt != values[k]: - setattr(geo, k, values[k]) - changed = True - elif getattr(geo, k) != values[k]: - setattr(geo, k, values[k]) - changed = True - if changed: - self.geo_updated += 1 - geo.save() - geo_datas[numero_insee] = geo - return - values.update({ - "source_content_type": town_content_type, - "data_type": town_data_type - }) - self.geo_created += 1 - geo = GeoVectorData.objects.create(**values) - geo_datas[numero_insee] = geo - - def get_children(self, insee, towns, children): - if insee not in towns: - sys.stdout.write(f"\n{BColors.FAIL}children : INSEE manquant {insee}{BColors.ENDC}\n") - return - town = towns[insee] - current_children = list(town.children.values_list("id", flat=True)) - for child in children[insee]: - if child not in towns: - sys.stdout.write(f"\n{BColors.FAIL}children-child : INSEE manquant {insee}{BColors.ENDC}\n") - continue - if towns[child].id in current_children: - continue - self.nb_rels += 1 - town.children.add(towns[child]) |