diff options
Diffstat (limited to 'ishtar_common/management')
-rw-r--r-- | ishtar_common/management/commands/migrate_to_geo_v4.py | 461 |
1 files changed, 265 insertions, 196 deletions
diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py index 2a316c2a4..d904b279a 100644 --- a/ishtar_common/management/commands/migrate_to_geo_v4.py +++ b/ishtar_common/management/commands/migrate_to_geo_v4.py @@ -3,6 +3,8 @@ import csv import datetime +from django.db import connection +from multiprocessing import Pool, Process import os import sys @@ -24,63 +26,269 @@ if not os.path.exists(log_path): os.mkdir(log_path, mode=0o770) -def write_output(model_name, idx, nb, ref_time=None): - lbl = f"\r[{get_percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}" +town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town") +data_type, __ = models_common.GeoDataType.objects.get_or_create( + txt_idx="town-limit", defaults={"label": "Limites commune"} +) +provider, __ = models_common.GeoProviderType.objects.get_or_create( + txt_idx="france-ign", defaults={"label": "IGN"} +) + +changed = [] + + +def _process_town(town_id): + connection.close() + town = models_common.Town.objects.get(pk=town_id) + attrs = { + "name": town._generate_cached_label(), + "source_content_type": town_content_type, + "source_id": town.pk, + "data_type": data_type, + "provider": provider, + } + if town.limit: + attrs["multi_polygon"] = town.limit + else: + attrs["point_2d"] = town.center + data, created = models_common.GeoVectorData.objects.get_or_create(**attrs) + town.main_geodata = data + town._post_save_geo_ok = False + town.save() + if created: + changed.append(["geovectordata", data.name, data.pk, "Création commune"]) + + +model_slug, model_name, model_full_name, model = None, None, None, None +model_content_type, data_type_area, data_type_center = None, None, None + +cls_labels = { + "ContextRecord": ["Context Record", "Unité d'Enregistrement"], + "BaseFind": ["Base find", "Mobilier d'origine"], + "Operation": ["Operation", "Opération"], + "ArchaeologicalSite": ["Entité (EA)", "Entité archéologique", + "Archaeological site"], +} + + +def _process_site_ope(obj): + connection.close() + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() # auto manage geo town association + q_towns = obj.towns.filter(main_geodata__multi_polygon__isnull=False) + if q_towns.count() > 1: + changed.append( + [model_slug, str(obj), obj.pk, "Association géo de zone communale"] + ) + elif q_towns.count() == 1: + changed.append( + [model_slug, str(obj), obj.pk, "Association géo de commune"] + ) + obj_verbose_names = cls_labels[obj.__class__.__name__] + if obj.multi_polygon_source == "P" and obj.multi_polygon \ + and obj.multi_polygon_source_item in obj_verbose_names: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "multi_polygon": obj.multi_polygon, + "data_type": data_type_area, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj._post_save_geo_ok = False + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Multi-polygone {model_name}", + ] + ) + if obj.point_source == "P" and obj.point_2d \ + and obj.point_source_item in obj_verbose_names: + if obj.x and obj.y: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + "x": obj.x, + "y": obj.y, + "z": obj.z, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Coordonnées {model_name}", + ] + ) + elif obj.point_2d: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + } + if obj.point: + attrs["point_3d"] = obj.point + else: + attrs["point_2d"] = obj.point_2d + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj._post_save_geo_ok = False + obj.save() + changed.append( + ["geovectordata", data.name, data.pk, f"Point {model_name}"] + ) + +data_type_outline = None + + +def _process_main(obj): + connection.close() + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() # auto manage geo town association + + if obj.main_geodata: + changed.append( + [model_slug, str(obj), obj.pk, "Association géo de zone communale"] + ) + obj_verbose_names = cls_labels[obj.__class__.__name__] + if obj.multi_polygon_source == "P" and obj.multi_polygon \ + and obj.multi_polygon_source_item in obj_verbose_names: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "multi_polygon": obj.multi_polygon, + "data_type": data_type_outline, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj._post_save_geo_ok = False + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Multi-polygone {model_name}", + ] + ) + if obj.point_source == "P" and obj.point_2d \ + and obj.point_source_item in obj_verbose_names: + if obj.x and obj.y: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + "x": obj.x, + "y": obj.y, + "z": obj.z, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj._post_save_geo_ok = False + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Coordonnées {model_name}", + ] + ) + elif obj.point_2d: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + } + if obj.point: + attrs["point_3d"] = obj.point + else: + attrs["point_2d"] = obj.point_2d + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj._post_save_geo_ok = False + obj.save() + changed.append( + ["geovectordata", data.name, data.pk, f"Point {model_name}"] + ) + + +def _process_simple(obj): + connection.close() + obj._post_save_geo_ok = False + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() + + +idx = 0 +total = 0 +ref_time = None + + +def write_output(arg): + global idx, total, model_name, ref_time, quiet + if quiet: + return + idx = idx + 1 + lbl = f"\r[{get_percent(idx, total)}] Migrate {model_name}s {idx + 1}/{total}" if ref_time: - lbl += f" ({get_eta(idx, nb, ref_time, datetime.datetime.now())} left)" + lbl += f" ({get_eta(idx, total, ref_time, datetime.datetime.now())} left)" sys.stdout.write(lbl) sys.stdout.flush() -def migrate(quiet=False, log=True): - changed = [] +def launch_job(lst, name, process_number, process_func): + global idx, total, model_name, ref_time + idx, total, model_name, ref_time = 0, len(lst), name, datetime.datetime.now() + pool = Pool(processes=process_number) + for item in lst: + pool.apply_async(process_func, (item,), callback=write_output) + pool.close() + pool.join() + + +quiet = False + + +def migrate(log=True, process_number=1): + global idx # create towns q = models_common.Town.objects.exclude( center__isnull=True, limit__isnull=True - ).exclude(main_geodata__isnull=False) - nb = q.count() - town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town") - data_type, __ = models_common.GeoDataType.objects.get_or_create( - txt_idx="town-limit", defaults={"label": "Limites commune"} - ) - provider, __ = models_common.GeoProviderType.objects.get_or_create( - txt_idx="france-ign", defaults={"label": "IGN"} - ) - ref_time = datetime.datetime.now() - for idx, town in enumerate(q.all()): - if not quiet: - write_output("town", idx, nb, ref_time) - attrs = { - "name": town._generate_cached_label(), - "source_content_type": town_content_type, - "source_id": town.pk, - "data_type": data_type, - "provider": provider, - } - if town.limit: - attrs["multi_polygon"] = town.limit - else: - attrs["point_2d"] = town.center - data, created = models_common.GeoVectorData.objects.get_or_create(**attrs) - if created: - changed.append(["geovectordata", data.name, data.pk, "Création commune"]) - town.main_geodata = data - town.save() - if not quiet and nb: - sys.stdout.write(f"\r[{get_log_time()}] Towns migrated \n") - sys.stdout.flush() + ).exclude(main_geodata__isnull=False).distinct() + town_ids = list(q.values_list("id", flat=True)) + idx = 0 + launch_job(town_ids, "town", process_number, _process_town) model_list = [ ("operation", "opération", "de l'opération", Operation), ("archaeologicalsite", "site", "du site", ArchaeologicalSite), ] + + global model_slug, model_name, model_full_name, model + global model_content_type, data_type_area, data_type_center, data_type_outline for model_slug, model_name, model_full_name, model in model_list: + connection.close() # manage operation vector sources model_content_type = ContentType.objects.get( app_label="archaeological_operations", model=model_slug ) - q = model.objects.exclude(main_geodata__isnull=False) - nb = q.count() data_type_area, __ = models_common.GeoDataType.objects.get_or_create( txt_idx=f"{model_slug}-area", defaults={"label": f"Emprise {model_full_name}"}, @@ -89,82 +297,10 @@ def migrate(quiet=False, log=True): txt_idx=f"{model_slug}-center", defaults={"label": f"Centre {model_full_name}"}, ) - ref_time = datetime.datetime.now() - for idx, obj in enumerate(q.all()): - if not quiet: - write_output(model_name, idx, nb, ref_time) - - obj._no_move = True - obj.skip_history_when_saving = True - obj.save() # auto manage geo town association - q_towns = obj.towns.filter(main_geodata__multi_polygon__isnull=False) - if q_towns.count() > 1: - changed.append( - [model_slug, str(obj), obj.pk, "Association géo de zone communale"] - ) - elif q_towns.count() == 1: - changed.append( - [model_slug, str(obj), obj.pk, "Association géo de commune"] - ) - if obj.multi_polygon_source == "P" and obj.multi_polygon: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "multi_polygon": obj.multi_polygon, - "data_type": data_type_area, - } - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - [ - "geovectordata", - data.name, - data.pk, - f"Multi-polygone {model_name}", - ] - ) - if obj.point_source == "P" and obj.point_2d: - if obj.x and obj.y: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "data_type": data_type_center, - "x": obj.x, - "y": obj.y, - "z": obj.z, - } - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - [ - "geovectordata", - data.name, - data.pk, - f"Coordonnées {model_name}", - ] - ) - elif obj.point_2d: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "data_type": data_type_center, - } - if obj.point: - attrs["point_3d"] = obj.point - else: - attrs["point_2d"] = obj.point_2d - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - ["geovectordata", data.name, data.pk, f"Point {model_name}"] - ) - if not quiet and nb: + q = model.objects.exclude(main_geodata__isnull=False) + launch_job(list(q.all()), model_name, process_number, _process_site_ope) + + if not quiet: sys.stdout.write( f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 @@ -189,9 +325,8 @@ def migrate(quiet=False, log=True): ), ] for app, model_slug, model_name, model_full_name, model in model_list: + connection.close() model_content_type = ContentType.objects.get(app_label=app, model=model_slug) - q = model.objects.exclude(main_geodata__isnull=False) - nb = q.count() data_type_outline, __ = models_common.GeoDataType.objects.get_or_create( txt_idx=f"{model_slug}-outline", defaults={"label": f"Contour d'{model_name}"}, @@ -200,77 +335,10 @@ def migrate(quiet=False, log=True): txt_idx=f"{model_slug}-center", defaults={"label": f"Centre {model_full_name}"}, ) - ref_time = datetime.datetime.now() - for idx, obj in enumerate(q.all()): - if not quiet: - write_output(model_name, idx, nb, ref_time) - obj._no_move = True - obj.skip_history_when_saving = True - obj.save() # auto manage geo town association - - if obj.main_geodata: - changed.append( - [model_slug, str(obj), obj.pk, "Association géo de zone communale"] - ) - if obj.multi_polygon_source == "P" and obj.multi_polygon: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "multi_polygon": obj.multi_polygon, - "data_type": data_type_outline, - } - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - [ - "geovectordata", - data.name, - data.pk, - f"Multi-polygone {model_name}", - ] - ) - if obj.point_source == "P" and obj.point_2d: - if obj.x and obj.y: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "data_type": data_type_center, - "x": obj.x, - "y": obj.y, - "z": obj.z, - } - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - [ - "geovectordata", - data.name, - data.pk, - f"Coordonnées {model_name}", - ] - ) - elif obj.point_2d: - attrs = { - "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", - "source_content_type": model_content_type, - "source_id": obj.pk, - "data_type": data_type_center, - } - if obj.point: - attrs["point_3d"] = obj.point - else: - attrs["point_2d"] = obj.point_2d - data = models_common.GeoVectorData.objects.create(**attrs) - obj.main_geodata = data - obj.save() - changed.append( - ["geovectordata", data.name, data.pk, f"Point {model_name}"] - ) - if not quiet and nb: + q = model.objects.exclude(main_geodata__isnull=False) + launch_job(q.all(), model_name, process_number, _process_main) + + if not quiet: sys.stdout.write( f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 @@ -279,14 +347,11 @@ def migrate(quiet=False, log=True): sys.stdout.flush() model_list = [Warehouse, Container] for model in model_list: - ref_time = datetime.datetime.now() + connection.close() q = model.objects.exclude(main_geodata__isnull=False) - nb = q.count() - for idx, obj in enumerate(q.all()): - if not quiet: - write_output(model.__name__, idx, nb, ref_time) - obj.save() - if not quiet and nb: + launch_job(q.all(), model.__name__, process_number, _process_simple) + + if not quiet: sys.stdout.write( f"\r[{get_log_time()}] {model.__name__.capitalize()} migrated" + " " * 20 + "\n" ) @@ -309,6 +374,9 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( + "--process", dest="process", help="Number of process" + ) + parser.add_argument( "--quiet", dest="quiet", action="store_true", help="Quiet output" ) parser.add_argument( @@ -317,10 +385,11 @@ class Command(BaseCommand): def handle(self, *args, **options): log = options["log"] + global quiet quiet = options["quiet"] if not quiet: sys.stdout.write(f"[{get_log_time()}] Processing migration\n") - errors = migrate(quiet=quiet, log=log) + errors = migrate(log=log, process_number=int(options["process"] or 1)) if not errors: if not quiet: sys.stdout.write(f"[{get_log_time()}] Migration finished\n") |