diff options
Diffstat (limited to 'ishtar_common')
| -rw-r--r-- | ishtar_common/management/commands/migrate_to_geo_v4.py | 152 | ||||
| -rw-r--r-- | ishtar_common/utils.py | 18 |
2 files changed, 148 insertions, 22 deletions
diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py index ed1a877f3..2c27eebe9 100644 --- a/ishtar_common/management/commands/migrate_to_geo_v4.py +++ b/ishtar_common/management/commands/migrate_to_geo_v4.py @@ -10,10 +10,11 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand -from ishtar_common.utils import ugettext_lazy as _ +from ishtar_common.utils import ugettext_lazy as _, get_log_time, get_percent, get_eta from ishtar_common import models_common, models from archaeological_operations.models import Operation, ArchaeologicalSite +from archaeological_context_records.models import ContextRecord log_path = os.sep.join([settings.ROOT_PATH, "logs"]) @@ -21,6 +22,14 @@ if not os.path.exists(log_path): os.mkdir(log_path, mode=0o770) +def write_output(model_name, idx, nb, ref_time=None): + lbl = f"\r[{get_percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}" + if ref_time: + lbl += f" ({get_eta(idx, nb, ref_time, datetime.datetime.now())} left)" + sys.stdout.write(lbl) + sys.stdout.flush() + + def migrate(quiet=False, log=True): changed = [] # create towns @@ -35,10 +44,10 @@ def migrate(quiet=False, log=True): provider, __ = models_common.GeoProviderType.objects.get_or_create( txt_idx="france-ign", defaults={"label": "IGN"} ) + ref_time = datetime.datetime.now() for idx, town in enumerate(q.all()): if not quiet: - sys.stdout.write(f"\r[{percent(idx, nb)}] Migrate towns {idx + 1}/{nb}") - sys.stdout.flush() + write_output("town", idx, nb, ref_time) attrs = { "name": town._generate_cached_label(), "source_content_type": town_content_type, @@ -56,7 +65,7 @@ def migrate(quiet=False, log=True): town.main_geodata = data town.save() if not quiet and nb: - sys.stdout.write(f"\r[{get_time()}] Towns migrated\n") + sys.stdout.write(f"\r[{get_log_time()}] Towns migrated \n") sys.stdout.flush() model_list = [ @@ -75,14 +84,13 @@ def migrate(quiet=False, log=True): defaults={"label": f"Emprise {model_full_name}"}, ) data_type_center, __ = models_common.GeoDataType.objects.get_or_create( - txt_idx="operation-center", defaults={"label": f"Centre {model_full_name}"} + txt_idx=f"{model_slug}-center", + defaults={"label": f"Centre {model_full_name}"}, ) + ref_time = datetime.datetime.now() for idx, obj in enumerate(q.all()): if not quiet: - sys.stdout.write( - f"\r[{percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}" - ) - sys.stdout.flush() + write_output(model_name, idx, nb, ref_time) obj._no_move = True obj.skip_history_when_saving = True @@ -155,11 +163,119 @@ def migrate(quiet=False, log=True): ["geovectordata", data.name, data.pk, f"Point {model_name}"] ) if not quiet and nb: - sys.stdout.write(f"\r[{get_time()}] {model_name.capitalize()} migrated\n") + sys.stdout.write( + f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + + " " * 20 + + "\n" + ) sys.stdout.flush() + model_list = [ + ( + "archaeological_context_records", + "contextrecord", + "unité d'enregistrement", + "de l'unité d'enregistrement", + ContextRecord, + ), + ( + "archaeological_finds", + "basefind", + "mobilier d'origine", + "du mobilier d'origine", + ContextRecord, + ), + ] + for app, model_slug, model_name, model_full_name, model in model_list: + model_content_type = ContentType.objects.get(app_label=app, model=model_slug) + q = model.objects.exclude(main_geodata__isnull=False) + nb = q.count() + data_type_outline, __ = models_common.GeoDataType.objects.get_or_create( + txt_idx=f"{model_slug}-outline", + defaults={"label": f"Contour d'{model_name}"}, + ) + data_type_center, __ = models_common.GeoDataType.objects.get_or_create( + txt_idx=f"{model_slug}-center", + defaults={"label": f"Centre {model_full_name}"}, + ) + ref_time = datetime.datetime.now() + for idx, obj in enumerate(q.all()[:100]): + if not quiet: + write_output(model_name, idx, nb, ref_time) + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() # auto manage geo town association + + if obj.main_geodata: + changed.append( + [model_slug, str(obj), obj.pk, "Association géo de zone communale"] + ) + if obj.multi_polygon_source == "P" and obj.multi_polygon: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "multi_polygon": obj.multi_polygon, + "data_type": data_type_outline, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Multi-polygone {model_name}", + ] + ) + if obj.point_source == "P" and obj.point_2d: + if obj.x and obj.y: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + "x": obj.x, + "y": obj.y, + "z": obj.z, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Coordonnées {model_name}", + ] + ) + elif obj.point_2d: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + } + if obj.point: + attrs["point_3d"] = obj.point + else: + attrs["point_2d"] = obj.point_2d + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + ["geovectordata", data.name, data.pk, f"Point {model_name}"] + ) + if not quiet and nb: + sys.stdout.write( + f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 + "\n" + ) + sys.stdout.flush() + if log and changed: - filename = f"geo_migration-created-{get_time().replace(':', '')}.csv" + filename = f"geo_migration-created-{get_log_time().replace(':', '')}.csv" path = os.sep.join([log_path, filename]) with open(path, "w+") as fle: writer = csv.writer(fle) @@ -167,15 +283,7 @@ def migrate(quiet=False, log=True): for change in changed: writer.writerow(change) if not quiet: - sys.stdout.write(f"[{get_time()}] Log: {path} written\n") - - -def percent(current, total): - return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" - - -def get_time(): - return datetime.datetime.now().isoformat().split(".")[0] + sys.stdout.write(f"[{get_log_time()}] Log: {path} written\n") class Command(BaseCommand): @@ -193,11 +301,11 @@ class Command(BaseCommand): log = options["log"] quiet = options["quiet"] if not quiet: - sys.stdout.write(f"[{get_time()}] Processing migration\n") + sys.stdout.write(f"[{get_log_time()}] Processing migration\n") errors = migrate(quiet=quiet, log=log) if not errors: if not quiet: - sys.stdout.write(f"[{get_time()}] Migration finished\n") + sys.stdout.write(f"[{get_log_time()}] Migration finished\n") sys.exit() if not quiet: sys.stdout.write("\n".join(errors)) diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index fc302166b..a7e07ebde 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -2335,3 +2335,21 @@ def create_osm_town(rel_id, name, numero_insee=None): town.center = town.limit.centroid town.save() return town + + +def get_percent(current, total): + return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" + + +def get_log_time(): + return datetime.datetime.now().isoformat().split(".")[0] + + +def get_eta(current, total, base_time, current_time): + if current < 5: + return "-" + elapsed_time = current_time - base_time + eta = elapsed_time.seconds / current * (total - current) + if eta < 1: + return "-" + return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}" |