Geodata redesign: context record, base find migration

2 files changed, 148 insertions, 22 deletions

diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py
index ed1a877f3..2c27eebe9 100644
--- a/ishtar_common/management/commands/migrate_to_geo_v4.py
+++ b/ishtar_common/management/commands/migrate_to_geo_v4.py
@@ -10,10 +10,11 @@ from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import BaseCommand
 
-from ishtar_common.utils import ugettext_lazy as _
+from ishtar_common.utils import ugettext_lazy as _, get_log_time, get_percent, get_eta
 
 from ishtar_common import models_common, models
 from archaeological_operations.models import Operation, ArchaeologicalSite
+from archaeological_context_records.models import ContextRecord
 
 
 log_path = os.sep.join([settings.ROOT_PATH, "logs"])
@@ -21,6 +22,14 @@ if not os.path.exists(log_path):
     os.mkdir(log_path, mode=0o770)
 
 
+def write_output(model_name, idx, nb, ref_time=None):
+    lbl = f"\r[{get_percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}"
+    if ref_time:
+        lbl += f" ({get_eta(idx, nb, ref_time, datetime.datetime.now())} left)"
+    sys.stdout.write(lbl)
+    sys.stdout.flush()
+
+
 def migrate(quiet=False, log=True):
     changed = []
     # create towns
@@ -35,10 +44,10 @@ def migrate(quiet=False, log=True):
     provider, __ = models_common.GeoProviderType.objects.get_or_create(
         txt_idx="france-ign", defaults={"label": "IGN"}
     )
+    ref_time = datetime.datetime.now()
     for idx, town in enumerate(q.all()):
         if not quiet:
-            sys.stdout.write(f"\r[{percent(idx, nb)}] Migrate towns {idx + 1}/{nb}")
-            sys.stdout.flush()
+            write_output("town", idx, nb, ref_time)
         attrs = {
             "name": town._generate_cached_label(),
             "source_content_type": town_content_type,
@@ -56,7 +65,7 @@ def migrate(quiet=False, log=True):
         town.main_geodata = data
         town.save()
     if not quiet and nb:
-        sys.stdout.write(f"\r[{get_time()}] Towns migrated\n")
+        sys.stdout.write(f"\r[{get_log_time()}] Towns migrated                 \n")
         sys.stdout.flush()
 
     model_list = [
@@ -75,14 +84,13 @@ def migrate(quiet=False, log=True):
             defaults={"label": f"Emprise {model_full_name}"},
         )
         data_type_center, __ = models_common.GeoDataType.objects.get_or_create(
-            txt_idx="operation-center", defaults={"label": f"Centre {model_full_name}"}
+            txt_idx=f"{model_slug}-center",
+            defaults={"label": f"Centre {model_full_name}"},
         )
+        ref_time = datetime.datetime.now()
         for idx, obj in enumerate(q.all()):
             if not quiet:
-                sys.stdout.write(
-                    f"\r[{percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}"
-                )
-                sys.stdout.flush()
+                write_output(model_name, idx, nb, ref_time)
 
             obj._no_move = True
             obj.skip_history_when_saving = True
@@ -155,11 +163,119 @@ def migrate(quiet=False, log=True):
                         ["geovectordata", data.name, data.pk, f"Point {model_name}"]
                     )
         if not quiet and nb:
-            sys.stdout.write(f"\r[{get_time()}] {model_name.capitalize()} migrated\n")
+            sys.stdout.write(
+                f"\r[{get_log_time()}] {model_name.capitalize()} migrated"
+                + " " * 20
+                + "\n"
+            )
             sys.stdout.flush()
 
+    model_list = [
+        (
+            "archaeological_context_records",
+            "contextrecord",
+            "unité d'enregistrement",
+            "de l'unité d'enregistrement",
+            ContextRecord,
+        ),
+        (
+            "archaeological_finds",
+            "basefind",
+            "mobilier d'origine",
+            "du mobilier d'origine",
+            ContextRecord,
+        ),
+    ]
+    for app, model_slug, model_name, model_full_name, model in model_list:
+        model_content_type = ContentType.objects.get(app_label=app, model=model_slug)
+        q = model.objects.exclude(main_geodata__isnull=False)
+        nb = q.count()
+        data_type_outline, __ = models_common.GeoDataType.objects.get_or_create(
+            txt_idx=f"{model_slug}-outline",
+            defaults={"label": f"Contour d'{model_name}"},
+        )
+        data_type_center, __ = models_common.GeoDataType.objects.get_or_create(
+            txt_idx=f"{model_slug}-center",
+            defaults={"label": f"Centre {model_full_name}"},
+        )
+        ref_time = datetime.datetime.now()
+        for idx, obj in enumerate(q.all()[:100]):
+            if not quiet:
+                write_output(model_name, idx, nb, ref_time)
+            obj._no_move = True
+            obj.skip_history_when_saving = True
+            obj.save()  # auto manage geo town association
+
+            if obj.main_geodata:
+                changed.append(
+                    [model_slug, str(obj), obj.pk, "Association géo de zone communale"]
+                )
+            if obj.multi_polygon_source == "P" and obj.multi_polygon:
+                attrs = {
+                    "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                    "source_content_type": model_content_type,
+                    "source_id": obj.pk,
+                    "multi_polygon": obj.multi_polygon,
+                    "data_type": data_type_outline,
+                }
+                data = models_common.GeoVectorData.objects.create(**attrs)
+                obj.main_geodata = data
+                obj.save()
+                changed.append(
+                    [
+                        "geovectordata",
+                        data.name,
+                        data.pk,
+                        f"Multi-polygone {model_name}",
+                    ]
+                )
+            if obj.point_source == "P" and obj.point_2d:
+                if obj.x and obj.y:
+                    attrs = {
+                        "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                        "source_content_type": model_content_type,
+                        "source_id": obj.pk,
+                        "data_type": data_type_center,
+                        "x": obj.x,
+                        "y": obj.y,
+                        "z": obj.z,
+                    }
+                    data = models_common.GeoVectorData.objects.create(**attrs)
+                    obj.main_geodata = data
+                    obj.save()
+                    changed.append(
+                        [
+                            "geovectordata",
+                            data.name,
+                            data.pk,
+                            f"Coordonnées {model_name}",
+                        ]
+                    )
+                elif obj.point_2d:
+                    attrs = {
+                        "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                        "source_content_type": model_content_type,
+                        "source_id": obj.pk,
+                        "data_type": data_type_center,
+                    }
+                    if obj.point:
+                        attrs["point_3d"] = obj.point
+                    else:
+                        attrs["point_2d"] = obj.point_2d
+                    data = models_common.GeoVectorData.objects.create(**attrs)
+                    obj.main_geodata = data
+                    obj.save()
+                    changed.append(
+                        ["geovectordata", data.name, data.pk, f"Point {model_name}"]
+                    )
+    if not quiet and nb:
+        sys.stdout.write(
+            f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 + "\n"
+        )
+        sys.stdout.flush()
+
     if log and changed:
-        filename = f"geo_migration-created-{get_time().replace(':', '')}.csv"
+        filename = f"geo_migration-created-{get_log_time().replace(':', '')}.csv"
         path = os.sep.join([log_path, filename])
         with open(path, "w+") as fle:
             writer = csv.writer(fle)
@@ -167,15 +283,7 @@ def migrate(quiet=False, log=True):
             for change in changed:
                 writer.writerow(change)
         if not quiet:
-            sys.stdout.write(f"[{get_time()}] Log: {path} written\n")
-
-
-def percent(current, total):
-    return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
-
-
-def get_time():
-    return datetime.datetime.now().isoformat().split(".")[0]
+            sys.stdout.write(f"[{get_log_time()}] Log: {path} written\n")
 
 
 class Command(BaseCommand):
@@ -193,11 +301,11 @@ class Command(BaseCommand):
         log = options["log"]
         quiet = options["quiet"]
         if not quiet:
-            sys.stdout.write(f"[{get_time()}] Processing migration\n")
+            sys.stdout.write(f"[{get_log_time()}] Processing migration\n")
         errors = migrate(quiet=quiet, log=log)
         if not errors:
             if not quiet:
-                sys.stdout.write(f"[{get_time()}] Migration finished\n")
+                sys.stdout.write(f"[{get_log_time()}] Migration finished\n")
             sys.exit()
         if not quiet:
             sys.stdout.write("\n".join(errors))
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index fc302166b..a7e07ebde 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -2335,3 +2335,21 @@ def create_osm_town(rel_id, name, numero_insee=None):
     town.center = town.limit.centroid
     town.save()
     return town
+
+
+def get_percent(current, total):
+    return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
+
+
+def get_log_time():
+    return datetime.datetime.now().isoformat().split(".")[0]
+
+
+def get_eta(current, total, base_time, current_time):
+    if current < 5:
+        return "-"
+    elapsed_time = current_time - base_time
+    eta = elapsed_time.seconds / current * (total - current)
+    if eta < 1:
+        return "-"
+    return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}"