Geodata redesign: context record, base find migration

4 files changed, 210 insertions, 27 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index 29e8793dc..e9bd70aa8 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -397,12 +397,61 @@ class CRBulkView(object):
     """
 
 
+class GeographicSubTownItem(GeoItem):
+    class Meta:
+        abstract = True
+
+    def _get_geo_town(self):
+        raise NotImplementedError()
+
+    def post_save_geo(self, save=True):
+        # manage geodata towns
+        if getattr(self, "_post_save_geo_ok", False):
+            # prevent infinite loop - should not happen, but...
+            return
+        self._post_save_geo_ok = True
+
+        q_geodata_current_town = self.geodata.filter(
+            source_content_type__model="town",
+            source_content_type__app_label="ishtar_common",
+        )
+        town = self._get_geo_town()
+        has_geo_town = (
+            town and town.main_geodata and town.main_geodata.multi_polygon
+        )
+        if has_geo_town:
+            bad_towns = q_geodata_current_town
+        else:
+            bad_towns = q_geodata_current_town.exclude(source_id=town.id)
+
+        modified = False
+        for bad_town in bad_towns.all():
+            self.geodata.remove(bad_town)
+            if self.main_geodata == bad_town:
+                self.main_geodata = None
+            modified = True
+
+        if not has_geo_town:
+            if modified and save:
+                self.skip_history_when_saving = True
+                self._no_move = True
+                self.save()
+            return
+
+        if not q_geodata_current_town.filter(source_id=town.id).count():
+            self.geodata.add(town.main_geodata)
+            if save:
+                self.skip_history_when_saving = True
+                self._no_move = True
+                self.save()
+
+
 class ContextRecord(
     BulkUpdatedItem,
     DocumentItem,
     BaseHistorizedItem,
     CompleteIdentifierItem,
-    GeoItem,
+    GeographicSubTownItem,
     OwnPerms,
     ValueGetter,
     MainItem,
@@ -510,7 +559,7 @@ class ContextRecord(
         ),
         "operation__common_name": SearchAltName(
             pgettext_lazy("key for text search", "operation-name"),
-            "operation__common_name__iexact"
+            "operation__common_name__iexact",
         ),
         "operation__code_patriarche": SearchAltName(
             pgettext_lazy("key for text search", "patriarche"),
@@ -810,6 +859,11 @@ class ContextRecord(
         if self.surface:
             return self.surface / 10000.0
 
+    def _get_geo_town(self):
+        if self.parcel:
+            return self.parcel.town
+        return self.town
+
     def public_representation(self):
         dct = super(ContextRecord, self).public_representation()
         dct.update(
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index bc1d64ee6..ce9a5e4af 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -55,7 +55,6 @@ from ishtar_common.models import (
     ValueGetter,
     get_current_profile,
     IshtarSiteProfile,
-    GeoItem,
     BulkUpdatedItem,
     QuickAction,
     MainItem,
@@ -76,7 +75,8 @@ from archaeological_operations.models import (
     Operation,
     CulturalAttributionType,
 )
-from archaeological_context_records.models import ContextRecord, Dating
+from archaeological_context_records.models import ContextRecord, Dating, \
+    GeographicSubTownItem
 from archaeological_warehouse.models import Warehouse
 
 
@@ -386,7 +386,7 @@ class BFBulkView(object):
 class BaseFind(
     BulkUpdatedItem,
     BaseHistorizedItem,
-    GeoItem,
+    GeographicSubTownItem,
     CompleteIdentifierItem,
     OwnPerms,
     ValueGetter,
@@ -486,6 +486,9 @@ class BaseFind(
     def __str__(self):
         return self.label
 
+    def _get_geo_town(self):
+        return self.context_record._get_geo_town()
+
     def natural_key(self):
         return (self.uuid,)
 
diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py
index ed1a877f3..2c27eebe9 100644
--- a/ishtar_common/management/commands/migrate_to_geo_v4.py
+++ b/ishtar_common/management/commands/migrate_to_geo_v4.py
@@ -10,10 +10,11 @@ from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import BaseCommand
 
-from ishtar_common.utils import ugettext_lazy as _
+from ishtar_common.utils import ugettext_lazy as _, get_log_time, get_percent, get_eta
 
 from ishtar_common import models_common, models
 from archaeological_operations.models import Operation, ArchaeologicalSite
+from archaeological_context_records.models import ContextRecord
 
 
 log_path = os.sep.join([settings.ROOT_PATH, "logs"])
@@ -21,6 +22,14 @@ if not os.path.exists(log_path):
     os.mkdir(log_path, mode=0o770)
 
 
+def write_output(model_name, idx, nb, ref_time=None):
+    lbl = f"\r[{get_percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}"
+    if ref_time:
+        lbl += f" ({get_eta(idx, nb, ref_time, datetime.datetime.now())} left)"
+    sys.stdout.write(lbl)
+    sys.stdout.flush()
+
+
 def migrate(quiet=False, log=True):
     changed = []
     # create towns
@@ -35,10 +44,10 @@ def migrate(quiet=False, log=True):
     provider, __ = models_common.GeoProviderType.objects.get_or_create(
         txt_idx="france-ign", defaults={"label": "IGN"}
     )
+    ref_time = datetime.datetime.now()
     for idx, town in enumerate(q.all()):
         if not quiet:
-            sys.stdout.write(f"\r[{percent(idx, nb)}] Migrate towns {idx + 1}/{nb}")
-            sys.stdout.flush()
+            write_output("town", idx, nb, ref_time)
         attrs = {
             "name": town._generate_cached_label(),
             "source_content_type": town_content_type,
@@ -56,7 +65,7 @@ def migrate(quiet=False, log=True):
         town.main_geodata = data
         town.save()
     if not quiet and nb:
-        sys.stdout.write(f"\r[{get_time()}] Towns migrated\n")
+        sys.stdout.write(f"\r[{get_log_time()}] Towns migrated                 \n")
         sys.stdout.flush()
 
     model_list = [
@@ -75,14 +84,13 @@ def migrate(quiet=False, log=True):
             defaults={"label": f"Emprise {model_full_name}"},
         )
         data_type_center, __ = models_common.GeoDataType.objects.get_or_create(
-            txt_idx="operation-center", defaults={"label": f"Centre {model_full_name}"}
+            txt_idx=f"{model_slug}-center",
+            defaults={"label": f"Centre {model_full_name}"},
         )
+        ref_time = datetime.datetime.now()
         for idx, obj in enumerate(q.all()):
             if not quiet:
-                sys.stdout.write(
-                    f"\r[{percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}"
-                )
-                sys.stdout.flush()
+                write_output(model_name, idx, nb, ref_time)
 
             obj._no_move = True
             obj.skip_history_when_saving = True
@@ -155,11 +163,119 @@ def migrate(quiet=False, log=True):
                         ["geovectordata", data.name, data.pk, f"Point {model_name}"]
                     )
         if not quiet and nb:
-            sys.stdout.write(f"\r[{get_time()}] {model_name.capitalize()} migrated\n")
+            sys.stdout.write(
+                f"\r[{get_log_time()}] {model_name.capitalize()} migrated"
+                + " " * 20
+                + "\n"
+            )
             sys.stdout.flush()
 
+    model_list = [
+        (
+            "archaeological_context_records",
+            "contextrecord",
+            "unité d'enregistrement",
+            "de l'unité d'enregistrement",
+            ContextRecord,
+        ),
+        (
+            "archaeological_finds",
+            "basefind",
+            "mobilier d'origine",
+            "du mobilier d'origine",
+            ContextRecord,
+        ),
+    ]
+    for app, model_slug, model_name, model_full_name, model in model_list:
+        model_content_type = ContentType.objects.get(app_label=app, model=model_slug)
+        q = model.objects.exclude(main_geodata__isnull=False)
+        nb = q.count()
+        data_type_outline, __ = models_common.GeoDataType.objects.get_or_create(
+            txt_idx=f"{model_slug}-outline",
+            defaults={"label": f"Contour d'{model_name}"},
+        )
+        data_type_center, __ = models_common.GeoDataType.objects.get_or_create(
+            txt_idx=f"{model_slug}-center",
+            defaults={"label": f"Centre {model_full_name}"},
+        )
+        ref_time = datetime.datetime.now()
+        for idx, obj in enumerate(q.all()[:100]):
+            if not quiet:
+                write_output(model_name, idx, nb, ref_time)
+            obj._no_move = True
+            obj.skip_history_when_saving = True
+            obj.save()  # auto manage geo town association
+
+            if obj.main_geodata:
+                changed.append(
+                    [model_slug, str(obj), obj.pk, "Association géo de zone communale"]
+                )
+            if obj.multi_polygon_source == "P" and obj.multi_polygon:
+                attrs = {
+                    "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                    "source_content_type": model_content_type,
+                    "source_id": obj.pk,
+                    "multi_polygon": obj.multi_polygon,
+                    "data_type": data_type_outline,
+                }
+                data = models_common.GeoVectorData.objects.create(**attrs)
+                obj.main_geodata = data
+                obj.save()
+                changed.append(
+                    [
+                        "geovectordata",
+                        data.name,
+                        data.pk,
+                        f"Multi-polygone {model_name}",
+                    ]
+                )
+            if obj.point_source == "P" and obj.point_2d:
+                if obj.x and obj.y:
+                    attrs = {
+                        "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                        "source_content_type": model_content_type,
+                        "source_id": obj.pk,
+                        "data_type": data_type_center,
+                        "x": obj.x,
+                        "y": obj.y,
+                        "z": obj.z,
+                    }
+                    data = models_common.GeoVectorData.objects.create(**attrs)
+                    obj.main_geodata = data
+                    obj.save()
+                    changed.append(
+                        [
+                            "geovectordata",
+                            data.name,
+                            data.pk,
+                            f"Coordonnées {model_name}",
+                        ]
+                    )
+                elif obj.point_2d:
+                    attrs = {
+                        "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}",
+                        "source_content_type": model_content_type,
+                        "source_id": obj.pk,
+                        "data_type": data_type_center,
+                    }
+                    if obj.point:
+                        attrs["point_3d"] = obj.point
+                    else:
+                        attrs["point_2d"] = obj.point_2d
+                    data = models_common.GeoVectorData.objects.create(**attrs)
+                    obj.main_geodata = data
+                    obj.save()
+                    changed.append(
+                        ["geovectordata", data.name, data.pk, f"Point {model_name}"]
+                    )
+    if not quiet and nb:
+        sys.stdout.write(
+            f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 + "\n"
+        )
+        sys.stdout.flush()
+
     if log and changed:
-        filename = f"geo_migration-created-{get_time().replace(':', '')}.csv"
+        filename = f"geo_migration-created-{get_log_time().replace(':', '')}.csv"
         path = os.sep.join([log_path, filename])
         with open(path, "w+") as fle:
             writer = csv.writer(fle)
@@ -167,15 +283,7 @@ def migrate(quiet=False, log=True):
             for change in changed:
                 writer.writerow(change)
         if not quiet:
-            sys.stdout.write(f"[{get_time()}] Log: {path} written\n")
-
-
-def percent(current, total):
-    return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
-
-
-def get_time():
-    return datetime.datetime.now().isoformat().split(".")[0]
+            sys.stdout.write(f"[{get_log_time()}] Log: {path} written\n")
 
 
 class Command(BaseCommand):
@@ -193,11 +301,11 @@ class Command(BaseCommand):
         log = options["log"]
         quiet = options["quiet"]
         if not quiet:
-            sys.stdout.write(f"[{get_time()}] Processing migration\n")
+            sys.stdout.write(f"[{get_log_time()}] Processing migration\n")
         errors = migrate(quiet=quiet, log=log)
         if not errors:
             if not quiet:
-                sys.stdout.write(f"[{get_time()}] Migration finished\n")
+                sys.stdout.write(f"[{get_log_time()}] Migration finished\n")
             sys.exit()
         if not quiet:
             sys.stdout.write("\n".join(errors))
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index fc302166b..a7e07ebde 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -2335,3 +2335,21 @@ def create_osm_town(rel_id, name, numero_insee=None):
     town.center = town.limit.centroid
     town.save()
     return town
+
+
+def get_percent(current, total):
+    return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
+
+
+def get_log_time():
+    return datetime.datetime.now().isoformat().split(".")[0]
+
+
+def get_eta(current, total, base_time, current_time):
+    if current < 5:
+        return "-"
+    elapsed_time = current_time - base_time
+    eta = elapsed_time.seconds / current * (total - current)
+    if eta < 1:
+        return "-"
+    return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}"