Geodata redesign: operation migrations (1/2) - operation geo post save - town area creation

6 files changed, 214 insertions, 11 deletions

diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index 9d0018f28..8fa72b423 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -36,6 +36,7 @@ from django.urls import reverse
 from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy
 
 from ishtar_common.models import (
+    Area,
     BaseHistorizedItem,
     Dashboard,
     DashboardFormItem,
@@ -2066,6 +2067,82 @@ class Operation(
         res["mode"] = " ; ".join([str(m) for m in mode(finds)])
         return res
 
+    def post_save_geo(self, save=True):
+        # manage geodata towns
+        if getattr(self, "_post_saved_geo", True):
+            # prevent infinite loop - should not happen, but...
+            return
+        self._post_saved_geo = True
+        q_towns = self.towns.filter(main_geodata__multi_polygon__isnull=False)
+        q_towns_nb = q_towns.count()
+        q_geodata_town = self.geodata.filter(
+            source_content_type__model="town",
+            source_content_type__app_label="ishtar_common")
+        q_geodata_area = self.geodata.filter(
+            source_content_type__model="area",
+            source_content_type__app_label="ishtar_common")
+        changed = False
+        if q_towns_nb != 1:
+            # no simple town - clean
+            for geo in q_geodata_town.all():
+                self.geodata.remove(geo)
+                if self.main_geodata == geo:
+                    self.main_geodata = None
+                changed = True
+        if q_towns_nb < 2:
+            # no area - clean
+            for geo in q_geodata_area.all():
+                self.geodata.remove(geo)
+                if self.main_geodata == geo:
+                    self.main_geodata = None
+                changed = True
+
+        current_geo_town = None
+        if q_towns_nb == 1:
+            current_geo_town = q_towns.all()[0]
+            if not q_geodata_town.filter(pk=current_geo_town.pk).count():
+                for geo in q_geodata_town.all():
+                    self.geodata.remove(geo)
+                    if self.main_geodata == geo:
+                        self.main_geodata = None
+                self.geodata.add(current_geo_town)
+                changed = True
+
+        current_geo_area = None
+        if q_towns_nb > 1:
+            current_geo_area = Area.get_or_create_by_towns(q_towns, get_geo=True)
+            if current_geo_area and not q_geodata_area.filter(pk=current_geo_area.pk).count():
+                for geo in q_geodata_area.all():
+                    self.geodata.remove(geo)
+                    if self.main_geodata == geo:
+                        self.main_geodata = None
+                self.geodata.add(current_geo_area)
+                changed = True
+
+        if current_geo_town:
+            q_extra_geo_town = q_geodata_town.exclude(pk=current_geo_town.pk)
+            if q_extra_geo_town.count():
+                # should not occur but bad migrations, bad imports...
+                for geo in q_extra_geo_town.all():
+                    self.geodata.remove(geo)
+                    if self.main_geodata == geo:
+                        self.main_geodata = None
+                changed = True
+        if current_geo_area:
+            q_extra_geo_area = q_geodata_area.exclude(pk=current_geo_area.pk)
+            if q_extra_geo_area.count():
+                # should not occur but bad migrations, bad imports...
+                for geo in q_extra_geo_area.all():
+                    self.geodata.remove(geo)
+                    if self.main_geodata == geo:
+                        self.main_geodata = None
+                changed = True
+
+        if changed and save:
+            self.skip_history_when_saving = True
+            self._no_move = True
+            self.save()
+
     def save(self, *args, **kwargs):
         # put a default year if start_date is defined
         if self.start_date and not self.year:
diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py
index f747cb72d..df275bd85 100644
--- a/ishtar_common/management/commands/migrate_to_geo_v4.py
+++ b/ishtar_common/management/commands/migrate_to_geo_v4.py
@@ -1,16 +1,19 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from django.contrib.contenttypes.models import ContentType
 import csv
 import datetime
 import os
 import sys
 
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import BaseCommand
 
-from ishtar_common import models_common
+from ishtar_common.utils import ugettext_lazy as _
+
+from ishtar_common import models_common, models
+from archaeological_operations.models import Operation
 
 
 log_path = os.sep.join([settings.ROOT_PATH, "logs"])
@@ -22,13 +25,16 @@ def migrate(quiet=False, log=True):
     changed = []
     # create towns
     q = models_common.Town.objects.exclude(
-        center__isnull=True, limit__isnull=True).exclude(main_geodata__isnull=False)
+        center__isnull=True, limit__isnull=True
+    ).exclude(main_geodata__isnull=False)
     nb = q.count()
-    town_content_type = ContentType.objects.get(app_label='ishtar_common', model='town')
+    town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town")
     data_type, __ = models_common.GeoDataType.objects.get_or_create(
-        txt_idx="town-limit", defaults={"label": "Limites commune"})
+        txt_idx="town-limit", defaults={"label": "Limites commune"}
+    )
     provider, __ = models_common.GeoProviderType.objects.get_or_create(
-        txt_idx="france-ign", defaults={"label": "IGN"})
+        txt_idx="france-ign", defaults={"label": "IGN"}
+    )
     for idx, town in enumerate(q.all()):
         if not quiet:
             sys.stdout.write(f"\r[{percent(idx, nb)}] Migrate towns {idx + 1}/{nb}")
@@ -46,16 +52,61 @@ def migrate(quiet=False, log=True):
             attrs["point_2d"] = town.center
         data, created = models_common.GeoVectorData.objects.get_or_create(**attrs)
         if created:
-            changed.append(["geovectordata", data.name, data.pk])
+            changed.append(["geovectordata", data.name, data.pk, "Création commune"])
         town.main_geodata = data
         town.save()
 
+    # manage operation vector sources
+    operation_content_type = ContentType.objects.get(
+        app_label="archaeological_operations", model="operation"
+    )
+    q = Operation.objects.exclude(main_geodata__isnull=False)
+    nb = q.count()
+    for idx, operation in enumerate(q.all()):
+        if operation.multi_polygon_source == "T":
+            operation._no_move = True
+            operation.skip_history_when_saving = True
+            operation.save()  # auto managed
+        elif operation.multi_polygon_source == "P":
+            # TODO
+
+            """
+            q = operation.towns.filter(limit__isnull=False)
+            nb = q.count
+            if not nb:
+                break
+            elif nb == 1:
+                town = q.all()[0]
+                geo = models_common.GeoVectorData.objects.get(
+                    source_content_type=town_content_type,
+                    source_id=town.pk
+                )
+            else:
+
+                attrs = {
+                    "name": name,
+                    "source_content_type": town_content_type,
+                    "source_id": town.pk,
+                    "data_type": data_type,
+                    "provider": provider,
+                    "multi_polygon": poly,
+                }
+                geo, created = models_common.GeoVectorData.objects.get_or_create(
+                    **attrs)
+
+            operation.main_geodata = geo
+            operation.skip_history_when_saving = True
+            operation._no_move = True
+            operation.save()
+            """
+
+
     if log and changed:
         filename = f"geo_migration-created-{get_time().replace(':', '')}.txt"
         path = os.sep.join([log_path, filename])
-        with open(path, 'w+') as fle:
+        with open(path, "w+") as fle:
             writer = csv.writer(fle)
-            writer.writerow(["model", "name", "id"])
+            writer.writerow(["model", "name", "id", "context"])
             for change in changed:
                 writer.writerow(change)
         if not quiet:
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a03b90434..a1f91282e 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -52,6 +52,7 @@ from django.contrib.auth.models import User, Group
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.gis.db import models
+from django.contrib.gis.db.models.aggregates import Union
 from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.sites.models import Site
@@ -141,6 +142,8 @@ from ishtar_common.models_common import (
     SearchAltName,
     DynamicRequest,
     GeoItem,
+    GeoDataType,
+    GeoVectorData,
     CompleteIdentifierItem,
     SearchVectorConfig,
     DocumentItem,
@@ -2251,8 +2254,8 @@ class Area(HierarchicalType):
     )
 
     class Meta:
-        verbose_name = _("Area")
-        verbose_name_plural = _("Areas")
+        verbose_name = _("Town - Area")
+        verbose_name_plural = _("Town - Areas")
         ordering = ("label",)
 
     def __str__(self):
@@ -2260,6 +2263,64 @@ class Area(HierarchicalType):
             return self.label
         return "{} ({})".format(self.label, self.reference)
 
+    @classmethod
+    def get_or_create_by_towns(cls, towns, get_geo=False):
+        if hasattr(towns, "all"):  # queryset
+            if not towns.count():
+                return
+            towns = towns.all()
+        elif not len(towns):
+            return
+        name = []
+        reference = []
+        for town in sorted(towns, key=lambda x: (x.numero_insee, x.name)):
+            name.append(town._generate_cached_label())
+            reference.append(town.numero_insee or slugify(town.name))
+        name = " / ".join(name)
+        reference = f"{_('area')}-{'/'.join(reference)}"
+        area, created = cls.objects.get_or_create(
+            reference=reference,
+            defaults={"label": name}
+        )
+
+        area_content_type = ContentType.objects.get(app_label="ishtar_common",
+                                                    model="area")
+        attrs = {
+            "source_content_type": area_content_type,
+            "source_id": area.pk,
+        }
+        q = GeoVectorData.objects.filter(**attrs)
+        if created or not q.count():
+            data_type, __ = GeoDataType.objects.get_or_create(
+                txt_idx="area-limit",
+                defaults={"label": str(_("Communal area boundaries"))}
+            )
+            attrs["data_type"] = data_type
+            geo = GeoVectorData.objects.create(**attrs)
+        else:
+            geo = q.all()[0]
+
+        q_poly_towns = GeoVectorData.objects.filter(
+            source_content_type__app_label="ishtar_common",
+            source_content_type__model="town",
+            source_id__in=[t.pk for t in towns])
+        q_poly = q_poly_towns.annotate(poly=Union("multi_polygon"))
+        poly = q_poly.all()[0].poly
+        if not geo.multi_polygon or not geo.multi_polygon.equals_exact(poly, 0.001):
+            origins, providers = [], []
+            for g in q_poly_towns:
+                origins.append(g.origin)
+                providers.append(g.provider)
+            if len(set(origins)) == 1:  # no ambiguous origin
+                geo.origin = origins[0]
+            if len(set(providers)) == 1:  # no ambiguous provider
+                geo.provider = providers[0]
+            geo.multi_polygon = poly
+            geo.save()
+        if get_geo:
+            return geo
+        return area
+
     @property
     def full_label(self):
         label = [str(self)]
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index cc3c61b05..95ceee948 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -2327,6 +2327,10 @@ class GeographicItem(models.Model):
         )
         if self.main_geodata and not self.geodata.filter(pk=self.main_geodata.pk):
             self.geodata.add(self.main_geodata)
+        elif not self.main_geodata and self.geodata.count():
+            # arbitrary associate the first to geodata
+            self.main_geodata = self.geodata.order_by("pk").all()[0]
+            self.save()
 
 
 class TownManager(models.Manager):
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index eab25a56f..02eeff93c 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -769,6 +769,11 @@ def _post_save_geodata(sender, **kwargs):
         return
 
     modified = False
+    if getattr(instance, "post_save_geo", False):
+        instance.post_save_geo(save=False)
+        modified = True
+
+    # managed cached coordinates
 
     cached_x, cached_y, cached_z = None, None, None
     coords = instance.display_coordinates(rounded=False, dim=3)
@@ -794,6 +799,8 @@ def _post_save_geodata(sender, **kwargs):
 
     if modified:
         instance._post_saved_geo = True
+        instance._no_move = True
+        instance.skip_history_when_saving = True
         instance.save()
     cache_key, __ = get_cache(sender, ("post_save_geo", instance.pk))
     cache.set(cache_key, None, settings.CACHE_TASK_TIMEOUT)
diff --git a/locale/fr/LC_MESSAGES/django.po b/locale/fr/LC_MESSAGES/django.po
index 4659c9951..04a46d1e6 100644
--- a/locale/fr/LC_MESSAGES/django.po
+++ b/locale/fr/LC_MESSAGES/django.po
@@ -19,6 +19,9 @@ msgstr ""
 "Plural-Forms: nplurals=2; plural=n > 1;\n"
 "X-Generator: Weblate 4.5.3\n"
 
+msgid "Communal area boundaries"
+msgstr "Limites de zone communale"
+
 #: archaeological_context_records/admin.py:54
 #: archaeological_operations/admin.py:99 ishtar_common/models_common.py:2789
 msgid "Point"