Geodata - geo forms: new forms - many adaptations

14 files changed, 376 insertions, 14 deletions

diff --git a/UPGRADE_V4.md b/UPGRADE_V4.md
new file mode 100644
index 000000000..33a2448db
--- /dev/null
+++ b/UPGRADE_V4.md
@@ -0,0 +1,14 @@
+# backup database and media
+sudo vim /etc/apt/sources.list.d/iggdrasil.list
+# buster -> bullseye
+sudo apt update
+sudo apt upgrade
+screen  # data migration is long...
+# for each instance in /etc/ishtar/instances
+cd /srv/ishtar/{instance_name}
+# update fixtures
+./manage.py loaddata /usr/share/python3-django-ishtar/fixtures/initial_data-auth-fr.json 
+# migrate to new geo management
+./manage.py migrate_to_geo_v4
+# edit profile type permissions for geovectordata
+-> http(s)://{my-ihstar}/admin/ishtar_common/profiletypesummary/
\ No newline at end of file
diff --git a/archaeological_finds/urls.py b/archaeological_finds/urls.py
index ae12c2cbd..4bb771359 100644
--- a/archaeological_finds/urls.py
+++ b/archaeological_finds/urls.py
@@ -631,6 +631,13 @@ urlpatterns = [
         "api/get/find/<int:pk>/", views_api.GetFindAPI.as_view(),
         name="api-get-find"
     ),
+    url(
+        r"autocomplete-basefind/$",
+        check_rights(["view_basefind", "view_own_basefind"])(
+            views.autocomplete_basefind
+        ),
+        name="autocomplete-basefind",
+    ),
 ]
 
 urlpatterns += get_urls_for_model(models.Find, views, own=True, autocomplete=True)
diff --git a/archaeological_finds/views.py b/archaeological_finds/views.py
index 4def2e14b..d643c81b7 100644
--- a/archaeological_finds/views.py
+++ b/archaeological_finds/views.py
@@ -115,6 +115,7 @@ get_find_inside_container = get_item(
 )
 
 autocomplete_find = get_autocomplete_item(model=models.Find)
+autocomplete_basefind = get_autocomplete_item(model=models.BaseFind)
 
 show_treatment = show_item(models.Treatment, "treatment")
 revert_treatment = revert_item(models.Treatment)
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 6e9e11006..695638120 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -30,6 +30,7 @@ from urllib.parse import urlparse, quote
 import zipfile
 
 from django import forms
+from django.contrib.gis import forms as gis_forms
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
@@ -71,7 +72,8 @@ from .forms import (
     LockForm,
 )
 from ishtar_common.data_importer import ImporterError
-from ishtar_common.utils import is_downloadable, clean_session_cache, max_size_help
+from ishtar_common.utils import is_downloadable, clean_session_cache, max_size_help, \
+    reverse_coordinates
 
 from archaeological_operations.models import Operation, OperationType
 from archaeological_context_records.models import ContextRecord
@@ -1972,7 +1974,7 @@ class DocumentForm(forms.ModelForm, CustomForm, ManageOldType):
             if cleaned_data.get(rel, None):
                 return cleaned_data
         raise forms.ValidationError(
-            _("A document has to be attached at least " "to one item")
+            _("A document has to be attached at least to one item")
         )
 
     def clean_publisher(self):
@@ -2562,3 +2564,201 @@ class QRSearchForm(forms.Form):
             os.makedirs(dest_dir)
         shutil.move(filename, dest_dir)
         return os.path.join(settings.MEDIA_URL, "tmp", base_filename)
+
+
+class GISForm(forms.ModelForm, CustomForm, ManageOldType):
+    form_label = _("Geo item")
+    form_admin_name = _("Geo item - General")
+    form_slug = "geoitem-general"
+
+    extra_form_modals = []
+    associated_models = {
+        "data_type": models.GeoDataType,
+        "origin": models.GeoOriginType,
+        "provider": models.GeoProviderType,
+    }
+
+    pk = forms.IntegerField(label="", required=False, widget=forms.HiddenInput)
+    name = forms.CharField(
+        label=_("Name"),
+        required=False,
+        validators=[validators.MaxLengthValidator(500)],
+    )
+    import_key = forms.CharField(
+        label=_("Import key"),
+        required=False,
+        disabled=True,
+        help_text=_("An update via import corresponding to the source element and "
+                    "this key will overwrite the data."),
+    )
+    source_content_type_id = forms.IntegerField(
+        label="", required=True, widget=forms.HiddenInput, disabled=True)
+    source_id = forms.IntegerField(
+        label="", required=True, widget=forms.HiddenInput, disabled=True)
+    data_type = widgets.ModelChoiceField(
+        model=models.GeoDataType, label=_("Data type"), choices=[], required=False
+    )
+    origin = widgets.ModelChoiceField(
+        model=models.GeoOriginType, label=_("Origin"), choices=[], required=False
+    )
+    provider = widgets.ModelChoiceField(
+        model=models.GeoProviderType, label=_("Provider"), choices=[], required=False
+    )
+    comment = forms.CharField(label=_("Comment"), widget=forms.Textarea, required=False)
+
+    TYPES = [
+        FieldType("origin", models.GeoOriginType),
+        FieldType("data_type", models.GeoDataType),
+        FieldType("provider", models.GeoProviderType),
+    ]
+
+    class Meta:
+        model = models.GeoVectorData
+        exclude = ["need_update", "imports", "cached_x", "cached_y", "cached_z",
+                   "point_3d"]
+
+    HEADERS = {
+        "related_items_ishtar_common_town": FormHeader(
+            _("Related items"), collapse=True),
+        "name": FormHeader(_("Meta-data")),
+        "geo_field": FormHeader(_("Geography")),
+    }
+    OPTIONS_PERMISSIONS = [
+        # field name, permission, options
+        ("tags", ("ishtar_common.add_documenttag",), {"new": True}),
+    ]
+
+    GEO_FIELDS = (
+        ("point_2d",),
+        ("multi_points",),
+        ("multi_line",),
+        ("multi_polygon",),
+        ("x", "z")
+    )
+
+    def __init__(self, *args, **kwargs):
+        main_items_fields = {}
+        if "main_items_fields" in kwargs:
+            main_items_fields = kwargs.pop("main_items_fields")
+        self.user = None
+        if kwargs.get("user", None):
+            self.user = kwargs.pop("user")
+        instance = kwargs.get("instance", False)
+        self.is_instancied = bool(instance)
+        super(GISForm, self).__init__(*args, **kwargs)
+        if not self.fields["import_key"].initial:
+            self.fields.pop("import_key")
+        self.source_content_type = kwargs.pop("source_content_type", None)
+        self.source_id = kwargs.pop("source_id", None)
+        if not self.source_content_type:
+            self.fields.pop("source_content_type_id")
+            self.fields.pop("source_id")
+        else:
+            self.fields["source_content_type_id"].initial = self.source_content_type
+            self.fields["source_id"].initial = self.source_id
+        self.geo_keys = []
+        if instance:
+            for keys in self.GEO_FIELDS:
+                if any(getattr(instance, key) for key in keys):
+                    if keys[0] != "x":
+                        map_srid = getattr(instance, keys[0]).srid or 4326
+                        widget = gis_forms.OSMWidget
+                        if map_srid == 4326:
+                            widget = widgets.ReversedOSMWidget
+                        self.fields[keys[0]].widget = widget(
+                            attrs={"map_srid": map_srid})
+                        self.fields.pop("spatial_reference_system")
+                        self.geo_keys = keys[:]
+                    else:
+                        self.geo_keys = [
+                            "x", "estimated_error_x",
+                            "y", "estimated_error_y",
+                            "z", "estimated_error_z",
+                            "spatial_reference_system",
+                        ]
+                    for geo_fields in self.GEO_FIELDS:
+                        if geo_fields != keys:
+                            for geo_field in geo_fields:
+                                self.fields.pop(geo_field)
+                                if geo_field == "x":
+                                    self.fields.pop("estimated_error_x")
+                                    self.fields.pop("y")
+                                    self.fields.pop("estimated_error_y")
+                                if geo_field == "z":
+                                    self.fields.pop("estimated_error_z")
+                    break
+            if not self.geo_keys:
+                # TODO....
+                pass
+
+        fields = OrderedDict()
+        for related_key in models.GeoVectorData.RELATED_MODELS:
+            model = models.GeoVectorData._meta.get_field(related_key).related_model
+            fields[related_key] = widgets.Select2MultipleField(
+                model=model,
+                remote=True,
+                label=model._meta.verbose_name_plural,
+                required=False,
+                style="width: 100%",
+            )
+            if related_key in main_items_fields:
+                for field_key, label in main_items_fields[related_key]:
+                    disabled = False
+                    if kwargs.get("initial", None) and kwargs["initial"].get(
+                            field_key, False
+                    ):
+                        disabled = True
+                    fields[field_key] = forms.BooleanField(
+                        label=label, required=False, disabled=disabled
+                    )
+        for k in self.geo_keys:
+            fields[k] = self.fields[k]
+        for k in self.fields:
+            if k not in self.geo_keys:
+                fields[k] = self.fields[k]
+        self.fields = fields
+
+    def get_headers(self):
+        headers = self.HEADERS.copy()
+        if self.geo_keys:
+            headers[self.geo_keys[0]] = headers.pop("geo_field")
+        return headers
+
+    def clean(self):
+        cleaned_data = self.cleaned_data
+        if "x" not in self.geo_keys:
+            # reverse...
+            geo_value = cleaned_data[self.geo_keys[0]]
+            if geo_value:
+                if not isinstance(geo_value, str):
+                    geo_value = geo_value.ewkt
+                if geo_value.startswith("SRID=4326;"):
+                    cleaned_data[self.geo_keys[0]] = reverse_coordinates(geo_value)
+        for rel in models.GeoVectorData.RELATED_MODELS:
+            if cleaned_data.get(rel, None):
+                return cleaned_data
+        raise forms.ValidationError(
+            _("A geo item has to be attached at least to one item")
+        )
+
+    def save(self, commit=True):
+        item = super().save(commit=commit)
+        for related_key in models.GeoVectorData.RELATED_MODELS:
+            related = getattr(item, related_key)
+            initial = dict([(rel.pk, rel) for rel in related.all()])
+            new = [int(pk) for pk in sorted(self.cleaned_data.get(related_key, []))]
+            for pk, value in initial.items():
+                if pk in new:
+                    continue
+                related.remove(value)
+            for new_pk in new:
+                related_item = related.model.objects.get(pk=new_pk)
+                if new_pk not in initial.keys():
+                    related.add(related_item)
+        item = models.GeoVectorData.objects.get(pk=item.pk)
+        if self.user:
+            item.history_creator = self.user
+        item.history_modifier = self.user
+        item.skip_history_when_saving = True
+        item.save()  # resave to regen the attached items
+        return item
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 2bd983906..870cce90b 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -145,6 +145,8 @@ from ishtar_common.models_common import (
     DynamicRequest,
     GeoItem,
     GeoDataType,
+    GeoOriginType,
+    GeoProviderType,
     GeoVectorData,
     CompleteIdentifierItem,
     SearchVectorConfig,
@@ -204,6 +206,9 @@ __all__ = [
     "State",
     "CompleteIdentifierItem",
     "GeoVectorData",
+    "GeoDataType",
+    "GeoOriginType",
+    "GeoProviderType",
 ]
 
 logger = logging.getLogger(__name__)
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 35dd4aef1..0c2d67140 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -2100,6 +2100,15 @@ GEOMETRY_TYPE_LBL = {
 
 class GeoVectorData(Imported, OwnPerms):
     SLUG = "geovectordata"
+    RELATED_MODELS = [
+        "related_items_ishtar_common_town",
+        "related_items_archaeological_operations_operation",
+        "related_items_archaeological_operations_archaeologicalsite",
+        "related_items_archaeological_context_records_contextrecord",
+        "related_items_archaeological_finds_basefind",
+        "related_items_archaeological_warehouse_warehouse",
+        "related_items_archaeological_warehouse_container",
+    ]
 
     name = models.TextField(_("Name"), default="-")
     source_content_type = models.ForeignKey(
@@ -2115,6 +2124,7 @@ class GeoVectorData(Imported, OwnPerms):
         null=True,
         on_delete=models.PROTECT,
         verbose_name=_("Origin"),
+        help_text=_("For instance: topographical surveys, georeferencing, ..."),
     )
     data_type = models.ForeignKey(
         GeoDataType,
@@ -2122,6 +2132,7 @@ class GeoVectorData(Imported, OwnPerms):
         null=True,
         on_delete=models.PROTECT,
         verbose_name=_("Data type"),
+        help_text=_("For instance: outline, z-sup, ..."),
     )
     provider = models.ForeignKey(
         GeoProviderType,
@@ -2129,6 +2140,7 @@ class GeoVectorData(Imported, OwnPerms):
         null=True,
         on_delete=models.PROTECT,
         verbose_name=_("Provider"),
+        help_text=_("Data provider"),
     )
     comment = models.TextField(_("Comment"), default="", blank=True)
     x = models.FloatField(_("X"), blank=True, null=True, help_text=_("User input"))
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 6f6d7109e..1fb5d4fcc 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -64,6 +64,7 @@ from ishtar_common.utils import (
     num2col,
     max_size_help,
     import_class,
+    reverse_coordinates,
 )
 from ishtar_common.data_importer import (
     Importer,
@@ -1086,14 +1087,6 @@ def delayed_check(import_pk):
     imp.check_modified()
 
 
-RE_NUMBER = r"[+-]?\d+(?:\.\d*)?"
-RE_COORDS = r"(" + RE_NUMBER + r") (" + RE_NUMBER + r")"
-
-
-def _reverse_coordinates(wkt):
-    return re.sub(RE_COORDS, r"\2 \1", wkt)
-
-
 def convert_geom(feature, srid):
     geo_type = feature["type"]
     if geo_type in ("LineString", "Polygon"):
@@ -1110,7 +1103,7 @@ def convert_geom(feature, srid):
         srs = profile.srs.srid
     if srs != srid:
         # Coordinates are reversed - should be fixed on Django 3.2
-        feature = _reverse_coordinates(
+        feature = reverse_coordinates(
             GEOSGeometry(feature).transform(srs, clone=True).ewkt)
     return feature
 
diff --git a/ishtar_common/templates/blocks/bs_form_snippet.html b/ishtar_common/templates/blocks/bs_form_snippet.html
index 3d84ce3dc..769234949 100644
--- a/ishtar_common/templates/blocks/bs_form_snippet.html
+++ b/ishtar_common/templates/blocks/bs_form_snippet.html
@@ -72,7 +72,7 @@
             </div>
             <div class="modal-body body-scroll search-fields">
 {% endif %}
-{% if field.name in form.HEADERS %}
+{% if field.name in form.get_headers %}
 {% if forloop.counter0 %}
 </div>{% endif %}
 
diff --git a/ishtar_common/templates/ishtar/blocks/sheet_geographic.html b/ishtar_common/templates/ishtar/blocks/sheet_geographic.html
index 69cd67fcc..e1a8a1200 100644
--- a/ishtar_common/templates/ishtar/blocks/sheet_geographic.html
+++ b/ishtar_common/templates/ishtar/blocks/sheet_geographic.html
@@ -14,7 +14,7 @@
 {% for geo in geo_item.geodata.all %}
     <tr>
         {% if permission_change_geo %}
-        <td><a href="#">{% if geo|can_edit_item:request %}<i class="fa fa-pencil"></i></a>{% else %}&ndash;{% endif %}</td>
+        <td><a href="{% url 'edit-geo' geo.pk %}">{% if geo|can_edit_item:request %}<i class="fa fa-pencil"></i></a>{% else %}&ndash;{% endif %}</td>
         {% endif %}
         <td>{% if geo.id == geo_item.main_geodata_id %}<i class="fa fa-check-circle text-success" aria-hidden="true"></i>{% else %}&ndash;{% endif %}</td>
         <td>{% if geo.data_type %}{{ geo.data_type }}{% else %}-{% endif %}</td>
diff --git a/ishtar_common/templates/ishtar/forms/document.html b/ishtar_common/templates/ishtar/forms/base_related_items.html
index fe3df8c74..fe3df8c74 100644
--- a/ishtar_common/templates/ishtar/forms/document.html
+++ b/ishtar_common/templates/ishtar/forms/base_related_items.html
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index ffa8daebf..60a2d8767 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -575,6 +575,20 @@ urlpatterns += [
         name="new-documenttag",
     ),
     url(
+        r"geo/create/(?P<app_source>[-\w]+)/(?P<model_source>[-\w]+)/(?P<source_pk>\d+)/$",
+        check_rights(["add_geovectordata", "add_own_geovectordata"])(
+            views.GeoCreateView.as_view()
+        ),
+        name="create-geo",
+    ),
+    url(
+        r"geo/edit/(?P<pk>\d+)/$",
+        check_rights(["change_geovectordata", "change_own_geovectordata"])(
+            views.GeoEditView.as_view()
+        ),
+        name="edit-geo",
+    ),
+    url(
         r"^qa-not-available(?:/(?P<context>[0-9a-z-]+))?/$",
         views.QANotAvailable.as_view(),
         name="qa-not-available",
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 3202afede..2a41ab0aa 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -2355,3 +2355,11 @@ def get_eta(current, total, base_time, current_time):
     if eta < 1:
         return "-"
     return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}"
+
+
+RE_NUMBER = r"[+-]?\d+(?:\.\d*)?"
+RE_COORDS = r"(" + RE_NUMBER + r") (" + RE_NUMBER + r")"
+
+
+def reverse_coordinates(wkt):
+    return re.sub(RE_COORDS, r"\2 \1", wkt)
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index b6686e0bd..361c32022 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -2131,7 +2131,7 @@ document_search_wizard = wizards.DocumentSearch.as_view(
 
 class DocumentFormMixin(IshtarMixin, LoginRequiredMixin):
     form_class = forms.DocumentForm
-    template_name = "ishtar/forms/document.html"
+    template_name = "ishtar/forms/base_related_items.html"
     model = models.Document
 
     def get_context_data(self, **kwargs):
@@ -2711,3 +2711,100 @@ class DisplayItemView(IshtarMixin, TemplateView):
         else:
             data["show_url"] = "/show-{}/{}/".format(item_type, pk)
         return data
+
+
+class GeoFormMixin(IshtarMixin, LoginRequiredMixin):
+    form_class = forms.GISForm
+    template_name = "ishtar/forms/base_related_items.html"
+    model = models.GeoVectorData
+
+    def _get_source(self, request):
+        self.success_url = request.GET.get("source_url")
+
+    def get(self, request, *args, **kwargs):
+        self._get_source(request)
+        return super().get(request, *args, **kwargs)
+
+    def post(self, request, *args, **kwargs):
+        self._get_source(request)
+        return super().post(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        data = super(GeoFormMixin, self).get_context_data(**kwargs)
+        data["extra_form_modals"] = self.form_class.extra_form_modals
+        return data
+
+    def get_success_url(self):
+        if not self.success_url:
+            return reverse("edit-geo", kwargs={"pk": self.object.pk})
+        return f"{self.success_url}?open_item={self.object.pk}"
+
+
+class GeoEditView(GeoFormMixin, UpdateView):
+    page_name = _("Geo item modification")
+
+    def get_form_kwargs(self):
+        kwargs = super(GeoEditView, self).get_form_kwargs()
+        try:
+            geo = models.GeoVectorData.objects.get(pk=self.kwargs.get("pk"))
+            assert check_permission(self.request, "geo/edit", geo.pk)
+        except (AssertionError, models.GeoVectorData.DoesNotExist):
+            raise Http404()
+        initial = {}
+
+        for k in (
+                list(self.form_class.base_fields.keys()) +
+                models.GeoVectorData.RELATED_MODELS
+        ):
+            value = getattr(geo, k)
+            if hasattr(value, "all"):
+                value = ",".join([str(v.pk) for v in value.all()])
+            if hasattr(value, "pk"):
+                value = value.pk
+            initial[k] = value
+
+        kwargs["initial"] = initial
+        kwargs["user"] = self.request.user
+        self.geo = geo
+        return kwargs
+
+    def get_context_data(self, **kwargs):
+        kwargs = super(GeoEditView, self).get_context_data(**kwargs)
+        return kwargs
+
+
+class GeoCreateView(GeoFormMixin, UpdateView):
+    page_name = _("Geo item creation")
+
+    def get_form_kwargs(self):
+        if not hasattr(self.request.user, "ishtaruser"):
+            raise Http404()
+        ishtaruser = self.request.user.ishtaruser
+        kwargs = super(GeoCreateView, self).get_form_kwargs()
+        try:
+            content_type = ContentType.objects.get(
+                app_label=self.kwargs.get("app_source"),
+                model=self.kwargs.get("model_source")
+            )
+        except ContentType.DoesNotExist:
+            raise Http404()
+        model = content_type.model_class()
+        try:
+            obj = model.objects.get(pk=self.kwargs.get("source_pk"))
+        except model.DoesNotExist:
+            raise Http404()
+        if not ishtaruser.has_perm("add_geovectordata"): # -> add_own_geovectordata
+            # check permission to view attached item
+            if not getattr(model, "SLUG", None):
+                raise Http404()
+            if not ishtaruser.has_right(f"view_{model.SLUG}") \
+                    or not ishtaruser.has_right(f"view_own_{model.SLUG}") \
+                    or not obj.is_own(ishtaruser):
+                # check permission to view own attached item
+                raise Http404()
+        kwargs["user"] = self.request.user
+        return kwargs
+
+    def get_context_data(self, **kwargs):
+        kwargs = super(GeoCreateView, self).get_context_data(**kwargs)
+        return kwargs
diff --git a/ishtar_common/widgets.py b/ishtar_common/widgets.py
index 05605a258..2b7204dc3 100644
--- a/ishtar_common/widgets.py
+++ b/ishtar_common/widgets.py
@@ -23,6 +23,7 @@ import logging
 
 from django import forms
 from django.conf import settings
+from django.contrib.gis import forms as gis_forms
 from django.core.exceptions import ValidationError
 from django.core.files import File
 from django.db.models import fields
@@ -43,6 +44,7 @@ from json import JSONEncoder
 from django.utils.translation import ugettext_lazy as _
 
 from ishtar_common import models
+from ishtar_common.utils import reverse_coordinates
 
 logger = logging.getLogger(__name__)
 
@@ -1334,3 +1336,12 @@ class DataTable(Select2Media, forms.RadioSelect):
 
 class RangeInput(NumberInput):
     input_type = "range"
+
+
+class ReversedOSMWidget(gis_forms.OSMWidget):
+    def get_context(self, name, value, attrs):
+        if value:
+            if not isinstance(value, str):  # should be geo
+                value = reverse_coordinates(value.ewkt)
+        context = super().get_context(name, value, attrs)
+        return context