Syndication - filter field on sheet (bis)

9 files changed, 127 insertions, 36 deletions

diff --git a/archaeological_operations/templates/ishtar/sheet_site.html b/archaeological_operations/templates/ishtar/sheet_site.html
index a7955572d..4904b5562 100644
--- a/archaeological_operations/templates/ishtar/sheet_site.html
+++ b/archaeological_operations/templates/ishtar/sheet_site.html
@@ -1,7 +1,7 @@
 {% extends "ishtar/sheet.html" %}
 {% load i18n window_tables window_header window_ope_tables window_field from_dict %}
 
-{% block head_title %}<strong>{{SITE_LABEL}}</strong> - {{item}}{% endblock %}
+{% block head_title %}<strong>{{SITE_LABEL}}</strong> - {{item.cached_label}}{% endblock %}
 
 {% block toolbar %}
 {% window_nav item window_id 'show-site' 'site_modify' 'show-historized-site' 'revert-site' previous next 1 %}
diff --git a/archaeological_operations/urls.py b/archaeological_operations/urls.py
index f177994be..e77ccc625 100644
--- a/archaeological_operations/urls.py
+++ b/archaeological_operations/urls.py
@@ -375,7 +375,7 @@ urlpatterns = [
         name="api-get-operation"
     ),
     path(
-        "api/get/archaeologicalsite/<int:pk>/", views_api.GetSiteAPI.as_view(),
+        "api/get/site/<int:pk>/", views_api.GetSiteAPI.as_view(),
         name="api-get-archaeologicalsite"
     ),
 ]
diff --git a/example_project/settings.py b/example_project/settings.py
index b98158b10..89964fe7e 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -290,6 +290,7 @@ ISHTAR_DPTS = []
 MAX_ATTEMPTS = 1  # django background tasks
 
 MAX_UPLOAD_SIZE = 100  # in Mo
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 10240
 
 # path to the "dot" program to generate graph
 DOT_BINARY = "/usr/bin/dot"
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index bb8daa372..5c2f33aa1 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -105,7 +105,7 @@ ISHTAR_FORMS = [
 class ImportGenericForm(forms.Form):
     csv_file = forms.FileField(
         label=_("CSV file"),
-        help_text=_("Only unicode encoding is managed - convert your" " file first"),
+        help_text=_("Only unicode encoding is managed - convert your file first"),
     )
 
 
@@ -2094,7 +2094,7 @@ class ApiUserAdmin(admin.ModelAdmin):
     list_display = ("user_ptr", "ip")
 
 
-admin_site.register(models.ApiUser, ApiUserAdmin)
+admin_site.register(models_rest.ApiUser, ApiUserAdmin)
 
 
 def get_main_content_types_query():
@@ -2110,7 +2110,7 @@ def get_main_content_types_query():
 
 class ApiSearchModelAdminForm(forms.ModelForm):
     class Meta:
-        model = models.ApiUser
+        model = models_rest.ApiUser
         exclude = []
 
     content_type = forms.ModelChoiceField(
@@ -2123,7 +2123,7 @@ class ApiSearchModelAdmin(admin.ModelAdmin):
     list_display = ("user", "content_type")
 
 
-admin_site.register(models.ApiSearchModel, ApiSearchModelAdmin)
+admin_site.register(models_rest.ApiSearchModel, ApiSearchModelAdmin)
 
 
 def send_error_message(request, msg, message_type=messages.ERROR):
@@ -2353,3 +2353,44 @@ class ApiKeyMatchAdmin(admin.ModelAdmin):
 
 
 admin_site.register(models_rest.ApiKeyMatch, ApiKeyMatchAdmin)
+
+
+class ApiSheetFilterForm(forms.ModelForm):
+    api_search_model = forms.ModelChoiceField(
+        models_rest.ApiSearchModel.objects,
+        label=_("API - Remote access - Search model"),
+    )
+    key = forms.ChoiceField(
+        label=_("Key"),
+        initial="-",
+        choices=(("-", "-"),),
+        help_text=_("Save first to choose a key"),
+    )
+
+    class Meta:
+        model = models_rest.ApiSheetFilter
+        fields = ["api_search_model", "key"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        instance = kwargs.get("instance")
+        if not instance:
+            return
+        if args:
+            query_dict = args[0]
+            query_dict._mutable = True
+            query_dict.setlist("api_search_model", [instance.api_search_model.pk])
+            query_dict._mutable = False
+        self.fields["api_search_model"].widget.attrs = {'disabled': 'disabled'}
+        self.fields["key"].help_text = ""
+        self.fields["key"].choices = [(k, k) for k in instance.get_keys()]
+
+
+class ApiSheetFilterAdmin(admin.ModelAdmin):
+    form = ApiSheetFilterForm
+    model = models_rest.ApiSheetFilter
+    list_display = ["api_search_model", "key"]
+    list_filter = ["api_search_model"]
+
+
+admin_site.register(models_rest.ApiSheetFilter, ApiSheetFilterAdmin)
diff --git a/ishtar_common/migrations/0218_apisheetfilter.py b/ishtar_common/migrations/0218_apisheetfilter.py
new file mode 100644
index 000000000..cf9d5f594
--- /dev/null
+++ b/ishtar_common/migrations/0218_apisheetfilter.py
@@ -0,0 +1,26 @@
+# Generated by Django 2.2.24 on 2021-12-20 12:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0217_auto_20211103_1422'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ApiSheetFilter',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('key', models.CharField(max_length=200, verbose_name='Key')),
+                ('api_search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiSearchModel')),
+            ],
+            options={
+                'verbose_name': 'API - Remote access - Sheet filter',
+                'verbose_name_plural': 'API - Remote access - Sheet filters',
+            },
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 1f0fd6095..8f916538c 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -113,7 +113,6 @@ from ishtar_common.models_imports import (
     TargetKeyGroup,
     ValueFormater,
 )
-from ishtar_common.models_rest import ApiUser, ApiSearchModel
 
 from ishtar_common.utils import (
     get_cache,
@@ -1721,24 +1720,6 @@ class CustomFormJsonField(models.Model):
         )
 
 
-class SheetFilter(models.Model):
-    key = models.CharField(_("Key"), max_length=200)
-
-    class Meta:
-        abstract = True
-
-    def get_template(self):
-        raise NotImplemented()
-
-    def get_keys(self):
-        tpl = os.path.abspath(
-            os.path.join(settings.ROOT_PATH, "..", self.get_template())
-        )
-        r = re.compile("item\.([_a-zA-Z])+")
-        with open(tpl, "r") as fle:
-            return list(set(r.findall(fle.read())))
-
-
 class GlobalVar(models.Model, Cached):
     slug = models.SlugField(_("Variable name"), unique=True)
     description = models.TextField(
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 83955ab73..3f54dd5b8 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -9,10 +9,10 @@ import copy
 from collections import OrderedDict
 import datetime
 import json
-import locale
 import logging
 import os
 import pyqrcode
+import re
 import shutil
 import tempfile
 import time
@@ -37,6 +37,7 @@ from django.core.validators import validate_slug
 from django.db import connection
 from django.db.models import Q, Count, Max
 from django.db.models.signals import post_save, post_delete, m2m_changed
+from django.template import loader
 from django.template.defaultfilters import slugify
 from django.utils.safestring import SafeText, mark_safe
 from django.utils.translation import activate, deactivate
@@ -798,6 +799,35 @@ class TemplateItem:
         return templates
 
 
+class SheetFilter(models.Model):
+    key = models.CharField(_("Key"), max_length=200)
+
+    class Meta:
+        abstract = True
+
+    def get_template(self):
+        raise NotImplemented()
+
+    def get_keys(self):
+        attrs = re.compile(r"item\.([_a-zA-Z]+)")
+        includes = re.compile(r"""\{\% *include *["'](/?(?:[^/]+/?)+)["'] *\%\}""")
+        main_template = self.get_template()
+        templates = [main_template]
+        with open(main_template, "r") as fle:
+            content = fle.read()
+        keys = attrs.findall(content)
+        for line in content.split("\n"):
+            for tpl_name in includes.findall(line):
+                if tpl_name in templates:
+                    continue
+                templates.append(tpl_name)
+                tpl = loader.get_template(tpl_name)
+                with open(tpl.template.origin.name, "r") as fle:
+                    sub_content = fle.read()
+                    keys += attrs.findall(sub_content)
+        return sorted(set(keys))
+
+
 class FullSearch(models.Model):
     search_vector = SearchVectorField(
         _("Search vector"), blank=True, null=True, help_text=_("Auto filled at save")
@@ -3275,16 +3305,20 @@ class SerializeItem:
     SERIALIZATION_FILES = []
     SERIALIZE_STRING = []
 
-    def full_serialize(self, recursion=False) -> dict:
+    def full_serialize(self, search_model=None, recursion=False) -> dict:
         """
         API serialization
         :return: data dict
         """
         full_result = {}
         serialize_fields = []
+
+        exclude = []
+        if search_model:
+            exclude = [sf.key for sf in search_model.sheet_filters.distinct().all()]
         for field in self._meta.get_fields():
             field_name = field.name
-            if field_name in self.SERIALIZE_EXCLUDE:
+            if field_name in self.SERIALIZE_EXCLUDE or field_name in exclude:
                 continue
             if field.many_to_one or field.one_to_one:
                 try:
@@ -3298,7 +3332,7 @@ class SerializeItem:
                         and not recursion
                     ):
                         # print(field.name, self.__class__, self)
-                        value = value.full_serialize(recursion=True)
+                        value = value.full_serialize(search_model, recursion=True)
                     elif field_name in self.SERIALIZATION_FILES:
                         try:
                             value = {"url": value.url}
@@ -3320,7 +3354,7 @@ class SerializeItem:
                     ):
                         # print(field.name, self.__class__, self)
                         values = [
-                            v.full_serialize(recursion=True) for v in values.all()
+                            v.full_serialize(search_model, recursion=True) for v in values.all()
                         ]
                     else:
                         if first_value in self.SERIALIZATION_FILES:
diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py
index e16d37a90..18f6c4c3f 100644
--- a/ishtar_common/models_rest.py
+++ b/ishtar_common/models_rest.py
@@ -17,6 +17,10 @@ try:
 except (AssertionError, ImportError):
     UnoCalc = None
 
+from django.apps import apps
+from django.template import loader
+
+from ishtar_common.models_common import SheetFilter
 from ishtar_common.utils import ugettext_lazy as _
 
 
@@ -38,8 +42,6 @@ MAIN_MODELS = dict(
 )
 
 
-
-
 class ApiUser(models.Model):
     user_ptr = models.OneToOneField(
         User, primary_key=True, related_name="apiuser", on_delete=models.CASCADE
@@ -68,9 +70,13 @@ class ApiSearchModel(models.Model):
         verbose_name = _("API - Remote access - Search model")
         verbose_name_plural = _("API - Remote access - Search models")
 
+    def __str__(self):
+        return f"{self.user} - {self.content_type}"
+
 
-class ApiSheetFilter(models.Model):
-    api_search_model = models.ForeignKey(ApiSearchModel, on_delete=models.CASCADE)
+class ApiSheetFilter(SheetFilter):
+    api_search_model = models.ForeignKey(ApiSearchModel, on_delete=models.CASCADE,
+                                         related_name="sheet_filters")
 
     class Meta:
         verbose_name = _("API - Remote access - Sheet filter")
@@ -78,7 +84,9 @@ class ApiSheetFilter(models.Model):
 
     def get_template(self):
         ct = self.api_search_model.content_type
-        return f"{ct.app_label}/templatestemplates/ishtar/sheet_{ct.model}.html"
+        model = apps.get_model(ct.app_label, ct.model)
+        tpl = loader.get_template(f"ishtar/sheet_{model.SLUG}.html")
+        return tpl.template.origin.name
 
 
 class ApiExternalSource(models.Model):
diff --git a/ishtar_common/rest.py b/ishtar_common/rest.py
index 1ca8824b0..f85061b69 100644
--- a/ishtar_common/rest.py
+++ b/ishtar_common/rest.py
@@ -185,5 +185,5 @@ class GetAPIView(generics.RetrieveAPIView):
         if not q.count():
             return Response({}, content_type="json")
         obj = q.all()[0]
-        result = obj.full_serialize()
+        result = obj.full_serialize(search_model)
         return Response(result, content_type="json")