Syndication - search limitation; API to send available types

7 files changed, 233 insertions, 45 deletions

diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index cd1f6ac6d..d13a21011 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -4452,15 +4452,17 @@ class ApiTest(OperationInitTest, APITestCase):
         self.auth_token = "Token " + Token.objects.create(user=self.user).key
         self.api_user = ApiUser.objects.create(user_ptr=self.user, ip="127.0.0.1")
 
-    def create_api_search_model(self):
+    def create_api_search_model(self, app_label="archaeological_operations",
+                                model="operation"):
         return ApiSearchModel.objects.create(
             user=self.api_user,
             content_type=ContentType.objects.get(
-                app_label="archaeological_operations",
-                model="operation"
-            ))
+                app_label=app_label, model=model
+            ),
+        )
 
     def test_api_permissions(self):
+        # POV: external
         url = reverse("api-search-operation")
         response = self.client.get(url, format="json")
         # nothing OK
@@ -4473,9 +4475,8 @@ class ApiTest(OperationInitTest, APITestCase):
         api_search_model = self.create_api_search_model()
         content_type_id = api_search_model.content_type.id
         api_search_model.content_type = ContentType.objects.get(
-                app_label="archaeological_operations",
-                model="archaeologicalsite"
-            )
+            app_label="archaeological_operations", model="archaeologicalsite"
+        )
         api_search_model.save()
         # token + IP + bad model
         response = self.client.get(
@@ -4500,48 +4501,85 @@ class ApiTest(OperationInitTest, APITestCase):
         self.api_user.save()
 
     def test_api_search(self):
+        # POV: external
         api_search_model = self.create_api_search_model()
         url = reverse("api-search-operation")
-        data = {
-            "submited": 1,
-            "search_vector": "28"
-        }
+        data = {"submited": 1, "search_vector": "28"}
         response = self.client.get(
-            url, format="json", HTTP_AUTHORIZATION=self.auth_token,
-            data=data
+            url, format="json", HTTP_AUTHORIZATION=self.auth_token, data=data
         )
         self.assertEqual(response.status_code, 200)
         j = json.loads(response.content.decode())
-        self.assertEqual(j['recordsTotal'], 2)
+        self.assertEqual(j["recordsTotal"], 2)
         # test default filter
         search_code_q = str(pgettext("key for text search", "patriarche"))
         api_search_model.limit_query = '{}="28124"'.format(search_code_q)
         api_search_model.save()
         response = self.client.get(
+            url, format="json", HTTP_AUTHORIZATION=self.auth_token, data=data
+        )
+        self.assertEqual(response.status_code, 200)
+        j = json.loads(response.content.decode())
+        self.assertEqual(j["recordsTotal"], 1, "api search limitation not effective")
+
+    def test_type_match_api(self):
+        # POV: external
+        # return export of tables
+        random_operation_type = models.OperationType.objects.all()[0]
+        random_operation_type.available = False
+        random_operation_type.save()
+        self.create_api_search_model()
+        url = reverse("api-facets-operation")
+        response = self.client.get(
+            url, format="json", HTTP_AUTHORIZATION=self.auth_token,
+        )
+        self.assertEqual(response.status_code, 200)
+        j = json.loads(response.content.decode())
+        self.assertIn('archaeological_operations.operation', j)
+        # no permissions for archaeological_site
+        self.assertNotIn('archaeological_operations.archaeological_site', j)
+
+        has_type = False
+        for content in j['archaeological_operations.operation']:
+            if content[0] == ["type", "type"]:
+                has_type = True
+                lst = content[1]
+                for tpe in models.OperationType.objects.filter(available=True).all():
+                    self.assertIn([tpe.txt_idx, tpe.label], lst)
+                # do not send not available
+                for tpe in models.OperationType.objects.filter(available=False).all():
+                    self.assertNotIn([tpe.txt_idx, tpe.label], lst)
+                break
+        self.assertTrue(has_type)
+
+        self.create_api_search_model(app_label="archaeological_operations",
+                                     model="archaeologicalsite")
+        url = reverse("api-facets-operation")
+        response = self.client.get(
             url, format="json", HTTP_AUTHORIZATION=self.auth_token,
-            data=data
         )
         self.assertEqual(response.status_code, 200)
         j = json.loads(response.content.decode())
-        self.assertEqual(j['recordsTotal'], 1, "api search limitation not effective")
+        self.assertIn('archaeological_operations.archaeologicalsite', j)
 
+    def test_type_admin(self):
+        # POV: local
+        # create and update matches from an external source
+        # delete old matches!
+        pass
 
     def test_query_transformation(self):
+        # POV: local
         # change query terms from a source Ishtar to match distant Ishtar
         pass
 
     def test_external_source_query(self):
+        # POV: local
         # send a query to an external source when activated
-        # test permissions for this query
+        # test permissions for this query (external source only allowed to a subset of users)
         # test timeout
         pass
 
-    def test_type_match_api(self):
-        pass
-
-    def test_type_admin(self):
-        pass
-
     def test_distant_sheet_display(self):
         # test query limitation
         pass
diff --git a/archaeological_operations/urls.py b/archaeological_operations/urls.py
index 4aee195f4..a7b41ff15 100644
--- a/archaeological_operations/urls.py
+++ b/archaeological_operations/urls.py
@@ -361,4 +361,8 @@ urlpatterns = [
         r"api/search/operation/$", views_api.SearchOperationAPI.as_view(),
         name="api-search-operation"
     ),
+    url(
+        r"api/facets/operation/$", views_api.FacetOperationAPIView.as_view(),
+        name="api-facets-operation"
+    ),
 ]
diff --git a/archaeological_operations/views_api.py b/archaeological_operations/views_api.py
index 48127ec4b..aca400ea3 100644
--- a/archaeological_operations/views_api.py
+++ b/archaeological_operations/views_api.py
@@ -1,7 +1,11 @@
-from ishtar_common.rest import SearchAPIView
-from archaeological_operations import models
+from ishtar_common.rest import SearchAPIView, FacetAPIView
+from archaeological_operations import models, forms
 
 
 class SearchOperationAPI(SearchAPIView):
     model = models.Operation
 
+
+class FacetOperationAPIView(FacetAPIView):
+    models = [models.Operation, models.ArchaeologicalSite]
+    select_forms = [forms.OperationSelect, forms.SiteSelect]
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index 0b73f5708..7444aa9cd 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -65,7 +65,7 @@ from django.views.decorators.csrf import csrf_protect
 
 from django import forms
 
-from ishtar_common import models, models_common
+from ishtar_common import models, models_common, models_rest
 from ishtar_common.apps import admin_site
 from ishtar_common.model_merging import merge_model_objects
 from ishtar_common.utils import get_cache, create_slug
@@ -2087,6 +2087,7 @@ class DocumentTemplateAdmin(admin.ModelAdmin):
 
 admin_site.register(models.DocumentTemplate, DocumentTemplateAdmin)
 
+
 class ApiUserAdmin(admin.ModelAdmin):
     list_display = ("user_ptr", "ip")
 
@@ -2095,11 +2096,8 @@ admin_site.register(models.ApiUser, ApiUserAdmin)
 
 
 def get_main_content_types_query():
-    CONTENT_TYPES = (
-        ("archaeological_operations", "operation"),
-    )
     pks = []
-    for app_label, model_name in CONTENT_TYPES:
+    for app_label, model_name in models_rest.MAIN_CONTENT_TYPES:
         try:
             ct = ContentType.objects.get(app_label=app_label, model=model_name)
             pks.append(ct.pk)
diff --git a/ishtar_common/migrations/0217_auto_20211006_1526.py b/ishtar_common/migrations/0217_auto_20211013_1517.py
index 33299b4c3..1429da5ef 100644
--- a/ishtar_common/migrations/0217_auto_20211006_1526.py
+++ b/ishtar_common/migrations/0217_auto_20211013_1517.py
@@ -1,6 +1,7 @@
-# Generated by Django 2.2.24 on 2021-10-06 15:26
+# Generated by Django 2.2.24 on 2021-10-13 15:17
 
 from django.conf import settings
+import django.contrib.postgres.fields
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations, models
 import django.db.models.deletion
@@ -16,14 +17,27 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.CreateModel(
+            name='ApiExternalSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(verbose_name='URL')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('key', models.CharField(max_length=40, verbose_name='Key')),
+            ],
+            options={
+                'verbose_name': 'API - External source',
+                'verbose_name_plural': 'API - External sources',
+            },
+        ),
+        migrations.CreateModel(
             name='ApiUser',
             fields=[
                 ('user_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='apiuser', serialize=False, to=settings.AUTH_USER_MODEL)),
                 ('ip', models.GenericIPAddressField(verbose_name='IP')),
             ],
             options={
-                'verbose_name': 'Api - User',
-                'verbose_name_plural': 'Api - Users',
+                'verbose_name': 'API - User',
+                'verbose_name_plural': 'API - Users',
             },
         ),
         migrations.AlterField(
@@ -170,8 +184,25 @@ class Migration(migrations.Migration):
                 ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiUser')),
             ],
             options={
-                'verbose_name': 'Api - Search model',
-                'verbose_name_plural': 'Api - Search models',
+                'verbose_name': 'API - Search model',
+                'verbose_name_plural': 'API - Search models',
+            },
+        ),
+        migrations.CreateModel(
+            name='ApiKeyMatch',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('search_keys', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, size=None, verbose_name='Search keys')),
+                ('distant_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Distant key')),
+                ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')),
+                ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')),
+                ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')),
+                ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Search model')),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')),
+            ],
+            options={
+                'verbose_name': 'API - Key match',
+                'verbose_name_plural': 'API - Keys matches',
             },
         ),
     ]
diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py
index 7d321ca92..db7d21834 100644
--- a/ishtar_common/models_rest.py
+++ b/ishtar_common/models_rest.py
@@ -1,10 +1,16 @@
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.gis.db import models
+from django.contrib.postgres.fields import ArrayField
 
 from ishtar_common.utils import ugettext_lazy as _
 
 
+MAIN_CONTENT_TYPES = (
+    ("archaeological_operations", "operation"),
+)
+
+
 class ApiUser(models.Model):
     user_ptr = models.OneToOneField(
         User, primary_key=True, related_name="apiuser", on_delete=models.CASCADE
@@ -12,8 +18,8 @@ class ApiUser(models.Model):
     ip = models.GenericIPAddressField(verbose_name=_("IP"))
 
     class Meta:
-        verbose_name = _("Api - User")
-        verbose_name_plural = _("Api - Users")
+        verbose_name = _("API - User")
+        verbose_name_plural = _("API - Users")
 
     def __str__(self):
         return self.user_ptr.username
@@ -28,5 +34,35 @@ class ApiSearchModel(models.Model):
     )
 
     class Meta:
-        verbose_name = _("Api - Search model")
-        verbose_name_plural = _("Api - Search models")
+        verbose_name = _("API - Search model")
+        verbose_name_plural = _("API - Search models")
+
+
+class ApiExternalSource(models.Model):
+    url = models.URLField(verbose_name=_("URL"))
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    key = models.CharField(_("Key"), max_length=40)
+
+    class Meta:
+        verbose_name = _("API - External source")
+        verbose_name_plural = _("API - External sources")
+
+
+class ApiKeyMatch(models.Model):
+    source = models.ForeignKey(ApiExternalSource, on_delete=models.CASCADE)
+    search_model = models.ForeignKey(ContentType, on_delete=models.CASCADE,
+                                     verbose_name=_("Search model"))
+    search_keys = ArrayField(models.CharField(max_length=200),
+                             verbose_name=_("Search keys"), blank=True)
+    distant_slug = models.SlugField(verbose_name=_("Distant key"), max_length=200,
+                                    allow_unicode=True)
+    distant_label = models.TextField(verbose_name=_("Distant value"), blank=True,
+                                     default="")
+    local_slug = models.SlugField(verbose_name=_("Local key"), max_length=200,
+                                  allow_unicode=True)
+    local_label = models.TextField(verbose_name=_("Local value"), blank=True,
+                                   default="")
+
+    class Meta:
+        verbose_name = _("API - Key match")
+        verbose_name_plural = _("API - Keys matches")
diff --git a/ishtar_common/rest.py b/ishtar_common/rest.py
index 8b4419e82..602bba293 100644
--- a/ishtar_common/rest.py
+++ b/ishtar_common/rest.py
@@ -1,8 +1,12 @@
+from django.conf import settings
+from django.db.models import Q
+from django.utils.translation import activate, deactivate
+
 from rest_framework import authentication, permissions
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from ishtar_common.models import ApiSearchModel
+from ishtar_common import models_rest
 from ishtar_common.views_item import get_item
 
 
@@ -10,7 +14,7 @@ class IpModelPermission(permissions.BasePermission):
     def has_permission(self, request, view):
         if not request.user or not getattr(request.user, "apiuser", None):
             return False
-        ip_addr = request.META['REMOTE_ADDR']
+        ip_addr = request.META["REMOTE_ADDR"]
         q = view.search_model_query(request).filter(user__ip=ip_addr)
         return bool(q.count())
 
@@ -25,14 +29,17 @@ class SearchAPIView(APIView):
         super(SearchAPIView, self).__init__(**kwargs)
 
     def search_model_query(self, request):
-        return ApiSearchModel.objects.filter(
+        return models_rest.ApiSearchModel.objects.filter(
             user=request.user.apiuser,
             content_type__app_label=self.model._meta.app_label,
-            content_type__model=self.model._meta.model_name)
+            content_type__model=self.model._meta.model_name,
+        )
 
     def get(self, request, format=None):
         _get_item = get_item(
-            self.model, "get_" + self.model.SLUG, self.model.SLUG,
+            self.model,
+            "get_" + self.model.SLUG,
+            self.model.SLUG,
             # TODO: own_table_cols=get_table_cols_for_ope() - adapt columns
         )
         search_model = self.search_model_query(request).all()[0]
@@ -45,4 +52,74 @@ class SearchAPIView(APIView):
             request.GET._mutable = False
         response = _get_item(request)
         return response
-        #return Response({})
+        # return Response({})
+
+
+class FacetAPIView(APIView):
+    authentication_classes = (authentication.TokenAuthentication,)
+    permission_classes = (permissions.IsAuthenticated, IpModelPermission)
+    # keep order for models and select_forms: first model match first select form, ...
+    models = []
+    select_forms = []
+
+    def __init__(self, **kwargs):
+        assert self.models
+        assert self.select_forms
+        assert len(self.models) == len(self.select_forms)
+        super().__init__(**kwargs)
+
+    def get(self, request, format=None):
+        values = {}
+        base_queries = self._get_base_search_model_queries()
+        for idx, select_form in enumerate(self.select_forms):
+            # only send types matching permissions
+            model, q = base_queries[idx]
+            if (
+                not models_rest.ApiSearchModel.objects.filter(user=request.user.apiuser)
+                .filter(q)
+                .count()
+            ):
+                continue
+            ct_model = f"{model._meta.app_label}.{model._meta.model_name}"
+            values[ct_model] = []
+            for type in select_form.TYPES:
+                values_ct = []
+                for item in type.model.objects.filter(available=True).all():
+                    key = item.slug if hasattr(item, "slug") else item.txt_idx
+                    values_ct.append((key, str(item)))
+                search_key = model.ALT_NAMES[type.key].search_key
+                search_keys = []
+                for language_code, language_lbl in settings.LANGUAGES:
+                    activate(language_code)
+                    search_keys.append(str(search_key).lower())
+                    deactivate()
+                values[ct_model].append(
+                    [search_keys, values_ct]
+                )
+        return Response(values)
+
+    def _get_base_search_model_queries(self):
+        """
+        Get list of (model, query) to match content_types
+        """
+        return [
+            (
+                model,
+                Q(
+                    content_type__app_label=model._meta.app_label,
+                    content_type__model=model._meta.model_name,
+                ),
+            )
+            for model in self.models
+        ]
+
+    def search_model_query(self, request):
+        q = None
+        for __, base_query in self._get_base_search_model_queries():
+            if not q:
+                q = base_query
+            else:
+                q |= base_query
+        return models_rest.ApiSearchModel.objects.filter(
+            user=request.user.apiuser
+        ).filter(q)