Syndication - initialize type match with a source

4 files changed, 136 insertions, 4 deletions

diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index 7444aa9cd..39d7de021 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -21,6 +21,7 @@ import csv
 import json
 from io import TextIOWrapper, BytesIO
 import os
+import requests
 import shutil
 import tempfile
 import urllib
@@ -1482,7 +1483,8 @@ serialize_importer_action.short_description = SERIALIZE_DESC
 @admin.register(models.ImporterType, site=admin_site)
 class ImporterTypeAdmin(ImportJSONActionAdmin):
     list_display = ("name", "associated_models", "available")
-    actions = importer_type_actions + [serialize_importer_action,
+    actions = importer_type_actions + [
+        serialize_importer_action,
         change_value("available", True, _("Make available")),
         change_value("available", False, _("Make unavailable")),
     ]
@@ -2110,6 +2112,7 @@ class ApiSearchModelAdminForm(forms.ModelForm):
     class Meta:
         model = models.ApiUser
         exclude = []
+
     content_type = forms.ModelChoiceField(
         label=_("Content type"), queryset=get_main_content_types_query()
     )
@@ -2121,3 +2124,73 @@ class ApiSearchModelAdmin(admin.ModelAdmin):
 
 
 admin_site.register(models.ApiSearchModel, ApiSearchModelAdmin)
+
+
+def send_error_message(request, msg, return_url, message_type=messages.ERROR):
+    messages.add_message(
+        request,
+        message_type,
+        msg,
+    )
+    return HttpResponseRedirect(return_url)
+
+
+def update_types_from_source(modeladmin, request, queryset):
+    return_url = (
+        reverse(
+            "admin:%s_%s_changelist"
+            % (modeladmin.model._meta.app_label, modeladmin.model._meta.model_name)
+        )
+        + "?"
+        + urllib.parse.urlencode(request.GET)
+    )
+    if queryset.count() != 1:
+        return send_error_message(
+            request,
+            str(_("Select only one source.")),
+            return_url,
+            message_type=messages.WARNING,
+        )
+    source = queryset.all()[0]
+    try:
+        response = requests.get(
+            source.url,
+            timeout=20,
+            headers={"Authorization": f"access_token {source.key}"},
+        )
+    except requests.exceptions.Timeout:
+        return send_error_message(
+            request,
+            str(_("Timeout: failed to join {}.")).format(source.url),
+            return_url
+        )
+    if response.status_code != 200:
+        return send_error_message(
+            request,
+            str(
+                _(
+                    "Bad response for {}. Response status code {} != 200. Check your key?"
+                )
+            ).format(source.name, response.status_code),
+            return_url
+        )
+    try:
+        content = response.json()
+    except ValueError:
+        return send_error_message(
+            request,
+            str(_("Response of {} is not a valid JSON message.")).format(source.url),
+            return_url
+        )
+    result = source.update_matches(content)
+    print(result)
+    return response
+
+
+class ApiExternalSource(admin.ModelAdmin):
+    model = models_rest.ApiExternalSource
+    actions = [update_types_from_source]
+    list_display = ("name", "url", "key")
+
+
+admin_site.register(models_rest.ApiExternalSource, ApiExternalSource)
diff --git a/ishtar_common/migrations/0217_auto_20211013_1517.py b/ishtar_common/migrations/0217_auto_20211015_1728.py
index 1429da5ef..47bfa7c6e 100644
--- a/ishtar_common/migrations/0217_auto_20211013_1517.py
+++ b/ishtar_common/migrations/0217_auto_20211015_1728.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2021-10-13 15:17
+# Generated by Django 2.2.24 on 2021-10-15 17:28
 
 from django.conf import settings
 import django.contrib.postgres.fields
@@ -197,6 +197,7 @@ class Migration(migrations.Migration):
                 ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')),
                 ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')),
                 ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')),
+                ('do_not_match', models.BooleanField(default=False, verbose_name='Disable match for this search')),
                 ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Search model')),
                 ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')),
             ],
diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py
index db7d21834..a74df2c8a 100644
--- a/ishtar_common/models_rest.py
+++ b/ishtar_common/models_rest.py
@@ -47,6 +47,58 @@ class ApiExternalSource(models.Model):
         verbose_name = _("API - External source")
         verbose_name_plural = _("API - External sources")
 
+    def update_matches(self, content):
+        result = {
+            "created": 0,
+            "updated": 0,
+            "deleted": 0,
+            "search_model do not exist": [],
+            "type do not exist": [],
+        }
+        updated = []
+        for search_model in content:
+            app, model_name = search_model.split(".")
+            try:
+                ct = ContentType.objects.get(app_label=app, model=model_name)
+            except ContentType.DoesNotExist:
+                result["search_model do not exist"].append(search_model)
+                continue
+            for ct_type, keys, values in content[search_model]:
+                tapp, tmodel_name = ct_type.split(".")
+                try:
+                    ct_type = ContentType.objects.get(
+                        app_label=tapp, model=tmodel_name
+                    )
+                except ContentType.DoesNotExist:
+                    result["type do not exist"].append(search_model)
+                    continue
+                t_model = ct_type.model_class()
+                for slug, label in values:
+                    m, created = ApiKeyMatch.objects.get_or_create(
+                        source=self, search_model=ct, search_keys=keys,
+                        distant_slug=slug, defaults={"distant_label": label})
+                    updated = False
+                    if not created and m.distant_label != label:
+                        updated = True
+                        m.distant_label = label
+                    if not m.do_not_match and not m.local_slug:
+                        slug_key = "txt_idx"
+                        if hasattr(t_model, "slug"):
+                            slug_key = "slug"
+                        q = t_model.objects.filter(**{slug_key: m.distant_slug})
+                        if q.count():
+                            local_value = q.all()[0]
+                            setattr(m, "local_slug", getattr(local_value, slug_key))
+                            m.local_value = str(local_value)
+                            updated = True
+                    if updated:
+                        m.save()
+                        if not created:
+                            result["updated"] += 1
+                    if created:
+                        result["created"] += 1
+        return result
+
 
 class ApiKeyMatch(models.Model):
     source = models.ForeignKey(ApiExternalSource, on_delete=models.CASCADE)
@@ -62,6 +114,8 @@ class ApiKeyMatch(models.Model):
                                   allow_unicode=True)
     local_label = models.TextField(verbose_name=_("Local value"), blank=True,
                                    default="")
+    do_not_match = models.BooleanField(
+        verbose_name=_("Disable match for this search"), default=False)
 
     class Meta:
         verbose_name = _("API - Key match")
diff --git a/ishtar_common/rest.py b/ishtar_common/rest.py
index 602bba293..0ae951dd1 100644
--- a/ishtar_common/rest.py
+++ b/ishtar_common/rest.py
@@ -1,3 +1,5 @@
+import json
+
 from django.conf import settings
 from django.db.models import Q
 from django.utils.translation import activate, deactivate
@@ -94,9 +96,11 @@ class FacetAPIView(APIView):
                     search_keys.append(str(search_key).lower())
                     deactivate()
                 values[ct_model].append(
-                    [search_keys, values_ct]
+                    [f"{type.model._meta.app_label}.{type.model._meta.model_name}",
+                     search_keys, values_ct]
                 )
-        return Response(values)
+        #values = json.dumps(values)
+        return Response(values, content_type="json")
 
     def _get_base_search_model_queries(self):
         """