Syndication - search interface

10 files changed, 189 insertions, 106 deletions

diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index b1f32620c..94306f6b6 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -2126,13 +2126,12 @@ class ApiSearchModelAdmin(admin.ModelAdmin):
 admin_site.register(models.ApiSearchModel, ApiSearchModelAdmin)
 
 
-def send_error_message(request, msg, return_url, message_type=messages.ERROR):
+def send_error_message(request, msg, message_type=messages.ERROR):
     messages.add_message(
         request,
         message_type,
         msg,
     )
-    return HttpResponseRedirect(return_url)
 
 
 def update_types_from_source(modeladmin, request, queryset):
@@ -2145,74 +2144,90 @@ def update_types_from_source(modeladmin, request, queryset):
         + urllib.parse.urlencode(request.GET)
     )
     if queryset.count() != 1:
-        return send_error_message(
+        send_error_message(
             request,
             str(_("Select only one source.")),
-            return_url,
             message_type=messages.WARNING,
         )
+        return HttpResponseRedirect(return_url)
     source = queryset.all()[0]
-    try:
-        response = requests.get(
-            source.url,
-            timeout=20,
-            headers={"Authorization": f"access_token {source.key}"},
-        )
-    except requests.exceptions.Timeout:
-        return send_error_message(
-            request,
-            str(_("Timeout: failed to join {}.")).format(source.url),
-            return_url,
-        )
-    if response.status_code != 200:
-        return send_error_message(
-            request,
-            str(
-                _(
-                    "Bad response for {}. Response status code {} != 200. Check your key?"
-                )
-            ).format(source.name, response.status_code),
-            return_url,
-        )
-    try:
-        content = response.json()
-    except ValueError:
-        return send_error_message(
-            request,
-            str(_("Response of {} is not a valid JSON message.")).format(source.url),
-            return_url,
-        )
-    result = source.update_matches(content)
-    if result.get("created", None):
+    created, updated, deleted = 0, 0, 0
+    missing_models, missing_types = [], []
+    for item_type in ("operation",):
+        curl = source.url
+        if not curl.endswith("/"):
+            curl += "/"
+        curl += f"api/facets/{item_type}/"
+        try:
+            response = requests.get(
+                curl,
+                timeout=20,
+                headers={"Authorization": f"Token {source.key}"},
+            )
+        except requests.exceptions.Timeout:
+            send_error_message(
+                request,
+                str(_("Timeout: failed to join {}.")).format(curl),
+            )
+            continue
+        if response.status_code != 200:
+            send_error_message(
+                request,
+                str(
+                    _(
+                        "Bad response for {} - {}. Response status code {} != 200. Check your key?"
+                    )
+                ).format(source.name, curl, response.status_code),
+            )
+            continue
+        try:
+            content = response.json()
+        except ValueError:
+            send_error_message(
+                request,
+                str(_("Response of {} is not a valid JSON message.")).format(curl),
+                return_url,
+            )
+            continue
+        result = source.update_matches(content)
+        if result.get("created", None):
+            created += result['created']
+        if result.get("updated", None):
+            updated += result['updated']
+        if result.get("search_model do not exist", None):
+            missing_models += result["search_model do not exist"]
+        if result.get("type do not exist", None):
+            missing_types += result["type do not exist"]
+    if created:
         messages.add_message(
             request,
             messages.INFO,
-            str(_(f"{result['created']} matches created")),
+            str(_(f"{created} matches created")),
         )
-    if result.get("updated", None):
+    if updated:
         messages.add_message(
             request,
             messages.INFO,
-            str(_(f"{result['updated']} matches updated")),
+            str(_(f"{updated} matches updated")),
         )
-    if result.get("deleted", None):
+    if deleted:
         messages.add_message(
             request,
             messages.INFO,
             str(_(f"{result['deleted']} matches deleted")),
         )
-    if result.get("search_model do not exist", None):
-        missing_search_models = ", ".join(result["search_model do not exist"])
+    if missing_models:
+        missing_models = ", ".join(missing_models)
         messages.add_message(
             request,
             messages.INFO,
             str(
                 _(
-                    f"Theses search models have not been found: {missing_search_models}. Not the same Ishtar version?"
+                    f"Theses search models have not been found: {missing_models}. Not the same Ishtar version?"
                 )
             ),
         )
-    if result.get("type do not exist", None):
+    if missing_types:
         missing_types = ", ".join(result["type do not exist"])
         messages.add_message(
             request,
@@ -2239,12 +2254,13 @@ def generate_match_document(modeladmin, request, queryset):
         + urllib.parse.urlencode(request.GET)
     )
     if queryset.count() != 1:
-        return send_error_message(
+        send_error_message(
             request,
             str(_("Select only one source.")),
             return_url,
             message_type=messages.WARNING,
         )
+        return HttpResponseRedirect(return_url)
     src_doc = queryset.all()[0].generate_match_document()
     in_memory = BytesIO()
     with open(src_doc, "rb") as fle:
diff --git a/ishtar_common/context_processors.py b/ishtar_common/context_processors.py
index a03f6429b..7e96bddbd 100644
--- a/ishtar_common/context_processors.py
+++ b/ishtar_common/context_processors.py
@@ -54,7 +54,7 @@ def get_base_context(request):
         request.session["EXTERNAL_SOURCES"] = {}
         if q.count():
             for source in q.all():
-                request.session["EXTERNAL_SOURCES"][f"{source.id}-{source.name}"] = [
+                request.session["EXTERNAL_SOURCES"][f"{source.id}||{source.name}"] = [
                     f"{app_label}-{model_name}"
                     for app_label, model_name in ApiKeyMatch.objects.values_list(
                         "search_model__app_label", "search_model__model"
diff --git a/ishtar_common/rest.py b/ishtar_common/rest.py
index 0ae951dd1..e2e8ac581 100644
--- a/ishtar_common/rest.py
+++ b/ishtar_common/rest.py
@@ -42,6 +42,7 @@ class SearchAPIView(APIView):
             self.model,
             "get_" + self.model.SLUG,
             self.model.SLUG,
+            no_permission_check=True
             # TODO: own_table_cols=get_table_cols_for_ope() - adapt columns
         )
         search_model = self.search_model_query(request).all()[0]
diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js
index 0fa058cca..91273b50c 100644
--- a/ishtar_common/static/js/ishtar.js
+++ b/ishtar_common/static/js/ishtar.js
@@ -1149,6 +1149,10 @@ var dt_multi_enable_disable_submit_button = function(e, dt, type, indexes){
     }
 };
 
+var dt_update_badge = function(source_id, number){
+	$("#source_badge_" + source_id).html(number);
+};
+
 var number_with_commas = function(number) {
     return number.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
 }
diff --git a/ishtar_common/templates/blocks/DataTables-content.html b/ishtar_common/templates/blocks/DataTables-content.html
index 814e10c18..9de143cd4 100644
--- a/ishtar_common/templates/blocks/DataTables-content.html
+++ b/ishtar_common/templates/blocks/DataTables-content.html
@@ -1,5 +1,5 @@
 {% load i18n %}
-    <div id="grid_{{name}}_meta_wrapper">
+    <div id="grid_{{name}}_meta_wrapper" class="sources sources-default">
         <table id='grid_{{name}}' class="display" width="100%">
             <thead>
             <tr>
@@ -11,7 +11,7 @@
         </table>
     </div>
 
-    <div id='foot_{{name}}' class="gridfooter row toolbar">
+    <div id='foot_{{name}}' class="sources sources-default gridfooter row toolbar">
         <div class='col col-sm-2'>
             <div class="btn-group btn-group-sm" role="group">
                 <button type='button' class="btn btn-secondary" data-toggle="modal"
@@ -63,3 +63,17 @@
     {% endif %}
 
     <input type="hidden" id="hidden_{{name}}" name="{{name}}"/>
+
+    {% for source_id, source_label, source_url in external_sources %}
+    <div id="grid_{{name}}_{{source_id}}_meta_wrapper" class="sources sources-{{source_id}}">
+        <table id='grid_{{name}}_{{source_id}}' class="display" width="100%">
+            <thead>
+            <tr>
+                <th></th>
+                <th></th>{% for col in col_names %}
+                <th>{{col}}</th>
+                {% endfor %}</tr>
+            </thead>
+        </table>
+    </div>
+    {% endfor %}
diff --git a/ishtar_common/templates/blocks/DataTables-external-sources.html b/ishtar_common/templates/blocks/DataTables-external-sources.html
index ce4d67ada..00cc7cf2c 100644
--- a/ishtar_common/templates/blocks/DataTables-external-sources.html
+++ b/ishtar_common/templates/blocks/DataTables-external-sources.html
@@ -1,14 +1,19 @@
 {% load i18n %}
 
 <div class="d-flex justify-content-center">
-<div class="btn-group btn-group-toggle" data-toggle="buttons">
-    <label class="btn btn-secondary active">
-        <input type="radio" name="_sources" id="default" autocomplete="off" checked> {% trans "Local" %}
-    </label>
-    {% for source_id, source_label in external_sources %}
-    <label class="btn btn-secondary">
-        <input type="radio" name="_sources" id="{{source_id}}" autocomplete="off"> {{source_label}}
-    </label>
-    {% endfor %}
-</div>
+    <div class="btn-group btn-group-toggle" data-toggle="buttons">
+        <label id="source_button_default" class="btn btn-secondary active">
+            <input type="radio" name="_sources" autocomplete="off" checked>
+            {% trans "Local" %}&nbsp;&nbsp;<span class="badge badge-light" id="source_badge_default">-</span>
+        </label>
+        {% for source_id, source_label, source_url in external_sources %}
+        <label id="source_button_{{source_id}}" class="btn btn-secondary">
+            <input type="radio" name="_sources" autocomplete="off">
+            {{source_label}}&nbsp;&nbsp;<span class="badge badge-light" id="source_badge_{{source_id}}">-</span>
+        </label>
+        {% endfor %}
+    </div>
 </div>
+
+<script>
+</script>
\ No newline at end of file
diff --git a/ishtar_common/templates/blocks/DataTables.html b/ishtar_common/templates/blocks/DataTables.html
index 135e2a51e..91e55579e 100644
--- a/ishtar_common/templates/blocks/DataTables.html
+++ b/ishtar_common/templates/blocks/DataTables.html
@@ -107,11 +107,11 @@ datatable_submit_search = function(not_submited){
 
     datatable_{{sname}}.ajax.url(url);
     datatable_{{sname}}.draw();
-{% if external_sources %}{% for source_id, source_label in external_sources %}
+{% if external_sources %}{% for source_id, source_label, source_url in external_sources %}
     if (!not_submited){
-        url = "{{source}}?submited=1&" + data;
+        url = "{{source_url}}?submited=1&" + data;
     } else {
-        url = "{{source}}?" + data;
+        url = "{{source_url}}?" + data;
     }
     datatable_{{sname}}_{{source_id}}.ajax.url(url);
     datatable_{{sname}}_{{source_id}}.draw();
@@ -149,6 +149,24 @@ jQuery(document).ready(function(){
     main_submit_search
   );
 
+  {% if external_sources %}
+  $("#source_button_default").click(
+    function(){
+        $(".sources").hide();
+        $(".sources-default").show();
+    }
+  );
+  {% for source_id, source_label, source_url in external_sources %}
+  $("#source_button_{{source_id}}").click(
+    function(){
+        $(".sources").hide();
+        $(".sources-{{source_id}}").show();
+    }
+  );
+  {% endfor %}
+  $("#source_button_default").click();
+  {% endif %}
+
   var base_source = "{{source}}";
 
   if (default_search_vector){
@@ -165,6 +183,7 @@ jQuery(document).ready(function(){
                 function(){
                     $('.modal-progress').modal('hide');
                 }, 50);
+            dt_update_badge("default", json.recordsTotal);
             return json.rows;
         }
     },
@@ -231,31 +250,37 @@ jQuery(document).ready(function(){
   datatable_{{sname}}.on('deselect', dt_multi_enable_disable_submit_button);
   {% endif %}
 
-{% if external_sources %}{% for source_id, source_label in external_sources %}
-  var base_external_source = "{{source}}";
+{% if external_sources %}{% for source_id, source_label, source_url in external_sources %}
+  var base_external_source = "{{source_url}}";
 
   if (default_search_vector){
-    base_source += "?search_vector=" + default_search_vector + "&submited=1";
+    base_external_source += "?search_vector=" + default_search_vector + "&submited=1";
   }
 
-  datatable_options = {
+  datatable_options_{{source_id}} = {
     "ajax": {
         "url": base_external_source,
         "dataSrc": function (json) {
-            if (!default_search_vector) manage_pinned_search("{{name}}", json);
-            update_submit_args();
             setTimeout(  // 50ms is waited on load so...
                 function(){
                     $('.modal-progress').modal('hide');
                 }, 500);
+            dt_update_badge("{{source_id}}", json.recordsTotal);
             return json.rows;
         }
     },
-    "deferLoading": 0
+    "deferLoading": 0,
+    "dom": 'litp',
+    "columns": [
+        { "data": "id", "visible": false },
+        { "data": "link", "orderable": false },{% for col in extra_cols %}
+        { "data": "{{col}}", "defaultContent": "-",
+          "render": $.fn.dataTable.render.ellipsis( 70, true ) }{% if not forloop.last %},{% endif %}{% endfor %}
+    ]
   };
-  $.extend(datatable_options, datatables_default);
-  if (datatables_i18n) datatable_options['language'] = datatables_i18n;
-  datatable_{{sname}}_{{source_id}} = jQuery("#grid_{{name}}_{{source_id}}").DataTable(datatable_options);
+  $.extend(datatable_options_{{source_id}}, datatables_default);
+  if (datatables_i18n) datatable_options_{{source_id}}['language'] = datatables_i18n;
+  datatable_{{sname}}_{{source_id}} = jQuery("#grid_{{name}}_{{source_id}}").DataTable(datatable_options_{{source_id}});
 
 {% endfor %}{% endif %}
 
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index 2f991c626..9fa0d59f0 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -24,7 +24,7 @@ from django.views.generic import TemplateView
 
 from .menus import Menu
 
-from ishtar_common import views, models
+from ishtar_common import views, models, views_item
 from ishtar_common.utils import check_rights, get_urls_for_model
 
 # be careful: each check_rights must be relevant with ishtar_menu
@@ -314,6 +314,11 @@ urlpatterns += [
         name="get-by-importer",
     ),
     url(
+        r"search-external/(?P<model>[a-z-]+)/(?P<external_source_id>\d+)/",
+        views_item.get_distant_item,
+        name="search-external"
+    ),
+    url(
         r"new-person/(?:(?P<parent_name>[^/]+)/)?(?:(?P<limits>[^/]+)/)?$",
         views.new_person,
         name="new-person",
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 9ce651f7d..0c766e82e 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -1515,6 +1515,7 @@ def get_item(
     model_for_perms=None,
     alt_query_own=None,
     search_form=None,
+    no_permission_check=False,
 ):
     """
     Generic treatment of tables
@@ -1573,14 +1574,17 @@ def get_item(
         if model_for_perms:
             model_to_check = model_for_perms
 
-        if return_query:
+        if no_permission_check:
             allowed, own = True, False
         else:
-            allowed, own = check_model_access_control(
-                request, model_to_check, available_perms
-            )
-        if not allowed:
-            return HttpResponse(EMPTY, content_type="text/plain")
+            if return_query:
+                allowed, own = True, False
+            else:
+                allowed, own = check_model_access_control(
+                    request, model_to_check, available_perms
+                )
+            if not allowed:
+                return HttpResponse(EMPTY, content_type="text/plain")
 
         if force_own:
             own = True
@@ -2345,27 +2349,30 @@ def get_item(
     return func
 
 
-def get_distant_item(model):
-    def func(
-            request,
-            data_type="json",
-            full=False,
-            force_own=False,
-            col_names=None,
-            no_link=False,
-            no_limit=False,
-            return_query=False,
-            **dct
+def get_distant_item(
+        request,
+        model,
+        external_source_id
     ):
-        if not request.GET.get("external_source", None):
-            return HttpResponse("{}", content_type="text/plain")
-        # TODO: check permissions
-        try:
-            src = models_rest.ApiExternalSource.objects.get(
-                pk=request.GET["external_source"])
-        except models_rest.ApiExternalSource.DoesNotExist:
-            return HttpResponse("{}", content_type="text/plain")
-        url = src.url
+    # TODO: check permissions
+    try:
+        src = models_rest.ApiExternalSource.objects.get(
+            pk=external_source_id)
+    except models_rest.ApiExternalSource.DoesNotExist:
+        return HttpResponse("{}", content_type="text/plain")
+    url = src.url
+    url += reverse(f"api-search-{model}")
+    params = {k: v for k, v in dict(request.GET).items() if not k.startswith("columns")}
+    params["submited"] = 1
+    try:
+        response = requests.get(
+            url,
+            params=params,
+            timeout=20,
+            headers={"Authorization": f"Token {src.key}"},
+        )
+    except requests.exceptions.Timeout:
+        return HttpResponse("{}", content_type="text/plain")
+    return HttpResponse(json.dumps(response.json()), content_type="application/json")
 
 
-    return func
diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py
index 2c2d4f6f9..dff2bf2c5 100644
--- a/ishtar_common/wizards.py
+++ b/ishtar_common/wizards.py
@@ -124,13 +124,19 @@ class IshtarWizard(NamedUrlWizardView):
             and "pk" in form.fields
             and self.request.session.get("EXTERNAL_SOURCES", {})
         ):
-            model_name = f"{self.model._meta.app_label}-{self.model.__name__.lower()}"
-            context["external_sources"] = [
-                source.split("-")
-                for source in self.request.session["EXTERNAL_SOURCES"]
-                if model_name in self.request.session["EXTERNAL_SOURCES"][source]
-            ]
-            form.fields["pk"].widget.external_sources = context["external_sources"]
+            base_model_name = self.model.__name__.lower()
+            model_name = f"{self.model._meta.app_label}-{base_model_name}"
+            context["external_sources"] = []
+            for source in self.request.session["EXTERNAL_SOURCES"]:
+                if model_name in self.request.session["EXTERNAL_SOURCES"][source]:
+                    try:
+                        src_id, lbl = source.split("||")
+                    except ValueError:
+                        continue
+                    url = reverse("search-external", args=[base_model_name, src_id])
+                    context["external_sources"].append((src_id, lbl, url))
+
+        form.fields["pk"].widget.external_sources = context["external_sources"]
 
         open_item_id = self.request.GET.get("open_item", None)
         if open_item_id and self.model and getattr(self.model, "SHOW_URL", None):