Map: performance optimization to manage large dataset

5 files changed, 204 insertions, 218 deletions

diff --git a/ishtar_common/static/js/ishtar-map.js b/ishtar_common/static/js/ishtar-map.js
index 1d84adb49..1488d8fcd 100644
--- a/ishtar_common/static/js/ishtar-map.js
+++ b/ishtar_common/static/js/ishtar-map.js
@@ -36,13 +36,8 @@ var get_layers = function(layers){
     return ol_layers;
 };
 
-/* get markers */
-
-var get_markers = function(points){
-};
-
 /* styles */
-var get_style = function(feature){
+var get_icon_style = function(feature){
     return new ol.style.Style({
         image: new ol.style.Icon({
             anchor: [17, 50],
@@ -54,98 +49,55 @@ var get_style = function(feature){
     });
 };
 
+var cluster_get_style = function(feature, resolution){
+    feature.set('key', 'cluster');
+    var cluster_features = feature.get('features');
+
+    var size = cluster_features.length;
+    feature.set('size', size);
+
+    var style = _styleCache[size];
+
+    if (!style && size == 1){
+        style = _styleCache[size] = [get_icon_style()];
+    } else if (!style && size > 1){
+        var color = size > 25 ? "192,0,0" : size > 8 ? "255,128,0" : "0,128,0";
+        var radius = Math.max(8, Math.min(size * 0.75, 20));
+        var lbl = size.toString();
+        style = _styleCache[size] = [
+            new ol.style.Style({
+                image: new ol.style.Circle({
+                    radius: radius,
+                    stroke: new ol.style.Stroke({
+                        color:"rgba("+color+",0.5)",
+                        width: 15
+                    }),
+                    fill: new ol.style.Fill({
+                        color:"rgba("+color+",1)"
+                    })
+                }),
+                text: new ol.style.Text({
+                    text: lbl,
+                    fill: new ol.style.Fill({
+                        color: '#fff'
+                    })
+                })
+            })
+        ];
+    }
+    return style;
+}
+
 /* clustering */
 
-var invisible_style_icon;
 var _styleCache;
-var _remindOldStyle;
-var _remindUpdated;
-var _currentRemind;
-var _revision = 1;
 var cluster_source;
 var cluster_layer;
 
 var enable_clustering = function(){
-
-    if (!invisible_style_icon){
-        invisible_style_icon = new ol.style.Style({
-            image: new ol.style.Icon({
-                //still need something even if it's invisible
-                src : static_path + marker_cluster,
-                opacity : 0
-            })
-        });
-    }
-
-    // Style clusters and hide items inside clusters
+    // cache for styles
     _styleCache = {};
-    _remindOldStyle = {};
-    _remindUpdated = {};
-    _currentRemind = -1;
-
-    function cluster_get_style (feature, resolution){
-        feature.set('key', 'cluster');
-        var cluster_features = feature.get('features');
-
-        var size = cluster_features.length;
-        feature.set('size', size);
-
-        var style = _styleCache[size];
-
-        // no cluster for lonely marker
-        if (!style && size > 1){
-            var color = size > 25 ? "192,0,0" : size > 8 ? "255,128,0" : "0,128,0";
-            var radius = Math.max(8, Math.min(size * 0.75, 20));
-            var lbl = size.toString();
-            style = _styleCache[size] = [
-                new ol.style.Style({
-                    image: new ol.style.Circle({
-                        radius: radius,
-                        stroke: new ol.style.Stroke({
-                            color:"rgba("+color+",0.5)",
-                            width: 15
-                        }),
-                        fill: new ol.style.Fill({
-                            color:"rgba("+color+",1)"
-                        })
-                    }),
-                    text: new ol.style.Text({
-                        text: lbl,
-                        fill: new ol.style.Fill({
-                            color: '#fff'
-                        })
-                    })
-                })
-            ];
-        }
-        // don't reapply the style when no modif have been opered
-        if (_currentRemind != _revision){
-            _remindUpdated = [];
-            _currentRemind = _revision;
-        }
-        if (size > 1){
-            // marker himself disappear
-            for (idx in cluster_features){
-                var feat = cluster_features[idx];
-                if (!_remindUpdated[feat.getProperties()['id']]){
-                    if (!_remindOldStyle[feat.getProperties()['id']]){
-                        _remindOldStyle[feat.getProperties()['id']] = feat.getStyle();
-                    }
-                    feat.setStyle(invisible_style_icon);
-                    _remindUpdated[feat.getProperties()['id']] = 1;
-                }
-            }
-        } else {
-            // or re-appear
-            var feat = cluster_features[0];
-            if (!_remindUpdated[feat.getProperties()['id']] &&
-                  _remindOldStyle[feat.getProperties()['id']]){
-                feat.setStyle(_remindOldStyle[feat.getProperties()['id']]);
-                _remindUpdated[feat.getProperties()['id']] = 1;
-            }
-        }
-        return style;
-    }
+
 
     // Cluster Source
     cluster_source = new ol.source.Cluster({
@@ -165,9 +117,6 @@ var enable_clustering = function(){
 var reinit_clustering = function(){
     cluster_source.getSource().clear();
     _styleCache = {};
-    _remindOldStyle = {};
-    _remindUpdated = {};
-    _currentRemind = -1;
 };
 
 /* manage clicks */
@@ -218,8 +167,6 @@ var click_on_feature = function(feature){
         var key = feature.get('key');
         if (key && key.length > 6 && key.substring(0, 7) == 'cluster'){
             feature = click_on_cluster(feature);
-        } else {
-            feature = click_on_pointer(feature);
         }
     }, timeout);
 };
@@ -230,10 +177,14 @@ var click_on_cluster = function(feature, zoom_level, duration, nb_zoom,
         // zoom animation must be slower
         duration = animation_duration * 2;
     }
+
     if (!nb_zoom) nb_zoom = 0;
 
     var props = feature.getProperties();
     if (!'features' in props) return feature;
+
+    if (props['features'].length == 1) return display_cluster_detail(feature);
+
     if (!current_nb_items){
         current_nb_items = props['features'].length;
     } else if(current_nb_items != props['features'].length) {
@@ -278,9 +229,7 @@ var click_on_cluster = function(feature, zoom_level, duration, nb_zoom,
             );
             if (new_feature){
                 if (zoom_level < min_auto_zoom_on_cluster){
-                    return display_cluster_detail(
-                        new_feature, zoom_level + 1, duration, nb_zoom,
-                        current_nb_items);
+                    return display_cluster_detail(new_feature);
                 }
                 return click_on_cluster(
                     new_feature, zoom_level + 1, duration, nb_zoom,
@@ -291,18 +240,17 @@ var click_on_cluster = function(feature, zoom_level, duration, nb_zoom,
         }, duration + 200);
 };
 
-var click_on_pointer = function(feature){
-    console.log("click_on_pointer");
-    var features = new Array();
-    features.push(feature)
-    display_items(features, -14, -54);
-};
-
 /* display info */
 
 var display_cluster_detail = function(cluster){
-    console.log("display_cluster_detail");
-    display_items(cluster.getProperties()['features'], -14, -21);
+    // console.log("display_cluster_detail");
+    var features = cluster.getProperties()['features']
+    var offset_x = -14;
+    var offset_y = -21;
+    if (features.length == 1){
+        offset_y = -54;
+    }
+    display_items(features, offset_x, offset_y);
 };
 
 var display_items = function(features, offset_x, offset_y){
@@ -310,22 +258,26 @@ var display_items = function(features, offset_x, offset_y){
 };
 
 var _display_items = function(features, offset_x, offset_y){
-    console.log("display_items");
+    // console.log("display_items");
     var feature = features[0];
     var geom = feature.getGeometry();
     var popup_content = "<ul>";
-    features.forEach(function(feat){
+    var has_extra = false;
+    for (idx_feat in features){
+        if (idx_feat >= 5){
+            has_extra = true;
+            popup_content += "<li><a href='#'>...</a></li>"
+            break;
+        }
+        var feat = features[idx_feat];
         var properties = feat.getProperties();
         popup_content += "<li>" + properties['link'] + " " + properties['name'] + "</li>"
-        console.log(properties);
-    });
+    }
     popup_content += "</ul>";
     $(popup_item).html(popup_content);
     popup.setPosition([0, 0]);  // out of the map
     $(popup_item).show(function(){
-
         offset_x -= $(popup_item).width() / 2;
-        console.log(offset_x);
         popup.setOffset([offset_x, offset_y]);
         popup.setPosition(geom.getCoordinates());
     });
@@ -366,8 +318,7 @@ var init_popup = function(){
 
 
 /* display map */
-var vector_source;
-var vector_layer;
+
 var center;
 var point_features;
 var map;
@@ -380,22 +331,10 @@ var proj_options = {
 var geojson_format = new ol.format.GeoJSON(proj_options);
 var wkt_format = new ol.format.WKT(proj_options);
 
-var initialize_map = function(layers){
+var initialize_base_map = function(layers){
     center = wkt_format.readGeometry(map_default_center).getCoordinates();
 
-    /*
-    vector_source = new ol.source.Vector({
-        features: geojson_format.readFeatures(points)
-    });
-    */
-    vector_source = new ol.source.Vector();
-    vector_layer = new ol.layer.Vector({
-        source: vector_source,
-        style: get_style
-    });
-
     map_layers = get_layers(layers);
-    map_layers.push(vector_layer);
 
     map_view = new ol.View({
         projection: view_projection,
@@ -414,7 +353,7 @@ var initialize_map = function(layers){
 var redraw_map = function(layers){
     map.setTarget(null);
     map = null;
-    initialize_map(layers);
+    initialize_base_map(layers);
     reinit_clustering();
     current_feature = null;
 };
@@ -425,25 +364,21 @@ var display_map = function(current_map_id, points, layers){
     if (map){
         redraw_map(layers);
     } else {
-        initialize_map(layers);
+        initialize_base_map(layers);
     }
     point_features = geojson_format.readFeatures(points);
-    vector_source.clear();
-    vector_source.addFeatures(point_features);
 
-    map.updateSize();
+    enable_clustering();
+    cluster_source.getSource().addFeatures(point_features);
+    init_popup();
+
     if (points.features.length){
-        map_view.fit(vector_source.getExtent());
+        map_view.fit(cluster_source.getSource().getExtent());
         if (map_view.getZoom() > 12){
             map_view.setZoom(12);
         }
     }
 
-    enable_clustering();
-    cluster_source.getSource().addFeatures(point_features);
-
-    init_popup();
-
     map.on('click', manage_click_on_map);
     map.on('pointermove', manage_hover);
 }
diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js
index 929629397..be3374e54 100644
--- a/ishtar_common/static/js/ishtar.js
+++ b/ishtar_common/static/js/ishtar.js
@@ -660,8 +660,8 @@ function load_url(url, callback){
 }
 
 function open_window(url){
-    var newwindow = window.open(url, '_blank',
-                                'height=400,width=600,scrollbars=yes');
+    var newwindow = window.open(
+        url, '_blank', 'height=400,width=600,scrollbars=yes');
     if (window.focus) {newwindow.focus()}
     return false;
 }
@@ -1010,11 +1010,9 @@ var ajax_post = function(url, data, target, callback, error_callback){
             if (error_callback) error_callback();
         }
     });
-
 };
 
 var qa_action_register = function(url) {
-
     $('#qa-action').on('submit', function(event){
         event.preventDefault();
         var fn = function(){
@@ -1217,7 +1215,7 @@ var render_paginate_button = function(nb, current, label, disabled, extra_class)
     if (disabled) extra_class += ' disabled';
     if (!label) label = nb;
 
-    var html = '<li class="paginate_button page-item' + extra_class +'">';
+    var html = '<li class="paginate_button page-item' + extra_class + '">';
     html += '<a href="#" data-dt-idx="' + nb + '" tabindex="0" class="page-link">';
     html += label + '</a></li>';
     return html;
diff --git a/ishtar_common/static/media/images/default-pointer.png b/ishtar_common/static/media/images/default-pointer.png
index 5990f1518..bcc836ab2 100644
--- a/ishtar_common/static/media/images/default-pointer.png
+++ b/ishtar_common/static/media/images/default-pointer.png
diff --git a/ishtar_common/templates/blocks/DataTables.html b/ishtar_common/templates/blocks/DataTables.html
index f9fba7ba4..3e0faffd0 100644
--- a/ishtar_common/templates/blocks/DataTables.html
+++ b/ishtar_common/templates/blocks/DataTables.html
@@ -79,6 +79,7 @@ gallery_submit_search = function(image_page){
     } else {
         current_image_page = 1;
     }
+    $('.modal-progress').modal('show');
     var data = search_get_query_data(query_vars, "{{name}}");
     var nb_select = jQuery("#id_{{name}}-length_image").val();
     if (!nb_select) nb_select = 10;
@@ -92,16 +93,18 @@ gallery_submit_search = function(image_page){
         $("#id_{{name}}-length_image").change(gallery_submit_search);
         register_image_gallery(gallery_id);
         $('.card[data-toggle="tooltip"]').tooltip();
+        if ($('.modal-progress').length > 0){
+            $('.modal-progress').modal('hide');
+        }
     });
-
-    if ($('.modal-progress').length > 0){
-        $('.modal-progress').modal('hide');
-    }
     return false;
 };
 {% endif %}
 {% if use_map %}
 map_submit_search = function(){
+    var modal_base_text = $('.modal-progress .modal-header').html();
+    $('.modal-progress .modal-header').html("{% trans 'Fetching data...' %}");
+    $('.modal-progress').modal('show');
     var data = search_get_query_data(query_vars, "{{name}}");
     var nb_select = jQuery("#id_{{name}}-length_map").val();
     if (!nb_select) nb_select = 10;
@@ -110,15 +113,17 @@ map_submit_search = function(){
     $.getJSON(url, function(data) {
         var timestamp = Math.floor(Date.now() / 1000);
         var map_id = "map-" + timestamp;
+        $('.modal-progress .modal-header').html("{% trans 'Render map...' %}");
         var result = render_map(data, "{{name}}", nb_select, map_id);
         $("#tab-content-map-{{name}}").html(result["html"]);
         $("#id_{{name}}-length_map").change(map_submit_search);
+        if ($('.modal-progress').length > 0){
+            $('.modal-progress').modal('hide');
+            $('.modal-progress .modal-header').html(modal_base_text);
+        }
         register_map(map_id, result["points"]);
     });
 
-    if ($('.modal-progress').length > 0){
-        $('.modal-progress').modal('hide');
-    }
     return false;
 };
 {% endif %}
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index 275414d0e..4a6337fe7 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -1030,6 +1030,110 @@ def _format_geojson(rows):
     return data
 
 
+def _get_data_from_query(items, query_table_cols, request, extra_request_keys,
+                         do_not_deduplicate=False):
+    for query_keys in query_table_cols:
+        if not isinstance(query_keys, (tuple, list)):
+            query_keys = [query_keys]
+        for query_key in query_keys:
+            if query_key in extra_request_keys:  # translate query term
+                query_key = extra_request_keys[query_key]
+                if isinstance(query_key, (list, tuple)):
+                    # only manage one level for display
+                    query_key = query_key[0]
+            # clean
+            for filtr in ('__icontains', '__contains', '__iexact',
+                          '__exact'):
+                if query_key.endswith(filtr):
+                    query_key = query_key[:len(query_key) - len(filtr)]
+            query_key.replace(".", "__")  # class style to query
+
+    values = ['id'] + query_table_cols
+
+    c_ids, data_list = [], []
+    for item in items.values(*values):
+        # manual deduplicate when distinct is not enough
+        if not do_not_deduplicate and item['id'] in c_ids:
+            continue
+        c_ids.append(item['id'])
+        data = [item['id']]
+        for key in query_table_cols:
+            data.append(item[key])
+        data_list.append(data)
+    return data_list
+
+
+def _get_data_from_query_old(items, query_table_cols, request,
+                             extra_request_keys, do_not_deduplicate=False):
+    c_ids, datas = [], []
+
+    for item in items:
+        # manual deduplicate when distinct is not enough
+        if not do_not_deduplicate and item.pk in c_ids:
+            continue
+        c_ids.append(item.pk)
+        data = [item.pk]
+        for keys in query_table_cols:
+            if type(keys) not in (list, tuple):
+                keys = [keys]
+            my_vals = []
+            for k in keys:
+                if k in extra_request_keys:
+                    k = extra_request_keys[k]
+                    if type(k) in (list, tuple):
+                        k = k[0]
+                for filtr in ('__icontains', '__contains', '__iexact',
+                              '__exact'):
+                    if k.endswith(filtr):
+                        k = k[:len(k) - len(filtr)]
+                vals = [item]
+                # foreign key may be divided by "." or "__"
+                splitted_k = []
+                for ky in k.split('.'):
+                    if '__' in ky:
+                        splitted_k += ky.split('__')
+                    else:
+                        splitted_k.append(ky)
+                for ky in splitted_k:
+                    new_vals = []
+                    for val in vals:
+                        if hasattr(val, 'all'):  # manage related objects
+                            val = list(val.all())
+                            for v in val:
+                                v = getattr(v, ky)
+                                new_vals += _get_values(request, v)
+                        elif val:
+                            try:
+                                val = getattr(val, ky)
+                                new_vals += _get_values(request, val)
+                            except (AttributeError, GEOSException):
+                                # must be a query key such as "contains"
+                                pass
+                    vals = new_vals
+                # manage last related objects
+                if vals and hasattr(vals[0], 'all'):
+                    new_vals = []
+                    for val in vals:
+                        new_vals += list(val.all())
+                    vals = new_vals
+                if not my_vals:
+                    my_vals = [_format_val(va) for va in vals]
+                else:
+                    new_vals = []
+                    if not vals:
+                        for idx, my_v in enumerate(my_vals):
+                            new_vals.append(u"{}{}{}".format(
+                                my_v, u' - ', ''))
+                    else:
+                        for idx, v in enumerate(vals):
+                            new_vals.append(u"{}{}{}".format(
+                                vals[idx], u' - ', _format_val(v)))
+                    my_vals = new_vals[:]
+            data.append(u" & ".join(my_vals) or u"")
+        datas.append(data)
+    return datas
+
+
 DEFAULT_ROW_NUMBER = 10
 # length is used by ajax DataTables requests
 EXCLUDED_FIELDS = ['length']
@@ -1210,7 +1314,7 @@ def get_item(model, func_name, default_name, extra_request_keys=None,
         except (ValueError, TypeError):
             row_nb = DEFAULT_ROW_NUMBER
 
-        if data_type == 'jso  # no limit for mapn-map':  # no limit for map
+        if data_type == 'json-map':  # no limit for map
             row_nb = None
 
         dct_request_items = {}
@@ -1524,74 +1628,18 @@ def get_item(model, func_name, default_name, extra_request_keys=None,
         else:
             items = items[start:end]
 
-        datas = []
         if old:
             items = [item.get_previous(old) for item in items]
-        c_ids = []
-        for item in items:
-            # manual deduplicate when distinct is not enough
-            if not do_not_deduplicate and item.pk in c_ids:
-                continue
-            c_ids.append(item.pk)
-            data = [item.pk]
-            for keys in query_table_cols:
-                if type(keys) not in (list, tuple):
-                    keys = [keys]
-                my_vals = []
-                for k in keys:
-                    if k in my_extra_request_keys:
-                        k = my_extra_request_keys[k]
-                        if type(k) in (list, tuple):
-                            k = k[0]
-                    for filtr in ('__icontains', '__contains', '__iexact',
-                                  '__exact'):
-                        if k.endswith(filtr):
-                            k = k[:len(k) - len(filtr)]
-                    vals = [item]
-                    # foreign key may be divided by "." or "__"
-                    splitted_k = []
-                    for ky in k.split('.'):
-                        if '__' in ky:
-                            splitted_k += ky.split('__')
-                        else:
-                            splitted_k.append(ky)
-                    for ky in splitted_k:
-                        new_vals = []
-                        for val in vals:
-                            if hasattr(val, 'all'):  # manage related objects
-                                val = list(val.all())
-                                for v in val:
-                                    v = getattr(v, ky)
-                                    new_vals += _get_values(request, v)
-                            elif val:
-                                try:
-                                    val = getattr(val, ky)
-                                    new_vals += _get_values(request, val)
-                                except (AttributeError, GEOSException):
-                                    # must be a query key such as "contains"
-                                    pass
-                        vals = new_vals
-                    # manage last related objects
-                    if vals and hasattr(vals[0], 'all'):
-                        new_vals = []
-                        for val in vals:
-                            new_vals += list(val.all())
-                        vals = new_vals
-                    if not my_vals:
-                        my_vals = [_format_val(va) for va in vals]
-                    else:
-                        new_vals = []
-                        if not vals:
-                            for idx, my_v in enumerate(my_vals):
-                                new_vals.append(u"{}{}{}".format(
-                                    my_v, u' - ', ''))
-                        else:
-                            for idx, v in enumerate(vals):
-                                new_vals.append(u"{}{}{}".format(
-                                    vals[idx], u' - ', _format_val(v)))
-                        my_vals = new_vals[:]
-                data.append(u" & ".join(my_vals) or u"")
-            datas.append(data)
+
+        if data_type == 'json-map':
+            datas = _get_data_from_query(
+                items, query_table_cols, request, my_extra_request_keys,
+                do_not_deduplicate)
+        else:
+            datas = _get_data_from_query_old(
+                items, query_table_cols, request, my_extra_request_keys,
+                do_not_deduplicate)
+
         if manual_sort_key:
             # +1 because the id is added as a first col
             idx_col = None