Manage search with trigrams - 2

3 files changed, 90 insertions, 46 deletions

diff --git a/chimere/migrations/0019_auto_20180927_1659.py b/chimere/migrations/0019_auto_20180928_0848.py
index e64d396..bf5186f 100644
--- a/chimere/migrations/0019_auto_20180927_1659.py
+++ b/chimere/migrations/0019_auto_20180928_0848.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.5 on 2018-09-27 16:59
+# Generated by Django 1.11.5 on 2018-09-28 08:48
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -16,6 +16,21 @@ class Migration(migrations.Migration):
     operations = [
         TrigramExtension(),
         UnaccentExtension(),
+        migrations.AddField(
+            model_name='marker',
+            name='search_for_trigram',
+            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search for trigram'),
+        ),
+        migrations.AddField(
+            model_name='polygon',
+            name='search_for_trigram',
+            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search for trigram'),
+        ),
+        migrations.AddField(
+            model_name='route',
+            name='search_for_trigram',
+            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search for trigram'),
+        ),
         migrations.AlterField(
             model_name='area',
             name='single_click_map',
@@ -26,19 +41,4 @@ class Migration(migrations.Migration):
             name='filtr',
             field=models.TextField(blank=True, help_text=' ', null=True, verbose_name='Filter'),
         ),
-        migrations.AlterField(
-            model_name='marker',
-            name='search_vector',
-            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
-        ),
-        migrations.AlterField(
-            model_name='polygon',
-            name='search_vector',
-            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
-        ),
-        migrations.AlterField(
-            model_name='route',
-            name='search_vector',
-            field=models.TextField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
-        ),
     ]
diff --git a/chimere/models.py b/chimere/models.py
index 7a803dc..a052b66 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -623,8 +623,10 @@ class GeographicItem(models.Model):
         _("Normalised weight"), blank=True, null=True,
         help_text=_("The weight normalised to be between 0 and 1. "
                     "Automatically recalculated."))
-    search_vector = models.TextField(
-        _("Search vector"), blank=True, null=True,
+    search_vector = SearchVectorField(_("Search vector"), blank=True, null=True,
+                                      help_text=_("Auto filled at save"))
+    search_for_trigram = models.TextField(
+        _("Search for trigram"), blank=True, null=True,
         help_text=_("Auto filled at save"))
 
     geom_attr = ''
@@ -1021,28 +1023,35 @@ class GeographicItem(models.Model):
         """
         subcats = SubCategory.getAvailable(instance=True, area=area)
         base_query = cls.objects.filter(categories__in=subcats)
-        search_factors = [('categories__name', 1),
-                          ('categories__category__name', 1),
-                          ('name', 1),
-                          ('keywords', 0.5),
-                          ('categories__keywords', 0.5),
-                          ('search_vector', 0.25)]
-
-        annotate_q = dict(
-            [("similarity_" + key, TrigramSimilarity(key, query))
-             for key, fac in search_factors])
-        for limit in [0.3, 0.1]:
-            filtr, order = None, []
-            for key, factor in search_factors:
-                q = Q(**{"similarity_" + key + "__gt": limit / factor})
-                if not filtr:
-                    filtr = q
-                else:
-                    filtr |= q
-                order.append("similarity_" + key)
-            q = base_query.annotate(**annotate_q).filter(filtr).order_by(*order)
-            if q.count():
-                break
+        if settings.CHIMERE_SEARCH_TECH == 'search_vector':
+            q = cls.objects.filter(categories__in=subcats,
+                                   search_vector=SearchQuery(
+                                       query,
+                                       config=settings.CHIMERE_SEARCH_LANGUAGE))
+        else:
+            search_factors = [('categories__name', 1),
+                              ('categories__category__name', 1),
+                              ('name', 1),
+                              ('keywords', 0.5),
+                              ('categories__keywords', 0.5),
+                              ('search_for_trigram', 0.25)]
+
+            annotate_q = dict(
+                [("similarity_" + key, TrigramSimilarity(key, query))
+                 for key, fac in search_factors])
+            for limit in [0.3, 0.1]:
+                filtr, order = None, []
+                for key, factor in search_factors:
+                    q = Q(**{"similarity_" + key + "__gt": limit / factor})
+                    if not filtr:
+                        filtr = q
+                    else:
+                        filtr |= q
+                    order.append("similarity_" + key)
+                q = base_query.annotate(**annotate_q).filter(filtr).order_by(
+                    *order)
+                if q.count():
+                    break
         if get_json:
             return cls.getGeoJSONs(q, slice=slice, check_next=check_next)
         if slice:
@@ -1061,30 +1070,64 @@ class GeographicItem(models.Model):
         :param save: True if you want to save the object immediately
         :return: True if modified
         """
-        old_search = ""
+        old_search, old_trigram = "", ""
         if self.search_vector:
             old_search = self.search_vector[:]
+        if self.search_for_trigram:
+            old_trigram = self.search_for_trigram[:]
         search_vectors = []
+        search_for_trigram = []
+
         base_q = self.__class__.objects.filter(pk=self.pk)
         cat_keys = ['categories__name', 'categories__keywords',
                     'categories__category__name']
         for cat in self.categories.values('pk').all():
             q = copy.copy(base_q).filter(categories__pk=cat['pk'])
+            q = q.annotate(
+                search=SearchVector(
+                    'categories__name',
+                    'categories__keywords',
+                    'categories__category__name',
+                    config=settings.CHIMERE_SEARCH_LANGUAGE
+                ))
+            search_vectors.append(q.all()[0].search)
+
+            q = copy.copy(base_q).filter(categories__pk=cat['pk'])
             q = q.values(*cat_keys)
             item = q.all()[0]
-            search_vectors += [item[key] for key in cat_keys if item[key]]
+            search_for_trigram += [item[key] for key in cat_keys if item[key]]
         for pm in self.properties.values('pk').all():
             q = copy.copy(base_q).filter(properties__pk=pm['pk'])
+            q = q.annotate(
+                search=SearchVector(
+                    'properties__search_value',
+                    config=settings.CHIMERE_SEARCH_LANGUAGE
+                ))
+            search_vectors.append(q.all()[0].search)
+
+            q = copy.copy(base_q).filter(properties__pk=pm['pk'])
             q = q.values('properties__search_value')
             item = q.all()[0]
             if item['properties__search_value']:
-                search_vectors.append(item['properties__search_value'])
+                search_for_trigram.append(item['properties__search_value'])
+        q = copy.copy(base_q).annotate(
+            search=SearchVector(
+                'name', 'description', 'keywords',
+                config=settings.CHIMERE_SEARCH_LANGUAGE
+            ))
+        search_vectors.append(q.all()[0].search)
+        self.search_vector = utils.merge_tsvectors(search_vectors)
+
         base_keys = ['name', 'description', 'keywords']
         q = base_q.values(*base_keys)
         item = q.all()[0]
-        search_vectors += [item[key] for key in base_keys if item[key]]
-        self.search_vector = " ".join(set(search_vectors))
-        changed = old_search != self.search_vector
+        search_for_trigram += [item[key] for key in base_keys if item[key]]
+        # deduplicate
+        search_for_trigram = " ".join(set(search_for_trigram)).split(" ")
+        self.search_for_trigram = " ".join(set(search_for_trigram))
+
+        changed = (old_search != self.search_vector) or (
+                old_trigram != self.search_for_trigram)
         if save and changed:
             self.save()
         return changed
diff --git a/settings.py b/settings.py
index 878d1cd..ebb64b6 100644
--- a/settings.py
+++ b/settings.py
@@ -136,6 +136,7 @@ CHIMERE_THUMBS_SCALE_WIDTH = None
 
 # search engine
 CHIMERE_SEARCH_LANGUAGE = 'french'
+CHIMERE_SEARCH_TECH = 'search_vector'  # "search_vector" or "trigram"
 HAYSTACK_SEARCH_ENGINE = False
 
 HAYSTACK_CONNECTIONS = {