Fix unaccented search in search vector (refs #3152)

5 files changed, 39 insertions, 8 deletions

diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 8fe1d8850..4f9f73a43 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -1269,6 +1269,24 @@ class OperationSearchTest(TestCase, OperationInitTest):
         result = json.loads(response.content)
         self.assertEqual(result['recordsTotal'], 2)
 
+    def test_search_vector(self):
+        c = Client()
+        response = c.get(reverse('get-operation'), {'year': '2010'})
+        # no result when no authentication
+        self.assertTrue(not json.loads(response.content))
+        c.login(username=self.username, password=self.password)
+        # test vector search with accents
+        operation = models.Operation.objects.get(pk=self.operations[0].pk)
+        operation.common_name = u"Opération : Château de Fougères"
+        operation.save()
+        response = c.get(reverse('get-operation'), {'search_vector': 'chaTEAU'})
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 1)
+        response = c.get(reverse('get-operation'), {'search_vector': 'château'})
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 1)
+
+
     def create_relations(self):
         rel1 = models.RelationType.objects.create(
             symmetrical=True, label='Include', txt_idx='include')
diff --git a/install/ishtar-install b/install/ishtar-install
index fcca1291f..b97beab90 100755
--- a/install/ishtar-install
+++ b/install/ishtar-install
@@ -352,7 +352,7 @@ EOF
                       python-bs4 libpangocairo-1.0-0 \
                       python-tidylib python-lxml python-imaging python-html5lib \
                       python-psycopg2 python-gdal gettext python-unicodecsv memcached \
-                      python-django-extra-views python-memcache python-dbf python-markdown' )
+                      python-django-extra-views python-memcache python-dbf python-markdown python-unidecode' )
                 echo "-------------------------------------------------------------------------------";
                 cecho y "Installing django-simple-history"
                 echo "";
@@ -405,7 +405,7 @@ EOF
                       python-tidylib python-lxml python-imaging python-html5lib \
                       python-psycopg2 python-gdal gettext python-unicodecsv memcached \
                       python-django-extra-views python-memcache python-dbf python-markdown \
-                      python-reportlab django-ajax-selects python-django-extensions' )
+                      python-reportlab django-ajax-selects python-django-extensions python-unidecode' )
                 echo "-------------------------------------------------------------------------------";
                 cecho y "Installing django-simple-history"
                 echo "";
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 201cf7c21..e33fb73e4 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -34,6 +34,7 @@ import shutil
 from subprocess import Popen, PIPE
 import tempfile
 import time
+from unidecode import unidecode
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
@@ -1093,7 +1094,11 @@ class FullSearch(models.Model):
                     *self.BASE_SEARCH_VECTORS,
                     config=settings.ISHTAR_SEARCH_LANGUAGE
                 )).values('search')
-            search_vectors.append(q.all()[0]['search'])
+            search_vectors.append(
+                unidecode(
+                    q.all()[0]['search'].decode('utf-8')
+                )
+            )
         self.search_vector = merge_tsvectors(search_vectors)
         changed = old_search != self.search_vector
         if save and changed:
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 8db0e9460..bd558a62e 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -17,8 +17,6 @@
 
 # See the file COPYING for details.
 
-from tidylib import tidy_document as tidy
-
 from copy import copy, deepcopy
 import csv
 import datetime
@@ -28,7 +26,9 @@ from markdown import markdown
 import optparse
 import re
 from tempfile import NamedTemporaryFile
+from tidylib import tidy_document as tidy
 import unicodedata
+from unidecode import unidecode
 from weasyprint import HTML, CSS
 from weasyprint.fonts import FontConfiguration
 
@@ -631,6 +631,15 @@ def _get_values(request, val):
     return new_vals
 
 
+def _search_manage_search_vector(dct):
+    if 'search_vector' in dct:
+        dct['search_vector'] = SearchQuery(
+            unidecode(dct['search_vector']),
+            config=settings.ISHTAR_SEARCH_LANGUAGE
+        )
+    return dct
+
+
 DEFAULT_ROW_NUMBER = 10
 # length is used by ajax DataTables requests
 EXCLUDED_FIELDS = ['length']
@@ -899,9 +908,7 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                         reqs |= q
                     and_reqs.append(reqs)
                     break
-        if 'search_vector' in dct:
-            dct['search_vector'] = SearchQuery(
-                dct['search_vector'], config=settings.ISHTAR_SEARCH_LANGUAGE)
+        dct = _search_manage_search_vector(dct)
         query = Q(**dct)
         for k, or_req in or_reqs:
             alt_dct = dct.copy()
diff --git a/requirements.txt b/requirements.txt
index ac68bc0c0..0ae8faba8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,6 +22,7 @@ django-compressor==2.1
 django-formtools==2.0
 
 secretary==0.2.14
+unidecode
 
 -e git+https://github.com/treyhunner/django-simple-history.git@1.8.2#egg=django-simple-history