Search: manage localized and non localized indexation

8 files changed, 256 insertions, 117 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index 6dad4f600..c88e60249 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -35,7 +35,7 @@ from ishtar_common.models import Document, GeneralType, \
     GeneralRelationType, GeneralRecordRelations, post_delete_record_relation,\
     post_save_cache, ValueGetter, BulkUpdatedItem, ExternalIdManager, \
     RelationItem, Town, get_current_profile, document_attached_changed, \
-    HistoryModel, SearchAltName, GeoItem, QRCodeItem
+    HistoryModel, SearchAltName, GeoItem, QRCodeItem, SearchVectorConfig
 from archaeological_operations.models import Operation, Period, Parcel, \
     ArchaeologicalSite
 
@@ -377,11 +377,20 @@ class ContextRecord(BulkUpdatedItem, BaseHistorizedItem, QRCodeItem, GeoItem,
     }
 
     PARENT_ONLY_SEARCH_VECTORS = ["operation", "archaeological_site", "parcel"]
-    BASE_SEARCH_VECTORS = ["cached_label", "label", "location", "town__name",
-                           "interpretation", "filling", "datings_comment",
-                           "identification__label", "activity__label",
-                           "excavation_technic__label"]
-    M2M_SEARCH_VECTORS = ["datings__period__label"]
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("cached_label"),
+        SearchVectorConfig("label"),
+        SearchVectorConfig("location"),
+        SearchVectorConfig("town__name"),
+        SearchVectorConfig("interpretation", 'local'),
+        SearchVectorConfig("filling", 'local'),
+        SearchVectorConfig("datings_comment", 'local'),
+        SearchVectorConfig("identification__label"),
+        SearchVectorConfig("activity__label"),
+        SearchVectorConfig("excavation_technic__label")]
+    M2M_SEARCH_VECTORS = [
+        SearchVectorConfig("datings__period__label", 'local')
+    ]
     UP_MODEL_QUERY = {
         "operation": (
             pgettext_lazy("key for text search", u"operation"),
diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index 1d60b9c17..a2cac6b15 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -33,7 +33,7 @@ from ishtar_common.models import GeneralType, BaseHistorizedItem, \
     HistoricalRecords, OwnPerms, Person, Organization, Department, Town, \
     Dashboard, DashboardFormItem, ValueGetter, ShortMenuItem, \
     OperationType, get_external_id, post_save_cache, Document, HistoryModel, \
-    SearchAltName
+    SearchAltName, SearchVectorConfig
 
 from archaeological_operations.models import get_values_town_related, \
     ClosedItem
@@ -116,11 +116,18 @@ class File(ClosedItem, BaseHistorizedItem, OwnPerms, ValueGetter,
         'in_charge__pk': 'in_charge__pk',
     }
     BASE_SEARCH_VECTORS = [
-        'name', 'internal_reference', 'file_type__label', 'saisine_type__label',
-        'permit_type__label', 'permit_reference', 'comment', 'research_comment'
+        SearchVectorConfig('name'),
+        SearchVectorConfig('internal_reference'),
+        SearchVectorConfig('file_type__label'),
+        SearchVectorConfig('saisine_type__label'),
+        SearchVectorConfig('permit_type__label'),
+        SearchVectorConfig('permit_reference'),
+        SearchVectorConfig('comment', 'local'),
+        SearchVectorConfig('research_comment', 'local')
     ]
-    INT_SEARCH_VECTORS = ['numeric_reference', 'year']
-    M2M_SEARCH_VECTORS = ['towns__name']
+    INT_SEARCH_VECTORS = [SearchVectorConfig('numeric_reference'),
+                          SearchVectorConfig('year')]
+    M2M_SEARCH_VECTORS = [SearchVectorConfig('towns__name')]
     PARENT_SEARCH_VECTORS = [
         'in_charge', 'general_contractor', 'corporation_general_contractor',
         'responsible_town_planning_service', 'planning_service', 'organization',
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index e93580418..f2897b02d 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -41,7 +41,7 @@ from ishtar_common.models import Document, GeneralType, \
     ValueGetter, get_current_profile, IshtarSiteProfile, PRIVATE_FIELDS, \
     GeoItem, BulkUpdatedItem, ExternalIdManager, QuickAction, \
     MainItem, document_attached_changed, HistoryModel, DynamicRequest, \
-    SearchAltName, QRCodeItem
+    SearchAltName, QRCodeItem, SearchVectorConfig
 
 
 from archaeological_operations.models import AdministrativeAct, Operation
@@ -304,8 +304,14 @@ class BaseFind(BulkUpdatedItem, BaseHistorizedItem, GeoItem, OwnPerms):
     RELATED_POST_PROCESS = ['find']
     CACHED_LABELS = ['cache_short_id', 'cache_complete_id']
     PARENT_SEARCH_VECTORS = ['context_record']
-    BASE_SEARCH_VECTORS = ["label", "description", "comment", "cache_short_id",
-                           "cache_complete_id", "excavation_id"]
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("label"),
+        SearchVectorConfig("description", "local"),
+        SearchVectorConfig("comment", "local"),
+        SearchVectorConfig("cache_short_id"),
+        SearchVectorConfig("cache_complete_id"),
+        SearchVectorConfig("excavation_id"),
+    ]
     objects = ExternalIdManager()
 
     class Meta:
@@ -1280,14 +1286,27 @@ class Find(BulkUpdatedItem, ValueGetter, BaseHistorizedItem, QRCodeItem,
 
     PARENT_SEARCH_VECTORS = ['base_finds']
     BASE_SEARCH_VECTORS = [
-        "cached_label", "label", "description", "container__location__name",
-        "container__reference", "mark", "comment", "dating_comment",
-        "previous_id", "denomination", "museum_id", "decoration",
-        "manufacturing_place"
+        SearchVectorConfig("cached_label"),
+        SearchVectorConfig("label"),
+        SearchVectorConfig("description", "local"),
+        SearchVectorConfig("container__location__name"),
+        SearchVectorConfig("container__reference"),
+        SearchVectorConfig("mark"),
+        SearchVectorConfig("comment", "local"),
+        SearchVectorConfig("dating_comment", "local"),
+        SearchVectorConfig("previous_id"),
+        SearchVectorConfig("denomination"),
+        SearchVectorConfig("museum_id"),
+        SearchVectorConfig("decoration"),
+        SearchVectorConfig("manufacturing_place"),
     ]
     M2M_SEARCH_VECTORS = [
-        "datings__period__label", "object_types__label", "integrities__label",
-        "remarkabilities__label", "material_types__label"]
+        SearchVectorConfig("datings__period__label", "local"),
+        SearchVectorConfig("object_types__label", "local"),
+        SearchVectorConfig("integrities__label"),
+        SearchVectorConfig("remarkabilities__label", "local"),
+        SearchVectorConfig("material_types__label", "local"),
+    ]
 
     QA_EDIT = QuickAction(
         url="find-qa-bulk-update", icon_class="fa fa-pencil",
diff --git a/archaeological_finds/models_treatments.py b/archaeological_finds/models_treatments.py
index e2882df35..3fc1a5afa 100644
--- a/archaeological_finds/models_treatments.py
+++ b/archaeological_finds/models_treatments.py
@@ -37,7 +37,7 @@ from ishtar_common.models import Document, GeneralType, \
     ImageModel, BaseHistorizedItem, OwnPerms, HistoricalRecords, Person, \
     Organization, ValueGetter, post_save_cache, ShortMenuItem, \
     DashboardFormItem, ExternalIdManager, document_attached_changed, \
-    HistoryModel, SearchAltName
+    HistoryModel, SearchAltName, SearchVectorConfig
 from ishtar_common.utils import cached_label_changed, get_current_year, \
     update_data, m2m_historization_changed
 
@@ -123,11 +123,23 @@ class Treatment(DashboardFormItem, ValueGetter, BaseHistorizedItem,
     ]
 
     BASE_SEARCH_VECTORS = [
-        'treatment_types__label', 'treatment_state__label', 'label', 'goal',
-        'external_id', 'comment', 'description', 'other_reference',
+        SearchVectorConfig("treatment_types__label"),
+        SearchVectorConfig("treatment_state__label"),
+        SearchVectorConfig("label"),
+        SearchVectorConfig("goal", "local"),
+        SearchVectorConfig("external_id"),
+        SearchVectorConfig("comment", "local"),
+        SearchVectorConfig("description", "local"),
+        SearchVectorConfig("other_reference"),
+    ]
+    INT_SEARCH_VECTORS = [
+        SearchVectorConfig("year"),
+        SearchVectorConfig("index"),
+    ]
+    M2M_SEARCH_VECTORS = [
+        SearchVectorConfig("downstream__cached_label"),
+        SearchVectorConfig("upstream__cached_label"),
     ]
-    INT_SEARCH_VECTORS = ["year", "index"]
-    M2M_SEARCH_VECTORS = ['downstream__cached_label', 'upstream__cached_label']
     PARENT_SEARCH_VECTORS = ['person', 'organization']
     objects = ExternalIdManager()
 
@@ -863,9 +875,15 @@ class TreatmentFile(DashboardFormItem, ClosedItem, BaseHistorizedItem,
     SHOW_URL = 'show-treatmentfile'
     TABLE_COLS = ['type', 'year', 'index', 'internal_reference', 'name']
     BASE_SEARCH_VECTORS = [
-        'type__label', 'internal_reference', 'name', 'comment'
+        SearchVectorConfig("type__label"),
+        SearchVectorConfig("internal_reference"),
+        SearchVectorConfig("name"),
+        SearchVectorConfig("comment", "local"),
+    ]
+    INT_SEARCH_VECTORS = [
+        SearchVectorConfig("year"),
+        SearchVectorConfig("index"),
     ]
-    INT_SEARCH_VECTORS = ['year', 'index']
     PARENT_SEARCH_VECTORS = ['in_charge', 'applicant', 'applicant_organisation']
 
     EXTRA_REQUEST_KEYS = {
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index aa1e2dcb4..860422980 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -39,7 +39,7 @@ from ishtar_common.models import BaseHistorizedItem, Dashboard, \
     post_delete_record_relation, post_save_cache, RelationItem, \
     ShortMenuItem, SourceType, Town, ValueGetter, get_current_profile, \
     document_attached_changed, HistoryModel, SearchAltName, \
-    GeoItem, QRCodeItem
+    GeoItem, QRCodeItem, SearchVectorConfig
 from ishtar_common.utils import cached_label_changed, \
     force_cached_label_changed, mode, m2m_historization_changed, post_save_geo
 
@@ -117,19 +117,23 @@ class ArchaeologicalSite(BaseHistorizedItem, QRCodeItem, GeoItem, OwnPerms,
     LONG_SLUG = 'archaeologicalsite'
 
     BASE_SEARCH_VECTORS = [
-        "comment",
-        "discovery_area",
-        "locality_cadastral",
-        "locality_ngi",
-        "name",
-        "oceanographic_service_localisation",
-        "reference",
-        "shipwreck_code",
-        "shipwreck_name",
-        "drassm_number",
-        "affmar_number",
+        SearchVectorConfig("comment", 'local'),
+        SearchVectorConfig("discovery_area", 'local'),
+        SearchVectorConfig("locality_cadastral", 'local'),
+        SearchVectorConfig("locality_ngi", 'local'),
+        SearchVectorConfig("name"),
+        SearchVectorConfig("oceanographic_service_localisation"),
+        SearchVectorConfig("reference"),
+        SearchVectorConfig("shipwreck_code"),
+        SearchVectorConfig("shipwreck_name"),
+        SearchVectorConfig("drassm_number"),
+        SearchVectorConfig("affmar_number"),
+    ]
+    M2M_SEARCH_VECTORS = [
+        SearchVectorConfig("periods__label", "local"),
+        SearchVectorConfig("remains__label", "local"),
+        SearchVectorConfig("towns__name"),
     ]
-    M2M_SEARCH_VECTORS = ["periods__label", "remains__label", "towns__name"]
     PARENT_SEARCH_VECTORS = ['operations']
 
     DATED_FIELDS = ['sinking_date']
@@ -517,8 +521,9 @@ class Operation(ClosedItem, BaseHistorizedItem, QRCodeItem, GeoItem, OwnPerms,\
     APP = "archaeological-operations"
     MODEL = "operation"
     SHOW_URL = 'show-operation'
-    TABLE_COLS = ['year', 'towns_label', 'common_name', 'operation_type',
-                  'start_date', 'excavation_end_date', 'remains']
+    TABLE_COLS = ['code_patriarche', 'year', 'towns_label', 'common_name',
+                  'operation_type', 'start_date', 'excavation_end_date',
+                  'remains']
 
     # search parameters
     BOOL_FIELDS = ['end_date__isnull', 'virtual_operation',
@@ -583,28 +588,36 @@ class Operation(ClosedItem, BaseHistorizedItem, QRCodeItem, GeoItem, OwnPerms,\
         'towns_label': _(u"Towns"),
     }
     BASE_SEARCH_VECTORS = [
-        "abstract",
-        "address",
-        "code_patriarche",
-        "comment",
-        "common_name",
-        "in_charge__cached_label",
-        "name_of_the_protagonist",
-        "official_report_number",
-        "old_code",
-        "operation_type__label",
-        "operator_reference",
-        "operator__cached_label",
-        "scientist__cached_label",
-        "scientific_documentation_comment",
-        "seizure_name",
-        "drassm_code",
+        SearchVectorConfig("abstract", "local"),
+        SearchVectorConfig("address", "local"),
+        SearchVectorConfig("code_patriarche"),
+        SearchVectorConfig("comment", "local"),
+        SearchVectorConfig("common_name"),
+        SearchVectorConfig("common_name", "local"),
+        SearchVectorConfig("in_charge__cached_label"),
+        SearchVectorConfig("name_of_the_protagonist"),
+        SearchVectorConfig("official_report_number"),
+        SearchVectorConfig("old_code"),
+        SearchVectorConfig("operation_type__label"),
+        SearchVectorConfig("operator_reference"),
+        SearchVectorConfig("operator__cached_label"),
+        SearchVectorConfig("scientist__cached_label"),
+        SearchVectorConfig("scientific_documentation_comment", "local"),
+        SearchVectorConfig("seizure_name"),
+        SearchVectorConfig("drassm_code"),
     ]
     PROPERTY_SEARCH_VECTORS = [
-        "full_reference", "short_code_patriarche"
+        SearchVectorConfig("full_reference"),
+        SearchVectorConfig("short_code_patriarche"),
+    ]
+    INT_SEARCH_VECTORS = [
+        SearchVectorConfig("year"),
+    ]
+    M2M_SEARCH_VECTORS = [
+        SearchVectorConfig("periods__label", "local"),
+        SearchVectorConfig("remains__label", "local"),
+        SearchVectorConfig("towns__name"),
     ]
-    INT_SEARCH_VECTORS = ["year"]
-    M2M_SEARCH_VECTORS = ["periods__label", "remains__label", "towns__name"]
     PARENT_SEARCH_VECTORS = ["associated_file"]
     PARENT_ONLY_SEARCH_VECTORS = ["archaeological_sites"]
     ASSOCIATED = {
@@ -849,8 +862,6 @@ class Operation(ClosedItem, BaseHistorizedItem, QRCodeItem, GeoItem, OwnPerms,\
     ## fr
     code_patriarche = models.TextField(u"Code PATRIARCHE", null=True,
                                        blank=True, unique=True)
-    TABLE_COLS = ['code_patriarche'] + TABLE_COLS
-    BASE_SEARCH_VECTORS = ['code_patriarche'] + BASE_SEARCH_VECTORS
     # preventive
     fnap_financing = models.FloatField(u"Financement FNAP (%)",
                                        blank=True, null=True)
@@ -1634,9 +1645,15 @@ class AdministrativeAct(BaseHistorizedItem, OwnPerms, ValueGetter):
         'associated_file__cached_label': _(u"Archaeological file"),
         'operation__cached_label': _(u"Operation"),
     }
-    BASE_SEARCH_VECTORS = ['act_type__label', 'act_object',
-                           'towns_label']
-    INT_SEARCH_VECTORS = ["year", "index"]
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("act_type__label"),
+        SearchVectorConfig("act_object", 'local'),
+        SearchVectorConfig("towns_label"),
+    ]
+    INT_SEARCH_VECTORS = [
+        SearchVectorConfig("year"),
+        SearchVectorConfig("index"),
+    ]
     PARENT_SEARCH_VECTORS = ['operator', 'scientist', 'signatory',
                              'associated_file', 'operation', 'treatment_file',
                              'treatment']
@@ -2048,7 +2065,11 @@ class ParcelManager(models.Manager):
 
 class Parcel(LightHistorizedItem):
     EXTERNAL_ID_KEY = 'parcel_external_id'
-    BASE_SEARCH_VECTORS = ['section', 'parcel_number', "cached_label"]
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("section"),
+        SearchVectorConfig("parcel_number"),
+        SearchVectorConfig("cached_label"),
+    ]
     PARENT_SEARCH_VECTORS = ['operation']
     objects = ParcelManager()
 
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 06898014c..b983ff6de 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -1619,6 +1619,10 @@ class OperationSearchTest(TestCase, OperationInitTest):
 
         # simple separation
         response = c.get(reverse('get-operation'),
+                         {'search_vector': 'chaTEAU fougeres'})
+        result = json.loads(response.content)
+        self.assertEqual(result['recordsTotal'], 1)
+        response = c.get(reverse('get-operation'),
                          {'search_vector': 'chaTEAU fougere'})
         result = json.loads(response.content)
         self.assertEqual(result['recordsTotal'], 1)
diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py
index 6e2df520e..cec8667cd 100644
--- a/archaeological_warehouse/models.py
+++ b/archaeological_warehouse/models.py
@@ -32,7 +32,7 @@ from ishtar_common.models import Document, GeneralType, get_external_id, \
     LightHistorizedItem, OwnPerms, Address, Person, post_save_cache, \
     DashboardFormItem, ExternalIdManager, ShortMenuItem, \
     document_attached_changed, SearchAltName, DynamicRequest, GeoItem, \
-    QRCodeItem
+    QRCodeItem, SearchVectorConfig
 from ishtar_common.utils import cached_label_changed, post_save_geo
 
 
@@ -54,8 +54,13 @@ class Warehouse(Address, GeoItem, QRCodeItem, DashboardFormItem, OwnPerms,
     MODEL = "warehouse"
     SHOW_URL = 'show-warehouse'
     TABLE_COLS = ['name', 'warehouse_type']
-    BASE_SEARCH_VECTORS = ['name', 'warehouse_type__label', "external_id",
-                           "town", "comment"]
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("name"),
+        SearchVectorConfig("warehouse_type__label"),
+        SearchVectorConfig("external_id"),
+        SearchVectorConfig("town"),
+        SearchVectorConfig("comment", "local"),
+    ]
 
     EXTRA_REQUEST_KEYS = {
         # used by dynamic_table_documents
@@ -333,10 +338,17 @@ class Container(LightHistorizedItem, QRCodeItem, GeoItem, OwnPerms):
     TABLE_COLS = ['reference', 'container_type__label', 'cached_location',
                   'cached_division', 'old_reference']
     IMAGE_PREFIX = 'containers/'
-    BASE_SEARCH_VECTORS = ['reference', 'container_type__label',
-                           'cached_location', 'old_reference', 'comment']
-    M2M_SEARCH_VECTORS = ['division__reference',
-                          'division__division__division__label']
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("reference"),
+        SearchVectorConfig("container_type__label"),
+        SearchVectorConfig("cached_location"),
+        SearchVectorConfig("old_reference"),
+        SearchVectorConfig("comment", "local"),
+    ]
+    M2M_SEARCH_VECTORS = [
+        SearchVectorConfig("division__reference"),
+        SearchVectorConfig("division__division__division__label"),
+    ]
 
     # search parameters
     EXTRA_REQUEST_KEYS = {
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index f802b434c..959c31a1f 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -1294,7 +1294,6 @@ class JsonData(models.Model, CachedGen):
         return choices
 
 
-
 class Imported(models.Model):
     imports = models.ManyToManyField(
         Import, blank=True,
@@ -1352,6 +1351,23 @@ class DynamicRequest(object):
         return alt_names
 
 
+class SearchVectorConfig(object):
+    def __init__(self, key, language=None, func=None):
+        self.key = key
+        if language:
+            self.language = language
+            if language == "local":
+                self.language = settings.ISHTAR_SEARCH_LANGUAGE
+        else:
+            self.language = "simple"
+        self.func = func
+
+    def format(self, value):
+        if not self.func:
+            return [value]
+        return self.func(value)
+
+
 class FullSearch(models.Model):
     search_vector = SearchVectorField(_("Search vector"), blank=True, null=True,
                                       help_text=_("Auto filled at save"))
@@ -1398,6 +1414,17 @@ class FullSearch(models.Model):
                 deactivate()
         return query_parameters
 
+    def _update_search_field(self, search_vector_conf, search_vectors, data):
+        for value in search_vector_conf.format(data):
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT to_tsvector(%s, %s)", [
+                    search_vector_conf.language, value])
+                row = cursor.fetchone()
+                search_vectors.append(row[0])
+
+    def _update_search_number_field(self, search_vectors, val):
+        search_vectors.append("'{}':1".format(val))
+
     def update_search_vector(self, save=True, exclude_parent=False):
         """
         Update the search vector
@@ -1428,25 +1455,25 @@ class FullSearch(models.Model):
         base_q = self.__class__.objects.filter(pk=self.pk)
 
         # many to many have to be queried one by one otherwise only one is fetch
-        for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS:
-            key = M2M_SEARCH_VECTOR.split('__')[0]
+        for m2m_search_vector in self.M2M_SEARCH_VECTORS:
+            key = m2m_search_vector.key.split('__')[0]
             rel_key = getattr(self, key)
             for item in rel_key.values('pk').all():
-                for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE):
-                    query_dct = {key + "__pk": item['pk']}
-                    q = copy.copy(base_q).filter(**query_dct)
-                    q = q.annotate(
-                        search=SearchVector(
-                            M2M_SEARCH_VECTOR,
-                            config=lang)
-                    ).values('search')
-                    search_vectors.append(q.all()[0]['search'])
+                query_dct = {key + "__pk": item['pk']}
+                q = copy.copy(base_q).filter(**query_dct)
+                q = q.annotate(
+                    search=SearchVector(
+                        m2m_search_vector.key,
+                        config=m2m_search_vector.language)
+                ).values('search')
+                search_vectors.append(q.all()[0]['search'])
 
         # int/float are not well managed by the SearchVector
-        for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS:
-            q = base_q.values(INT_SEARCH_VECTOR)
-            search_vectors.append(
-                "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR]))
+        for int_search_vector in self.INT_SEARCH_VECTORS:
+            q = base_q.values(int_search_vector.key)
+            for val in int_search_vector.format(
+                    q.all()[0][int_search_vector.key]):
+                self._update_search_number_field(search_vectors, val)
 
         if not exclude_parent:
             # copy parent vector fields
@@ -1472,32 +1499,24 @@ class FullSearch(models.Model):
 
         if self.BASE_SEARCH_VECTORS:
             # query "simple" fields
-            q = base_q.values(*self.BASE_SEARCH_VECTORS)
+            q = base_q.values(*[sv.key for sv in self.BASE_SEARCH_VECTORS])
             res = q.all()[0]
             for base_search_vector in self.BASE_SEARCH_VECTORS:
-                data = res[base_search_vector]
+                data = res[base_search_vector.key]
                 data = unidecode(unicode(data))
-                for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE):
-                    with connection.cursor() as cursor:
-                        cursor.execute("SELECT to_tsvector(%s, %s)",
-                                       [lang, data])
-                        row = cursor.fetchone()
-                        search_vectors.append(row[0])
+                self._update_search_field(base_search_vector,
+                                          search_vectors, data)
 
         if self.PROPERTY_SEARCH_VECTORS:
-            for attr in self.PROPERTY_SEARCH_VECTORS:
-                data = getattr(self, attr)
+            for property_search_vector in self.PROPERTY_SEARCH_VECTORS:
+                data = getattr(self, property_search_vector.key)
                 if callable(data):
                     data = data()
                 if not data:
                     continue
                 data = unicode(data)
-                for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE):
-                    with connection.cursor() as cursor:
-                        cursor.execute("SELECT to_tsvector(%s, %s)",
-                                       [lang, data])
-                        row = cursor.fetchone()
-                        search_vectors.append(row[0])
+                self._update_search_field(property_search_vector,
+                                          search_vectors, data)
 
         if hasattr(self, 'data') and self.data:
             content_type = ContentType.objects.get_for_model(self)
@@ -1511,7 +1530,20 @@ class FullSearch(models.Model):
                         no_data = True
                         break
                     data = data[key]
-                if no_data:
+                if no_data or not data:
+                    continue
+
+                if json_field.value_type == 'B':
+                    if data is True:
+                        data = json_field.name
+                    else:
+                        continue
+                elif json_field.value_type in ('I', 'F'):
+                    self._update_search_number_field(search_vectors, data)
+                    continue
+                elif json_field.value_type == 'D':
+                    # only index year
+                    self._update_search_number_field(search_vectors, data.year)
                     continue
                 for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE):
                     with connection.cursor() as cursor:
@@ -3430,7 +3462,8 @@ class Organization(Address, Merge, OwnPerms, ValueGetter):
 
     # search parameters
     EXTRA_REQUEST_KEYS = {}
-    BASE_SEARCH_VECTORS = ['name', 'town']
+    BASE_SEARCH_VECTORS = [SearchVectorConfig('name'),
+                           SearchVectorConfig('town')]
 
     # alternative names of fields for searches
     ALT_NAMES = {
@@ -3564,8 +3597,13 @@ class Person(Address, Merge, OwnPerms, ValueGetter):
                           'profiles_list', 'attached_to', 'town')
     SHOW_URL = 'show-person'
     MODIFY_URL = 'person_modify'
-    BASE_SEARCH_VECTORS = ['name', 'surname', 'raw_name', 'town',
-                           'attached_to__name', 'email']
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig('name'),
+        SearchVectorConfig('surname'),
+        SearchVectorConfig('raw_name'),
+        SearchVectorConfig('town'),
+        SearchVectorConfig('attached_to__name'),
+        SearchVectorConfig('email')]
 
     # search parameters
     REVERSED_BOOL_FIELDS = ['ishtaruser__isnull']
@@ -3965,8 +4003,12 @@ class IshtarUser(FullSearch):
                   'person__email', 'person__person_types_list',
                   'person__attached_to__name')
     BASE_SEARCH_VECTORS = [
-        'user_ptr__username', 'person__name', 'person__surname',
-        'person__email', 'person__town', 'person__attached_to__name']
+        SearchVectorConfig('user_ptr__username'),
+        SearchVectorConfig('person__name'),
+        SearchVectorConfig('person__surname'),
+        SearchVectorConfig('person__email'),
+        SearchVectorConfig('person__town'),
+        SearchVectorConfig('person__attached_to__name')]
 
     # search parameters
     EXTRA_REQUEST_KEYS = {
@@ -4109,8 +4151,9 @@ class Basket(FullSearch, OwnPerms):
 
     TABLE_COLS = ['label', 'user']
 
-    BASE_SEARCH_VECTORS = ['label', 'comment']
-    M2M_SEARCH_VECTORS = ['items']
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig('label'), SearchVectorConfig('comment', 'local')]
+    M2M_SEARCH_VECTORS = [SearchVectorConfig('items')]
 
     class Meta:
         abstract = True
@@ -4320,9 +4363,15 @@ class Document(BaseHistorizedItem, OwnPerms, ImageModel):
                   'authors__cached_label',
                   'associated_url']
     COL_LINK = ['associated_url']
-    BASE_SEARCH_VECTORS = ['title', 'source_type__label', 'external_id',
-                           'reference', 'description', 'comment',
-                           'additional_information']
+    BASE_SEARCH_VECTORS = [
+        SearchVectorConfig("title"),
+        SearchVectorConfig("source_type__label"),
+        SearchVectorConfig("external_id"),
+        SearchVectorConfig("reference"),
+        SearchVectorConfig("description", "local"),
+        SearchVectorConfig("comment", "local"),
+        SearchVectorConfig("additional_information", "local"),
+    ]
     PARENT_SEARCH_VECTORS = ['authors', ]
 
     BOOL_FIELDS = ['duplicate']