diff options
Diffstat (limited to 'ishtar_common/models.py')
| -rw-r--r-- | ishtar_common/models.py | 135 |
1 files changed, 92 insertions, 43 deletions
diff --git a/ishtar_common/models.py b/ishtar_common/models.py index f802b434c..959c31a1f 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1294,7 +1294,6 @@ class JsonData(models.Model, CachedGen): return choices - class Imported(models.Model): imports = models.ManyToManyField( Import, blank=True, @@ -1352,6 +1351,23 @@ class DynamicRequest(object): return alt_names +class SearchVectorConfig(object): + def __init__(self, key, language=None, func=None): + self.key = key + if language: + self.language = language + if language == "local": + self.language = settings.ISHTAR_SEARCH_LANGUAGE + else: + self.language = "simple" + self.func = func + + def format(self, value): + if not self.func: + return [value] + return self.func(value) + + class FullSearch(models.Model): search_vector = SearchVectorField(_("Search vector"), blank=True, null=True, help_text=_("Auto filled at save")) @@ -1398,6 +1414,17 @@ class FullSearch(models.Model): deactivate() return query_parameters + def _update_search_field(self, search_vector_conf, search_vectors, data): + for value in search_vector_conf.format(data): + with connection.cursor() as cursor: + cursor.execute("SELECT to_tsvector(%s, %s)", [ + search_vector_conf.language, value]) + row = cursor.fetchone() + search_vectors.append(row[0]) + + def _update_search_number_field(self, search_vectors, val): + search_vectors.append("'{}':1".format(val)) + def update_search_vector(self, save=True, exclude_parent=False): """ Update the search vector @@ -1428,25 +1455,25 @@ class FullSearch(models.Model): base_q = self.__class__.objects.filter(pk=self.pk) # many to many have to be queried one by one otherwise only one is fetch - for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS: - key = M2M_SEARCH_VECTOR.split('__')[0] + for m2m_search_vector in self.M2M_SEARCH_VECTORS: + key = m2m_search_vector.key.split('__')[0] rel_key = getattr(self, key) for item in rel_key.values('pk').all(): - for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE): - query_dct = {key + "__pk": item['pk']} - q = copy.copy(base_q).filter(**query_dct) - q = q.annotate( - search=SearchVector( - M2M_SEARCH_VECTOR, - config=lang) - ).values('search') - search_vectors.append(q.all()[0]['search']) + query_dct = {key + "__pk": item['pk']} + q = copy.copy(base_q).filter(**query_dct) + q = q.annotate( + search=SearchVector( + m2m_search_vector.key, + config=m2m_search_vector.language) + ).values('search') + search_vectors.append(q.all()[0]['search']) # int/float are not well managed by the SearchVector - for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS: - q = base_q.values(INT_SEARCH_VECTOR) - search_vectors.append( - "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR])) + for int_search_vector in self.INT_SEARCH_VECTORS: + q = base_q.values(int_search_vector.key) + for val in int_search_vector.format( + q.all()[0][int_search_vector.key]): + self._update_search_number_field(search_vectors, val) if not exclude_parent: # copy parent vector fields @@ -1472,32 +1499,24 @@ class FullSearch(models.Model): if self.BASE_SEARCH_VECTORS: # query "simple" fields - q = base_q.values(*self.BASE_SEARCH_VECTORS) + q = base_q.values(*[sv.key for sv in self.BASE_SEARCH_VECTORS]) res = q.all()[0] for base_search_vector in self.BASE_SEARCH_VECTORS: - data = res[base_search_vector] + data = res[base_search_vector.key] data = unidecode(unicode(data)) - for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE): - with connection.cursor() as cursor: - cursor.execute("SELECT to_tsvector(%s, %s)", - [lang, data]) - row = cursor.fetchone() - search_vectors.append(row[0]) + self._update_search_field(base_search_vector, + search_vectors, data) if self.PROPERTY_SEARCH_VECTORS: - for attr in self.PROPERTY_SEARCH_VECTORS: - data = getattr(self, attr) + for property_search_vector in self.PROPERTY_SEARCH_VECTORS: + data = getattr(self, property_search_vector.key) if callable(data): data = data() if not data: continue data = unicode(data) - for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE): - with connection.cursor() as cursor: - cursor.execute("SELECT to_tsvector(%s, %s)", - [lang, data]) - row = cursor.fetchone() - search_vectors.append(row[0]) + self._update_search_field(property_search_vector, + search_vectors, data) if hasattr(self, 'data') and self.data: content_type = ContentType.objects.get_for_model(self) @@ -1511,7 +1530,20 @@ class FullSearch(models.Model): no_data = True break data = data[key] - if no_data: + if no_data or not data: + continue + + if json_field.value_type == 'B': + if data is True: + data = json_field.name + else: + continue + elif json_field.value_type in ('I', 'F'): + self._update_search_number_field(search_vectors, data) + continue + elif json_field.value_type == 'D': + # only index year + self._update_search_number_field(search_vectors, data.year) continue for lang in ("simple", settings.ISHTAR_SEARCH_LANGUAGE): with connection.cursor() as cursor: @@ -3430,7 +3462,8 @@ class Organization(Address, Merge, OwnPerms, ValueGetter): # search parameters EXTRA_REQUEST_KEYS = {} - BASE_SEARCH_VECTORS = ['name', 'town'] + BASE_SEARCH_VECTORS = [SearchVectorConfig('name'), + SearchVectorConfig('town')] # alternative names of fields for searches ALT_NAMES = { @@ -3564,8 +3597,13 @@ class Person(Address, Merge, OwnPerms, ValueGetter): 'profiles_list', 'attached_to', 'town') SHOW_URL = 'show-person' MODIFY_URL = 'person_modify' - BASE_SEARCH_VECTORS = ['name', 'surname', 'raw_name', 'town', - 'attached_to__name', 'email'] + BASE_SEARCH_VECTORS = [ + SearchVectorConfig('name'), + SearchVectorConfig('surname'), + SearchVectorConfig('raw_name'), + SearchVectorConfig('town'), + SearchVectorConfig('attached_to__name'), + SearchVectorConfig('email')] # search parameters REVERSED_BOOL_FIELDS = ['ishtaruser__isnull'] @@ -3965,8 +4003,12 @@ class IshtarUser(FullSearch): 'person__email', 'person__person_types_list', 'person__attached_to__name') BASE_SEARCH_VECTORS = [ - 'user_ptr__username', 'person__name', 'person__surname', - 'person__email', 'person__town', 'person__attached_to__name'] + SearchVectorConfig('user_ptr__username'), + SearchVectorConfig('person__name'), + SearchVectorConfig('person__surname'), + SearchVectorConfig('person__email'), + SearchVectorConfig('person__town'), + SearchVectorConfig('person__attached_to__name')] # search parameters EXTRA_REQUEST_KEYS = { @@ -4109,8 +4151,9 @@ class Basket(FullSearch, OwnPerms): TABLE_COLS = ['label', 'user'] - BASE_SEARCH_VECTORS = ['label', 'comment'] - M2M_SEARCH_VECTORS = ['items'] + BASE_SEARCH_VECTORS = [ + SearchVectorConfig('label'), SearchVectorConfig('comment', 'local')] + M2M_SEARCH_VECTORS = [SearchVectorConfig('items')] class Meta: abstract = True @@ -4320,9 +4363,15 @@ class Document(BaseHistorizedItem, OwnPerms, ImageModel): 'authors__cached_label', 'associated_url'] COL_LINK = ['associated_url'] - BASE_SEARCH_VECTORS = ['title', 'source_type__label', 'external_id', - 'reference', 'description', 'comment', - 'additional_information'] + BASE_SEARCH_VECTORS = [ + SearchVectorConfig("title"), + SearchVectorConfig("source_type__label"), + SearchVectorConfig("external_id"), + SearchVectorConfig("reference"), + SearchVectorConfig("description", "local"), + SearchVectorConfig("comment", "local"), + SearchVectorConfig("additional_information", "local"), + ] PARENT_SEARCH_VECTORS = ['authors', ] BOOL_FIELDS = ['duplicate'] |