diff options
Diffstat (limited to 'ishtar_common')
| -rw-r--r-- | ishtar_common/admin.py | 25 | ||||
| -rw-r--r-- | ishtar_common/management/commands/update_search_vectors.py | 24 | ||||
| -rw-r--r-- | ishtar_common/migrations/0015_auto_20171011_1644.py | 36 | ||||
| -rw-r--r-- | ishtar_common/migrations/0016_auto_20171016_1104.py | 30 | ||||
| -rw-r--r-- | ishtar_common/migrations/0017_auto_20171016_1320.py | 29 | ||||
| -rw-r--r-- | ishtar_common/models.py | 114 | ||||
| -rw-r--r-- | ishtar_common/tests.py | 9 | ||||
| -rw-r--r-- | ishtar_common/utils.py | 44 | ||||
| -rw-r--r-- | ishtar_common/views.py | 26 |
9 files changed, 328 insertions, 9 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index cec61a51e..fbd9f4d29 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -20,6 +20,8 @@ import csv from ajax_select import make_ajax_form +from ajax_select.fields import AutoCompleteSelectField, \ + AutoCompleteSelectMultipleField from django.conf import settings from django.contrib import admin @@ -171,12 +173,31 @@ class PersonAdmin(HistorizedObjectAdmin): admin_site.register(models.Person, PersonAdmin) +class AdminRelatedTownForm(forms.ModelForm): + class Meta: + model = models.Town.children.through + exclude = [] + from_town = AutoCompleteSelectField( + 'town', required=True, label=_(u"Parent")) + + class AdminTownForm(forms.ModelForm): class Meta: model = models.Town exclude = [] center = PointField(label=_(u"center"), required=False, widget=OSMWidget) + children = AutoCompleteSelectMultipleField('town', required=False, + label=_(u"Town children")) + + +class TownParentInline(admin.TabularInline): + model = models.Town.children.through + fk_name = 'to_town' + form = AdminRelatedTownForm + verbose_name = _(u"Parent") + verbose_name_plural = _(u"Parents") + extra = 1 class TownAdmin(admin.ModelAdmin): @@ -186,9 +207,11 @@ class TownAdmin(admin.ModelAdmin): list_display += ['numero_insee', 'departement', ] search_fields += ['numero_insee', 'departement__label', ] list_filter = ("departement",) - readonly_fields = ['imports'] + readonly_fields = ['cached_label', 'imports'] model = models.Town form = AdminTownForm + inlines = [TownParentInline] + admin_site.register(models.Town, TownAdmin) diff --git a/ishtar_common/management/commands/update_search_vectors.py b/ishtar_common/management/commands/update_search_vectors.py new file mode 100644 index 000000000..c73a6e88e --- /dev/null +++ b/ishtar_common/management/commands/update_search_vectors.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from django.core.management.base import BaseCommand +import django.apps + + +class Command(BaseCommand): + help = "./manage.py update_search_vectors\n\n"\ + "Update full texte search vectors." + + def handle(self, *args, **options): + for model in django.apps.apps.get_models(): + if hasattr(model, "update_search_vector") and \ + getattr(model, "BASE_SEARCH_VECTORS", None): + self.stdout.write("\n* update {}".format(model)) + total = model.objects.count() + for idx, item in enumerate(model.objects.all()): + sys.stdout.write("\r{}/{} ".format(idx, total)) + sys.stdout.flush() + item.update_search_vector() + self.stdout.write("\n") diff --git a/ishtar_common/migrations/0015_auto_20171011_1644.py b/ishtar_common/migrations/0015_auto_20171011_1644.py new file mode 100644 index 000000000..a9f4499c2 --- /dev/null +++ b/ishtar_common/migrations/0015_auto_20171011_1644.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0014_ishtarsiteprofile_preservation'), + ] + + operations = [ + migrations.AddField( + model_name='historicalorganization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalperson', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='organization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='person', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/ishtar_common/migrations/0016_auto_20171016_1104.py b/ishtar_common/migrations/0016_auto_20171016_1104.py new file mode 100644 index 000000000..1d9209bdd --- /dev/null +++ b/ishtar_common/migrations/0016_auto_20171016_1104.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 11:04 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0015_auto_20171011_1644'), + ] + + operations = [ + migrations.AddField( + model_name='town', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'), + ), + migrations.AddField( + model_name='town', + name='children', + field=models.ManyToManyField(blank=True, related_name='parents', to='ishtar_common.Town', verbose_name='Town children'), + ), + migrations.AddField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='If not filled considered as the older town known.', null=True, verbose_name='Year of creation'), + ), + ] diff --git a/ishtar_common/migrations/0017_auto_20171016_1320.py b/ishtar_common/migrations/0017_auto_20171016_1320.py new file mode 100644 index 000000000..a48b36ce7 --- /dev/null +++ b/ishtar_common/migrations/0017_auto_20171016_1320.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 13:20 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0016_auto_20171016_1104'), + ] + + operations = [ + migrations.AlterField( + model_name='town', + name='numero_insee', + field=models.CharField(max_length=6, verbose_name='Num\xe9ro INSEE'), + ), + migrations.AlterField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='Filling this field is relevant to distinguish old towns to new towns.', null=True, verbose_name='Year of creation'), + ), + migrations.AlterUniqueTogether( + name='town', + unique_together=set([('numero_insee', 'year')]), + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 28a24115b..2904e51cd 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -35,6 +35,7 @@ import tempfile import time from django.conf import settings +from django.contrib.postgres.search import SearchVectorField, SearchVector from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.uploadedfile import SimpleUploadedFile @@ -58,7 +59,7 @@ from simple_history.models import HistoricalRecords as BaseHistoricalRecords from ishtar_common.model_merging import merge_model_objects from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\ - get_all_field_names + get_all_field_names, merge_tsvectors, cached_label_changed from ishtar_common.models_imports import ImporterModel, ImporterType, \ ImporterDefault, ImporterDefaultValues, ImporterColumn, \ @@ -917,9 +918,85 @@ class Imported(models.Model): abstract = True -class BaseHistorizedItem(Imported): +class FullSearch(models.Model): + search_vector = SearchVectorField(_("Search vector"), blank=True, null=True, + help_text=_("Auto filled at save")) + BASE_SEARCH_VECTORS = [] + INT_SEARCH_VECTORS = [] + M2M_SEARCH_VECTORS = [] + PARENT_SEARCH_VECTORS = [] + + class Meta: + abstract = True + + def update_search_vector(self, save=True): + """ + Update the search vector + :param save: True if you want to save the object immediately + :return: True if modified + """ + if not self.BASE_SEARCH_VECTORS and not self.M2M_SEARCH_VECTORS: + logger.warning("No search_vectors defined for {}".format( + self.__class__)) + return + if getattr(self, '_search_updated', None): + return + self._search_updated = True + + old_search = "" + if self.search_vector: + old_search = self.search_vector[:] + search_vectors = [] + base_q = self.__class__.objects.filter(pk=self.pk) + + # many to many have to be queried one by one otherwise only one is fetch + for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS: + key = M2M_SEARCH_VECTOR.split('__')[0] + rel_key = getattr(self, key) + for item in rel_key.values('pk').all(): + query_dct = {key + "__pk": item['pk']} + q = copy.copy(base_q).filter(**query_dct) + q = q.annotate( + search=SearchVector( + M2M_SEARCH_VECTOR, + config=settings.ISHTAR_SEARCH_LANGUAGE) + ).values('search') + search_vectors.append(q.all()[0]['search']) + + # int/float are not well managed by the SearchVector + for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS: + q = base_q.values(INT_SEARCH_VECTOR) + search_vectors.append( + "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR])) + + # copy parent vector fields + for PARENT_SEARCH_VECTOR in self.PARENT_SEARCH_VECTORS: + parent = getattr(self, PARENT_SEARCH_VECTOR) + if hasattr(parent, 'all'): # m2m + for p in parent.all(): + search_vectors.append(p.search_vector) + else: + search_vectors.append(parent.search_vector) + + # query "simple" fields + q = base_q.annotate( + search=SearchVector( + *self.BASE_SEARCH_VECTORS, + config=settings.ISHTAR_SEARCH_LANGUAGE + )).values('search') + search_vectors.append(q.all()[0]['search']) + self.search_vector = merge_tsvectors(search_vectors) + changed = old_search != self.search_vector + if save and changed: + self.skip_history_when_saving = True + self.save() + return changed + + +class BaseHistorizedItem(FullSearch, Imported): """ - Historized item with external ID management + Historized item with external ID management. + All historized items are searcheable """ IS_BASKET = False EXTERNAL_ID_KEY = '' @@ -1187,6 +1264,7 @@ class LightHistorizedItem(BaseHistorizedItem): super(LightHistorizedItem, self).save(*args, **kwargs) return True + PARSE_FORMULA = re.compile("{([^}]*)}") FORMULA_FILTERS = { @@ -1409,6 +1487,7 @@ def get_current_profile(force=False): def cached_site_changed(sender, **kwargs): get_current_profile(force=True) + post_save.connect(cached_site_changed, sender=IshtarSiteProfile) post_delete.connect(cached_site_changed, sender=IshtarSiteProfile) @@ -2490,12 +2569,20 @@ class Town(Imported, models.Model): center = models.PointField(_(u"Localisation"), srid=settings.SRID, blank=True, null=True) if settings.COUNTRY == 'fr': - numero_insee = models.CharField(u"Numéro INSEE", max_length=6, - unique=True) + numero_insee = models.CharField(u"Numéro INSEE", max_length=6) departement = models.ForeignKey( Department, verbose_name=u"Département", null=True, blank=True) canton = models.ForeignKey(Canton, verbose_name=u"Canton", null=True, blank=True) + year = models.IntegerField( + _("Year of creation"), null=True, blank=True, + help_text=_(u"Filling this field is relevant to distinguish old towns " + u"to new towns.")) + children = models.ManyToManyField( + 'Town', verbose_name=_(u"Town children"), blank=True, + related_name='parents') + cached_label = models.CharField(_(u"Cached name"), max_length=500, + null=True, blank=True, db_index=True) objects = models.GeoManager() class Meta: @@ -2503,11 +2590,24 @@ class Town(Imported, models.Model): verbose_name_plural = _(u"Towns") if settings.COUNTRY == 'fr': ordering = ['numero_insee'] + unique_together = (('numero_insee', 'year'),) def __unicode__(self): + if self.cached_label: + return self.cached_label + self.save() + return self.cached_label + + def _generate_cached_label(self): + cached_label = self.name if settings.COUNTRY == "fr": - return u"%s (%s)" % (self.name, self.numero_insee[:2]) - return self.name + cached_label = u"%s - %s" % (self.name, self.numero_insee[:2]) + if self.year: + cached_label += " ({})".format(self.year) + return cached_label + + +post_save.connect(cached_label_changed, sender=Town) class OperationType(GeneralType): diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 349408465..8a951b29a 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -1046,6 +1046,15 @@ class IshtarBasicTest(TestCase): self.assertEqual(response.status_code, 200) self.assertIn('class="sheet"', response.content) + def test_town_cache(self): + models.Town.objects.create(name="Sin City", numero_insee="99999") + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99") + town.year = 2050 + town.save() + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99 (2050)") + class GeomaticTest(TestCase): def test_post_save_point(self): diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index c6a4032f0..5d9e85c60 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -104,9 +104,12 @@ def cached_label_changed(sender, **kwargs): setattr(instance, cached_label, lbl) changed = True if changed: + instance._search_updated = False if hasattr(instance, '_cascade_change') and instance._cascade_change: instance.skip_history_when_saving = True instance.save() + if hasattr(instance, 'update_search_vector'): + instance.update_search_vector() updated = False if hasattr(instance, '_cached_labels_bulk_update'): updated = instance._cached_labels_bulk_update() @@ -117,6 +120,7 @@ def cached_label_changed(sender, **kwargs): item.test_obj = instance.test_obj cached_label_changed(item.__class__, instance=item) + SHORTIFY_STR = ugettext(" (...)") @@ -289,3 +293,43 @@ def get_all_related_objects(model): and f.auto_created and not f.concrete ] + +def merge_tsvectors(vectors): + """ + Parse tsvector to merge them in one string + :param vectors: list of tsvector string + :return: merged tsvector + """ + result_dict = {} + for vector in vectors: + if not vector: + continue + + current_position = 0 + if result_dict: + for key in result_dict: + max_position = max(result_dict[key]) + if max_position > current_position: + current_position = max_position + + for dct_member in vector.split(" "): + splitted = dct_member.split(':') + key = ":".join(splitted[:-1]) + positions = splitted[-1] + key = key[1:-1] # remove quotes + positions = [int(pos) + current_position + for pos in positions.split(',')] + if key in result_dict: + result_dict[key] += positions + else: + result_dict[key] = positions + + # {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"} + result_dict = {k: ",".join([str(val) for val in result_dict[k]]) + for k in result_dict} + # {'lamelie': "1,5", "hagarde": "2", "regarde": "4"} => + # "'lamelie':1,5 'hagarde':2 'regarde':4" + result = " ".join(["'{}':{}".format(k, result_dict[k]) + for k in result_dict]) + + return result diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 997acd7df..b8350c62a 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -41,6 +41,7 @@ from extra_views import ModelFormSetView from django.conf import settings from django.contrib.auth import logout from django.contrib.auth.decorators import login_required +from django.contrib.postgres.search import SearchQuery from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import Q, ImageField @@ -814,6 +815,23 @@ def get_item(model, func_name, default_name, extra_request_keys=[], dct.pop(k) # manage hierarchic conditions for req in dct.copy(): + if req.endswith('town__pk') or req.endswith('towns__pk'): + val = dct.pop(req) + reqs = Q(**{req: val}) + base_req = req[:-2] + '__' + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'parents__pk' + q = Q(**{req: val}) + reqs |= q + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'children__pk' + q = Q(**{req: val}) + reqs |= q + and_reqs.append(reqs) + continue + for k_hr in HIERARCHIC_FIELDS: if type(req) in (list, tuple): val = dct.pop(req) @@ -829,12 +847,15 @@ def get_item(model, func_name, default_name, extra_request_keys=[], val = dct.pop(req) reqs = Q(**{req: val}) req = req[:-2] + '__' - for idx in xrange(HIERARCHIC_LEVELS): + for idx in range(HIERARCHIC_LEVELS): req = req[:-2] + 'parent__pk' q = Q(**{req: val}) reqs |= q and_reqs.append(reqs) break + if 'search_vector' in dct: + dct['search_vector'] = SearchQuery( + dct['search_vector'], config=settings.ISHTAR_SEARCH_LANGUAGE) query = Q(**dct) for k, or_req in or_reqs: alt_dct = dct.copy() @@ -908,6 +929,9 @@ def get_item(model, func_name, default_name, extra_request_keys=[], items = model.objects.filter(query).distinct() # print(items.query) + if 'search_vector' in dct: # for serialization + dct['search_vector'] = dct['search_vector'].value + # table cols if own_table_cols: table_cols = own_table_cols |