diff options
32 files changed, 852 insertions, 19 deletions
diff --git a/UPGRADE.md b/UPGRADE.md index 34d1560db..de01ca78b 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -35,3 +35,10 @@ cd <application-path> ./manage.py migrate --fake archaeological_warehouse 0002_auto_20170414_2123 ./manage.py migrate ``` + +Finally create indexes the new full text search engine + +``` +cd <application-path> +./manage.py update_search_vectors +``` diff --git a/archaeological_context_records/forms.py b/archaeological_context_records/forms.py index e5c244fde..c310e98fa 100644 --- a/archaeological_context_records/forms.py +++ b/archaeological_context_records/forms.py @@ -56,6 +56,7 @@ class OperationFormSelection(forms.Form): class RecordSelect(TableSelect): + search_vector = forms.CharField(label=_(u"Full text search")) label = forms.CharField(label=_(u"ID"), max_length=100) parcel__town = get_town_field() if settings.COUNTRY == 'fr': diff --git a/archaeological_context_records/migrations/0010_auto_20171011_1644.py b/archaeological_context_records/migrations/0010_auto_20171011_1644.py new file mode 100644 index 000000000..379110e44 --- /dev/null +++ b/archaeological_context_records/migrations/0010_auto_20171011_1644.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_context_records', '0009_auto_20170829_1639'), + ] + + operations = [ + migrations.AddField( + model_name='contextrecord', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalcontextrecord', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/archaeological_context_records/migrations/0011_auto_20171012_1316.py b/archaeological_context_records/migrations/0011_auto_20171012_1316.py new file mode 100644 index 000000000..95b042c43 --- /dev/null +++ b/archaeological_context_records/migrations/0011_auto_20171012_1316.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-12 13:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_context_records', '0010_auto_20171011_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='contextrecord', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='historicalcontextrecord', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + ] diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py index 2f02ed9df..a8517ed26 100644 --- a/archaeological_context_records/models.py +++ b/archaeological_context_records/models.py @@ -302,7 +302,14 @@ class ContextRecord(BulkUpdatedItem, BaseHistorizedItem, point_2d = models.PointField(_(u"Point (2D)"), blank=True, null=True) point = models.PointField(_(u"Point (3D)"), blank=True, null=True, dim=3) polygon = models.PolygonField(_(u"Polygon"), blank=True, null=True) - cached_label = models.TextField(_(u"Cached name"), null=True, blank=True) + cached_label = models.TextField(_(u"Cached name"), null=True, blank=True, + db_index=True) + PARENT_SEARCH_VECTORS = ['operation'] + BASE_SEARCH_VECTORS = ["cached_label", "label", "location", + "interpretation", "filling", "datings_comment", + "identification__label", "activity__label", + "excavation_technic__label"] + M2M_SEARCH_VECTORS = ["datings__period__label"] history = HistoricalRecords() class Meta: diff --git a/archaeological_context_records/tests.py b/archaeological_context_records/tests.py index 89b15fbbf..b0f4b8f9e 100644 --- a/archaeological_context_records/tests.py +++ b/archaeological_context_records/tests.py @@ -273,6 +273,20 @@ class ContextRecordTest(ContextRecordInit, TestCase): cr.operation ) + def test_search_vector_update(self): + cr = self.create_context_record(force=True)[0] + cr = models.ContextRecord.objects.get(pk=cr.pk) + cr.label = "Label label" + cr.location = "I am heeere" + cr.save() + for key in ('label', 'heeer'): + self.assertIn(key, cr.search_vector) + cr.operation.code_patriarche = "PATRIARCHE" + cr.operation.save() + cr = models.ContextRecord.objects.get(pk=cr.pk) + self.assertIn(settings.ISHTAR_OPE_PREFIX.lower() + "patriarch", + cr.search_vector) + def test_upstream_cache_update(self): cr = self.create_context_record()[0] cr_pk = cr.pk @@ -399,6 +413,44 @@ class ContextRecordSearchTest(ContextRecordInit, TestCase): models.RecordRelations.objects.create( left_record=cr_1, right_record=cr_2, relation_type=sym_rel_type) + def test_town_search(self): + c = Client() + c.login(username=self.username, password=self.password) + + data = {'numero_insee': '98989', 'name': 'base_town'} + base_town = self.create_towns(datas=data)[-1] + + parcel = self.create_parcel(data={'town': base_town, + 'section': 'A', 'parcel_number': '1'})[-1] + self.context_records[0].parcel = parcel + self.context_records[0].save() + + data = {'numero_insee': '56789', 'name': 'parent_town'} + parent_town = self.create_towns(datas=data)[-1] + parent_town.children.add(base_town) + + data = {'numero_insee': '01234', 'name': 'child_town'} + child_town = self.create_towns(datas=data)[-1] + base_town.children.add(child_town) + + # simple search + search = {'parcel__town': base_town.pk} + response = c.get(reverse('get-contextrecord'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + + # parent search + search = {'parcel__town': parent_town.pk} + response = c.get(reverse('get-contextrecord'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + + # child search + search = {'parcel__town': child_town.pk} + response = c.get(reverse('get-contextrecord'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + def testSearchExport(self): c = Client() response = c.get(reverse('get-contextrecord')) @@ -416,6 +468,14 @@ class ContextRecordSearchTest(ContextRecordInit, TestCase): {'label': 'cr 1', 'cr_relation_types_0': self.cr_rel_type.pk}) self.assertEqual(json.loads(response.content)['total'], 2) + # test search vector + response = c.get(reverse('get-contextrecord'), + {'search_vector': 'CR'}) + self.assertEqual(json.loads(response.content)['total'], 2) + # the 2 context records have the same operation + response = c.get(reverse('get-contextrecord'), + {'search_vector': 'op2010'}) + self.assertEqual(json.loads(response.content)['total'], 2) # test search between related operations first_ope = self.operations[0] first_ope.year = 2010 diff --git a/archaeological_files/migrations/0008_auto_20171011_1644.py b/archaeological_files/migrations/0008_auto_20171011_1644.py new file mode 100644 index 000000000..33dfbf59e --- /dev/null +++ b/archaeological_files/migrations/0008_auto_20171011_1644.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_files', '0007_auto_20170826_1152'), + ] + + operations = [ + migrations.AddField( + model_name='file', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalfile', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/archaeological_files/migrations/0009_auto_20171012_1316.py b/archaeological_files/migrations/0009_auto_20171012_1316.py new file mode 100644 index 000000000..cd33d8243 --- /dev/null +++ b/archaeological_files/migrations/0009_auto_20171012_1316.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-12 13:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_files', '0008_auto_20171011_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='file', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='historicalfile', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + ] diff --git a/archaeological_files/models.py b/archaeological_files/models.py index 0d5b4b3e8..b0f53f11c 100644 --- a/archaeological_files/models.py +++ b/archaeological_files/models.py @@ -204,7 +204,8 @@ class File(ClosedItem, BaseHistorizedItem, OwnPerms, ValueGetter, mh_listing = models.NullBooleanField( u"Sur Monument Historique inscrit", blank=True, null=True) # <-- research archaeology - cached_label = models.TextField(_(u"Cached name"), null=True, blank=True) + cached_label = models.TextField(_(u"Cached name"), null=True, blank=True, + db_index=True) imported_line = models.TextField(_(u"Imported line"), null=True, blank=True) history = HistoricalRecords() diff --git a/archaeological_finds/forms.py b/archaeological_finds/forms.py index aa0ae4621..1f81cf52f 100644 --- a/archaeological_finds/forms.py +++ b/archaeological_finds/forms.py @@ -366,6 +366,7 @@ DatingFormSet.form_label = _("Dating") class FindSelect(TableSelect): + search_vector = forms.CharField(label=_(u"Full text search")) base_finds__cache_short_id = forms.CharField(label=_(u"Short ID")) base_finds__cache_complete_id = forms.CharField(label=_(u"Complete ID")) label = forms.CharField(label=_(u"Free ID")) diff --git a/archaeological_finds/migrations/0010_auto_20171011_1644.py b/archaeological_finds/migrations/0010_auto_20171011_1644.py new file mode 100644 index 000000000..ce892e96d --- /dev/null +++ b/archaeological_finds/migrations/0010_auto_20171011_1644.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_finds', '0009_auto_20171010_1644'), + ] + + operations = [ + migrations.AddField( + model_name='basefind', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='find', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalbasefind', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalfind', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicaltreatment', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicaltreatmentfile', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='property', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='treatment', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='treatmentfile', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/archaeological_finds/migrations/0011_auto_20171012_1316.py b/archaeological_finds/migrations/0011_auto_20171012_1316.py new file mode 100644 index 000000000..6fabd578f --- /dev/null +++ b/archaeological_finds/migrations/0011_auto_20171012_1316.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-12 13:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_finds', '0010_auto_20171011_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='basefind', + name='cache_complete_id', + field=models.TextField(blank=True, db_index=True, help_text='Cached value - do not edit', null=True, verbose_name='Complete ID'), + ), + migrations.AlterField( + model_name='basefind', + name='cache_short_id', + field=models.TextField(blank=True, db_index=True, help_text='Cached value - do not edit', null=True, verbose_name='Short ID'), + ), + migrations.AlterField( + model_name='find', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='historicalbasefind', + name='cache_complete_id', + field=models.TextField(blank=True, db_index=True, help_text='Cached value - do not edit', null=True, verbose_name='Complete ID'), + ), + migrations.AlterField( + model_name='historicalbasefind', + name='cache_short_id', + field=models.TextField(blank=True, db_index=True, help_text='Cached value - do not edit', null=True, verbose_name='Short ID'), + ), + migrations.AlterField( + model_name='historicalfind', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='historicaltreatment', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='historicaltreatmentfile', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='treatment', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='treatmentfile', + name='cached_label', + field=models.TextField(blank=True, db_index=True, null=True, verbose_name='Cached name'), + ), + ] diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py index e58d14f7e..8052601bf 100644 --- a/archaeological_finds/models_finds.py +++ b/archaeological_finds/models_finds.py @@ -231,14 +231,17 @@ class BaseFind(BulkUpdatedItem, BaseHistorizedItem, OwnPerms): line = models.LineStringField(_(u"Line"), blank=True, null=True) polygon = models.PolygonField(_(u"Polygon"), blank=True, null=True) cache_short_id = models.TextField( - _(u"Short ID"), blank=True, null=True, + _(u"Short ID"), blank=True, null=True, db_index=True, help_text=_(u"Cached value - do not edit")) cache_complete_id = models.TextField( - _(u"Complete ID"), blank=True, null=True, + _(u"Complete ID"), blank=True, null=True, db_index=True, help_text=_(u"Cached value - do not edit")) history = HistoricalRecords() RELATED_POST_PROCESS = ['find'] CACHED_LABELS = ['cache_short_id', 'cache_complete_id'] + PARENT_SEARCH_VECTORS = ['context_record'] + BASE_SEARCH_VECTORS = ["label", "description", "comment", "cache_short_id", + "cache_complete_id"] class Meta: verbose_name = _(u"Base find") @@ -748,9 +751,18 @@ class Find(BulkUpdatedItem, ValueGetter, BaseHistorizedItem, ImageModel, appraisal_date = models.DateField(_(u"Appraisal date"), blank=True, null=True) - cached_label = models.TextField(_(u"Cached name"), null=True, blank=True) + cached_label = models.TextField(_(u"Cached name"), null=True, blank=True, + db_index=True) history = HistoricalRecords() BASKET_MODEL = FindBasket + PARENT_SEARCH_VECTORS = ['base_finds'] + BASE_SEARCH_VECTORS = [ + "cached_label", "label", "description", "container__location__name", + "container__reference", "mark", "comment", "dating_comment", + "previous_id"] + M2M_SEARCH_VECTORS = [ + "datings__period__label", "object_types__label", "integrities__label", + "remarkabilities__label", "material_types__label"] class Meta: verbose_name = _(u"Find") diff --git a/archaeological_finds/models_treatments.py b/archaeological_finds/models_treatments.py index 0ffcd87fa..b4d98528b 100644 --- a/archaeological_finds/models_treatments.py +++ b/archaeological_finds/models_treatments.py @@ -115,7 +115,8 @@ class Treatment(DashboardFormItem, ValueGetter, BaseHistorizedItem, blank=True, null=True) target_is_basket = models.BooleanField(_(u"Target a basket"), default=False) - cached_label = models.TextField(_(u"Cached name"), null=True, blank=True) + cached_label = models.TextField(_(u"Cached name"), null=True, blank=True, + db_index=True) history = HistoricalRecords() class Meta: @@ -506,7 +507,8 @@ class TreatmentFile(DashboardFormItem, ClosedItem, BaseHistorizedItem, reception_date = models.DateField(_(u'Reception date'), blank=True, null=True) comment = models.TextField(_(u"Comment"), null=True, blank=True) - cached_label = models.TextField(_(u"Cached name"), null=True, blank=True) + cached_label = models.TextField(_(u"Cached name"), null=True, blank=True, + db_index=True) history = HistoricalRecords() class Meta: diff --git a/archaeological_operations/forms.py b/archaeological_operations/forms.py index 651cd740f..841131da6 100644 --- a/archaeological_operations/forms.py +++ b/archaeological_operations/forms.py @@ -480,6 +480,7 @@ RecordRelationsFormSet.form_label = _(u"Relations") class OperationSelect(TableSelect): + search_vector = forms.CharField(label=_(u"Full text search")) year = forms.IntegerField(label=_("Year")) operation_code = forms.IntegerField(label=_(u"Numeric reference")) if settings.COUNTRY == 'fr': diff --git a/archaeological_operations/migrations/0009_auto_20171011_1644.py b/archaeological_operations/migrations/0009_auto_20171011_1644.py new file mode 100644 index 000000000..18a284a21 --- /dev/null +++ b/archaeological_operations/migrations/0009_auto_20171011_1644.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_operations', '0008_auto_20170829_1639'), + ] + + operations = [ + migrations.AddField( + model_name='administrativeact', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='archaeologicalsite', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicaladministrativeact', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicaloperation', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='operation', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='parcel', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='parcelowner', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/archaeological_operations/migrations/0010_auto_20171012_1316.py b/archaeological_operations/migrations/0010_auto_20171012_1316.py new file mode 100644 index 000000000..3a847a803 --- /dev/null +++ b/archaeological_operations/migrations/0010_auto_20171012_1316.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-12 13:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_operations', '0009_auto_20171011_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='historicaloperation', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'), + ), + migrations.AlterField( + model_name='operation', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'), + ), + ] diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py index 54ed96cec..dd3e65534 100644 --- a/archaeological_operations/models.py +++ b/archaeological_operations/models.py @@ -248,6 +248,10 @@ class Operation(ClosedItem, BaseHistorizedItem, ImageModel, OwnPerms, 'archaeological_sites__reference': _(u"Archaeological sites (" u"reference)"), } + BASE_SEARCH_VECTORS = ["scientist__raw_name", "cached_label", + "common_name", "comment", "address", "old_code"] + INT_SEARCH_VECTORS = ["year"] + M2M_SEARCH_VECTORS = ["towns__name"] # fields definition creation_date = models.DateField(_(u"Creation date"), @@ -309,6 +313,7 @@ class Operation(ClosedItem, BaseHistorizedItem, ImageModel, OwnPerms, code_patriarche = models.TextField(u"Code PATRIARCHE", null=True, blank=True, unique=True) TABLE_COLS = ['full_code_patriarche'] + TABLE_COLS + BASE_SEARCH_VECTORS = ['code_patriarche'] + BASE_SEARCH_VECTORS # preventive fnap_financing = models.FloatField(u"Financement FNAP (%)", blank=True, null=True) @@ -340,7 +345,7 @@ class Operation(ClosedItem, BaseHistorizedItem, ImageModel, OwnPerms, scientific_documentation_comment = models.TextField( _(u"Comment about scientific documentation"), null=True, blank=True) cached_label = models.CharField(_(u"Cached name"), max_length=500, - null=True, blank=True) + null=True, blank=True, db_index=True) archaeological_sites = models.ManyToManyField( ArchaeologicalSite, verbose_name=_(u"Archaeological sites"), blank=True, related_name='operations') diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index 0d6908374..9d99ed67d 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -895,6 +895,21 @@ class OperationTest(TestCase, OperationInitTest): self.assertEqual(ope_id, 'OP2011-1') self.assertEqual(town, self.towns[0].name) + def test_search_vector_update(self): + operation = self.operations[0] + town = self.create_towns({'numero_insee': '12346', 'name': 'Daisy'})[-1] + operation.towns.add(town) + town = self.create_towns( + {'numero_insee': '12347', 'name': 'Dirty old'})[-1] + operation.towns.add(town) + operation = models.Operation.objects.get(pk=operation.pk) + operation.comment = u"Zardoz" + operation.code_patriarche = u"HUIAAA5" + operation.save() + for key in ('old', 'op2010', 'dirty', 'daisy', "'2010'", "zardoz", + "huiaaa5"): + self.assertIn(key, operation.search_vector) + def test_cache_bulk_update(self): if settings.USE_SPATIALITE_FOR_TESTS: # using views - can only be tested with postgresql @@ -1104,6 +1119,42 @@ class OperationSearchTest(TestCase, OperationInitTest): self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content)['total'], 1) + def test_town_search(self): + c = Client() + c.login(username=self.username, password=self.password) + + data = {'numero_insee': '98989', 'name': 'base_town'} + base_town = self.create_towns(datas=data)[-1] + + data = {'numero_insee': '56789', 'name': 'parent_town'} + parent_town = self.create_towns(datas=data)[-1] + parent_town.children.add(base_town) + + data = {'numero_insee': '01234', 'name': 'child_town'} + child_town = self.create_towns(datas=data)[-1] + base_town.children.add(child_town) + + ope = self.operations[1] + ope.towns.add(base_town) + + # simple search + search = {'towns': base_town.pk} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + + # parent search + search = {'towns': parent_town.pk} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + + # child search + search = {'towns': child_town.pk} + response = c.get(reverse('get-operation'), search) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content)['total'], 1) + def testOwnSearch(self): c = Client() response = c.get(reverse('get-operation'), {'year': '2010'}) diff --git a/archaeological_warehouse/migrations/0008_auto_20171011_1644.py b/archaeological_warehouse/migrations/0008_auto_20171011_1644.py new file mode 100644 index 000000000..82245647d --- /dev/null +++ b/archaeological_warehouse/migrations/0008_auto_20171011_1644.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_warehouse', '0007_auto_20171004_1125'), + ] + + operations = [ + migrations.AddField( + model_name='collection', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='container', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='warehouse', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AlterField( + model_name='container', + name='index', + field=models.IntegerField(default=0, verbose_name='Container ID'), + ), + ] diff --git a/archaeological_warehouse/migrations/0009_auto_20171012_1316.py b/archaeological_warehouse/migrations/0009_auto_20171012_1316.py new file mode 100644 index 000000000..a25a2d2f2 --- /dev/null +++ b/archaeological_warehouse/migrations/0009_auto_20171012_1316.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-12 13:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archaeological_warehouse', '0008_auto_20171011_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='container', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Localisation'), + ), + migrations.AlterField( + model_name='container', + name='cached_location', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached location'), + ), + ] diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py index 41891b341..71f31981a 100644 --- a/archaeological_warehouse/models.py +++ b/archaeological_warehouse/models.py @@ -283,9 +283,9 @@ class Container(LightHistorizedItem, ImageModel): reference = models.CharField(_(u"Container ref."), max_length=40) comment = models.TextField(_(u"Comment"), null=True, blank=True) cached_label = models.CharField(_(u"Localisation"), max_length=500, - null=True, blank=True) + null=True, blank=True, db_index=True) cached_location = models.CharField(_(u"Cached location"), max_length=500, - null=True, blank=True) + null=True, blank=True, db_index=True) index = models.IntegerField(u"Container ID", default=0) external_id = models.TextField(_(u"External ID"), blank=True, null=True) auto_external_id = models.BooleanField( diff --git a/example_project/settings.py b/example_project/settings.py index ea50daffb..6ca8cb5fc 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -240,6 +240,7 @@ ISHTAR_PERIODS = {} ISHTAR_PERMIT_TYPES = {} ISHTAR_DOC_TYPES = {u"undefined": u"Undefined"} +ISHTAR_SEARCH_LANGUAGE = "french" ISHTAR_DPTS = [] diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index cec61a51e..fbd9f4d29 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -20,6 +20,8 @@ import csv from ajax_select import make_ajax_form +from ajax_select.fields import AutoCompleteSelectField, \ + AutoCompleteSelectMultipleField from django.conf import settings from django.contrib import admin @@ -171,12 +173,31 @@ class PersonAdmin(HistorizedObjectAdmin): admin_site.register(models.Person, PersonAdmin) +class AdminRelatedTownForm(forms.ModelForm): + class Meta: + model = models.Town.children.through + exclude = [] + from_town = AutoCompleteSelectField( + 'town', required=True, label=_(u"Parent")) + + class AdminTownForm(forms.ModelForm): class Meta: model = models.Town exclude = [] center = PointField(label=_(u"center"), required=False, widget=OSMWidget) + children = AutoCompleteSelectMultipleField('town', required=False, + label=_(u"Town children")) + + +class TownParentInline(admin.TabularInline): + model = models.Town.children.through + fk_name = 'to_town' + form = AdminRelatedTownForm + verbose_name = _(u"Parent") + verbose_name_plural = _(u"Parents") + extra = 1 class TownAdmin(admin.ModelAdmin): @@ -186,9 +207,11 @@ class TownAdmin(admin.ModelAdmin): list_display += ['numero_insee', 'departement', ] search_fields += ['numero_insee', 'departement__label', ] list_filter = ("departement",) - readonly_fields = ['imports'] + readonly_fields = ['cached_label', 'imports'] model = models.Town form = AdminTownForm + inlines = [TownParentInline] + admin_site.register(models.Town, TownAdmin) diff --git a/ishtar_common/management/commands/update_search_vectors.py b/ishtar_common/management/commands/update_search_vectors.py new file mode 100644 index 000000000..c73a6e88e --- /dev/null +++ b/ishtar_common/management/commands/update_search_vectors.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from django.core.management.base import BaseCommand +import django.apps + + +class Command(BaseCommand): + help = "./manage.py update_search_vectors\n\n"\ + "Update full texte search vectors." + + def handle(self, *args, **options): + for model in django.apps.apps.get_models(): + if hasattr(model, "update_search_vector") and \ + getattr(model, "BASE_SEARCH_VECTORS", None): + self.stdout.write("\n* update {}".format(model)) + total = model.objects.count() + for idx, item in enumerate(model.objects.all()): + sys.stdout.write("\r{}/{} ".format(idx, total)) + sys.stdout.flush() + item.update_search_vector() + self.stdout.write("\n") diff --git a/ishtar_common/migrations/0015_auto_20171011_1644.py b/ishtar_common/migrations/0015_auto_20171011_1644.py new file mode 100644 index 000000000..a9f4499c2 --- /dev/null +++ b/ishtar_common/migrations/0015_auto_20171011_1644.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0014_ishtarsiteprofile_preservation'), + ] + + operations = [ + migrations.AddField( + model_name='historicalorganization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalperson', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='organization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='person', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/ishtar_common/migrations/0016_auto_20171016_1104.py b/ishtar_common/migrations/0016_auto_20171016_1104.py new file mode 100644 index 000000000..1d9209bdd --- /dev/null +++ b/ishtar_common/migrations/0016_auto_20171016_1104.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 11:04 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0015_auto_20171011_1644'), + ] + + operations = [ + migrations.AddField( + model_name='town', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'), + ), + migrations.AddField( + model_name='town', + name='children', + field=models.ManyToManyField(blank=True, related_name='parents', to='ishtar_common.Town', verbose_name='Town children'), + ), + migrations.AddField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='If not filled considered as the older town known.', null=True, verbose_name='Year of creation'), + ), + ] diff --git a/ishtar_common/migrations/0017_auto_20171016_1320.py b/ishtar_common/migrations/0017_auto_20171016_1320.py new file mode 100644 index 000000000..a48b36ce7 --- /dev/null +++ b/ishtar_common/migrations/0017_auto_20171016_1320.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 13:20 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0016_auto_20171016_1104'), + ] + + operations = [ + migrations.AlterField( + model_name='town', + name='numero_insee', + field=models.CharField(max_length=6, verbose_name='Num\xe9ro INSEE'), + ), + migrations.AlterField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='Filling this field is relevant to distinguish old towns to new towns.', null=True, verbose_name='Year of creation'), + ), + migrations.AlterUniqueTogether( + name='town', + unique_together=set([('numero_insee', 'year')]), + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 28a24115b..2904e51cd 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -35,6 +35,7 @@ import tempfile import time from django.conf import settings +from django.contrib.postgres.search import SearchVectorField, SearchVector from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.uploadedfile import SimpleUploadedFile @@ -58,7 +59,7 @@ from simple_history.models import HistoricalRecords as BaseHistoricalRecords from ishtar_common.model_merging import merge_model_objects from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\ - get_all_field_names + get_all_field_names, merge_tsvectors, cached_label_changed from ishtar_common.models_imports import ImporterModel, ImporterType, \ ImporterDefault, ImporterDefaultValues, ImporterColumn, \ @@ -917,9 +918,85 @@ class Imported(models.Model): abstract = True -class BaseHistorizedItem(Imported): +class FullSearch(models.Model): + search_vector = SearchVectorField(_("Search vector"), blank=True, null=True, + help_text=_("Auto filled at save")) + BASE_SEARCH_VECTORS = [] + INT_SEARCH_VECTORS = [] + M2M_SEARCH_VECTORS = [] + PARENT_SEARCH_VECTORS = [] + + class Meta: + abstract = True + + def update_search_vector(self, save=True): + """ + Update the search vector + :param save: True if you want to save the object immediately + :return: True if modified + """ + if not self.BASE_SEARCH_VECTORS and not self.M2M_SEARCH_VECTORS: + logger.warning("No search_vectors defined for {}".format( + self.__class__)) + return + if getattr(self, '_search_updated', None): + return + self._search_updated = True + + old_search = "" + if self.search_vector: + old_search = self.search_vector[:] + search_vectors = [] + base_q = self.__class__.objects.filter(pk=self.pk) + + # many to many have to be queried one by one otherwise only one is fetch + for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS: + key = M2M_SEARCH_VECTOR.split('__')[0] + rel_key = getattr(self, key) + for item in rel_key.values('pk').all(): + query_dct = {key + "__pk": item['pk']} + q = copy.copy(base_q).filter(**query_dct) + q = q.annotate( + search=SearchVector( + M2M_SEARCH_VECTOR, + config=settings.ISHTAR_SEARCH_LANGUAGE) + ).values('search') + search_vectors.append(q.all()[0]['search']) + + # int/float are not well managed by the SearchVector + for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS: + q = base_q.values(INT_SEARCH_VECTOR) + search_vectors.append( + "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR])) + + # copy parent vector fields + for PARENT_SEARCH_VECTOR in self.PARENT_SEARCH_VECTORS: + parent = getattr(self, PARENT_SEARCH_VECTOR) + if hasattr(parent, 'all'): # m2m + for p in parent.all(): + search_vectors.append(p.search_vector) + else: + search_vectors.append(parent.search_vector) + + # query "simple" fields + q = base_q.annotate( + search=SearchVector( + *self.BASE_SEARCH_VECTORS, + config=settings.ISHTAR_SEARCH_LANGUAGE + )).values('search') + search_vectors.append(q.all()[0]['search']) + self.search_vector = merge_tsvectors(search_vectors) + changed = old_search != self.search_vector + if save and changed: + self.skip_history_when_saving = True + self.save() + return changed + + +class BaseHistorizedItem(FullSearch, Imported): """ - Historized item with external ID management + Historized item with external ID management. + All historized items are searcheable """ IS_BASKET = False EXTERNAL_ID_KEY = '' @@ -1187,6 +1264,7 @@ class LightHistorizedItem(BaseHistorizedItem): super(LightHistorizedItem, self).save(*args, **kwargs) return True + PARSE_FORMULA = re.compile("{([^}]*)}") FORMULA_FILTERS = { @@ -1409,6 +1487,7 @@ def get_current_profile(force=False): def cached_site_changed(sender, **kwargs): get_current_profile(force=True) + post_save.connect(cached_site_changed, sender=IshtarSiteProfile) post_delete.connect(cached_site_changed, sender=IshtarSiteProfile) @@ -2490,12 +2569,20 @@ class Town(Imported, models.Model): center = models.PointField(_(u"Localisation"), srid=settings.SRID, blank=True, null=True) if settings.COUNTRY == 'fr': - numero_insee = models.CharField(u"Numéro INSEE", max_length=6, - unique=True) + numero_insee = models.CharField(u"Numéro INSEE", max_length=6) departement = models.ForeignKey( Department, verbose_name=u"Département", null=True, blank=True) canton = models.ForeignKey(Canton, verbose_name=u"Canton", null=True, blank=True) + year = models.IntegerField( + _("Year of creation"), null=True, blank=True, + help_text=_(u"Filling this field is relevant to distinguish old towns " + u"to new towns.")) + children = models.ManyToManyField( + 'Town', verbose_name=_(u"Town children"), blank=True, + related_name='parents') + cached_label = models.CharField(_(u"Cached name"), max_length=500, + null=True, blank=True, db_index=True) objects = models.GeoManager() class Meta: @@ -2503,11 +2590,24 @@ class Town(Imported, models.Model): verbose_name_plural = _(u"Towns") if settings.COUNTRY == 'fr': ordering = ['numero_insee'] + unique_together = (('numero_insee', 'year'),) def __unicode__(self): + if self.cached_label: + return self.cached_label + self.save() + return self.cached_label + + def _generate_cached_label(self): + cached_label = self.name if settings.COUNTRY == "fr": - return u"%s (%s)" % (self.name, self.numero_insee[:2]) - return self.name + cached_label = u"%s - %s" % (self.name, self.numero_insee[:2]) + if self.year: + cached_label += " ({})".format(self.year) + return cached_label + + +post_save.connect(cached_label_changed, sender=Town) class OperationType(GeneralType): diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 349408465..8a951b29a 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -1046,6 +1046,15 @@ class IshtarBasicTest(TestCase): self.assertEqual(response.status_code, 200) self.assertIn('class="sheet"', response.content) + def test_town_cache(self): + models.Town.objects.create(name="Sin City", numero_insee="99999") + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99") + town.year = 2050 + town.save() + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99 (2050)") + class GeomaticTest(TestCase): def test_post_save_point(self): diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index c6a4032f0..5d9e85c60 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -104,9 +104,12 @@ def cached_label_changed(sender, **kwargs): setattr(instance, cached_label, lbl) changed = True if changed: + instance._search_updated = False if hasattr(instance, '_cascade_change') and instance._cascade_change: instance.skip_history_when_saving = True instance.save() + if hasattr(instance, 'update_search_vector'): + instance.update_search_vector() updated = False if hasattr(instance, '_cached_labels_bulk_update'): updated = instance._cached_labels_bulk_update() @@ -117,6 +120,7 @@ def cached_label_changed(sender, **kwargs): item.test_obj = instance.test_obj cached_label_changed(item.__class__, instance=item) + SHORTIFY_STR = ugettext(" (...)") @@ -289,3 +293,43 @@ def get_all_related_objects(model): and f.auto_created and not f.concrete ] + +def merge_tsvectors(vectors): + """ + Parse tsvector to merge them in one string + :param vectors: list of tsvector string + :return: merged tsvector + """ + result_dict = {} + for vector in vectors: + if not vector: + continue + + current_position = 0 + if result_dict: + for key in result_dict: + max_position = max(result_dict[key]) + if max_position > current_position: + current_position = max_position + + for dct_member in vector.split(" "): + splitted = dct_member.split(':') + key = ":".join(splitted[:-1]) + positions = splitted[-1] + key = key[1:-1] # remove quotes + positions = [int(pos) + current_position + for pos in positions.split(',')] + if key in result_dict: + result_dict[key] += positions + else: + result_dict[key] = positions + + # {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"} + result_dict = {k: ",".join([str(val) for val in result_dict[k]]) + for k in result_dict} + # {'lamelie': "1,5", "hagarde": "2", "regarde": "4"} => + # "'lamelie':1,5 'hagarde':2 'regarde':4" + result = " ".join(["'{}':{}".format(k, result_dict[k]) + for k in result_dict]) + + return result diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 997acd7df..b8350c62a 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -41,6 +41,7 @@ from extra_views import ModelFormSetView from django.conf import settings from django.contrib.auth import logout from django.contrib.auth.decorators import login_required +from django.contrib.postgres.search import SearchQuery from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import Q, ImageField @@ -814,6 +815,23 @@ def get_item(model, func_name, default_name, extra_request_keys=[], dct.pop(k) # manage hierarchic conditions for req in dct.copy(): + if req.endswith('town__pk') or req.endswith('towns__pk'): + val = dct.pop(req) + reqs = Q(**{req: val}) + base_req = req[:-2] + '__' + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'parents__pk' + q = Q(**{req: val}) + reqs |= q + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'children__pk' + q = Q(**{req: val}) + reqs |= q + and_reqs.append(reqs) + continue + for k_hr in HIERARCHIC_FIELDS: if type(req) in (list, tuple): val = dct.pop(req) @@ -829,12 +847,15 @@ def get_item(model, func_name, default_name, extra_request_keys=[], val = dct.pop(req) reqs = Q(**{req: val}) req = req[:-2] + '__' - for idx in xrange(HIERARCHIC_LEVELS): + for idx in range(HIERARCHIC_LEVELS): req = req[:-2] + 'parent__pk' q = Q(**{req: val}) reqs |= q and_reqs.append(reqs) break + if 'search_vector' in dct: + dct['search_vector'] = SearchQuery( + dct['search_vector'], config=settings.ISHTAR_SEARCH_LANGUAGE) query = Q(**dct) for k, or_req in or_reqs: alt_dct = dct.copy() @@ -908,6 +929,9 @@ def get_item(model, func_name, default_name, extra_request_keys=[], items = model.objects.filter(query).distinct() # print(items.query) + if 'search_vector' in dct: # for serialization + dct['search_vector'] = dct['search_vector'].value + # table cols if own_table_cols: table_cols = own_table_cols |